├── LICENSE ├── README.md ├── bin ├── agentstart ├── agentstatus ├── agentstop ├── customstart ├── customstartall ├── customstop ├── customstopall ├── getdata ├── gpfdiststart ├── gpfdiststop ├── jobstart ├── jobstop ├── jobstopall ├── osstart ├── osstatus ├── osstop ├── start_all ├── stop_all ├── uistart └── uistop ├── config.properties ├── jar ├── gpdb.jar ├── nanohttpd.jar └── ojdbc6.jar ├── oracle2gp_compile.sh ├── oracle2gp_install.sh ├── oracle2gp_path.sh ├── sql ├── 01_create_ext_schema_extinstaller.sql ├── 02_create_os_schema_osinstaller.sql ├── 03_create_os_table_schedule_osinstaller.sql ├── 04_create_os_table_job_osinstaller.sql ├── 05_create_os_table_queue_osinstaller.sql ├── 06_create_variables.variables.sql ├── 07_create_os_table_ext_connection_osinstaller.sql ├── 08_create_osstop.gpfdist.sql ├── 09_create_osstart.gpfdist.sql ├── 10_create_osstatus.gpfdist.sql ├── 11_create_agentstop.gpfdist.sql ├── 12_create_agentstart.gpfdist.sql ├── 13_create_agentstatus.gpfdist.sql ├── 14_create_os_table_sessions_osinstaller.sql ├── 15_create_fn_get_variable.replace.sql ├── 16_create_fn_update_status.replace.sql ├── 17_create_fn_update_status2.replace.sql ├── 18_create_fn_queue_all.replace.sql ├── 19_create_fn_queue.replace.sql ├── 20_create_fn_replication_setup.replace.sql ├── 21_create_fn_replication.replace.sql ├── 22_create_fn_create_ext_table.replace.sql ├── 23_create_fn_start_schedule.replace.sql ├── 24_create_fn_schedule.replace.sql ├── 25_migrate.old_job.upgrade.sql ├── 26_migrate.new_job.upgrade.sql ├── 27_migrate.remaining.upgrade.sql ├── 28_create_sessions.gpfdist.sql ├── 29_create_uistop.gpfdist.sql ├── 30_create_uistart.gpfdist.sql ├── 31_create_fn_cancel_job.replace.sql ├── 32_create_custom_sql.install_os5.sql ├── 33_create_jobstart.gpfdist.sql ├── 34_create_jobstop.gpfdist.sql ├── 35_create_fn_jobstop.replace.sql ├── 36_create_customstart.gpfdist.sql ├── 37_create_customstop.gpfdist.sql ├── 38_create_fn_custom_stop.replace.sql ├── 39_create_fn_custom_startall.replace.sql ├── 40_fix_custom_sql.fix_os5.sql ├── 41_create_fn_create_ext_table.replace.sql └── distribution.txt ├── src ├── AgentD.java ├── CommonDB.java ├── CustomSQL.java ├── CustomSQLControl.java ├── CustomSQLModel.java ├── CustomSQLView.java ├── EnvironmentControl.java ├── EnvironmentModel.java ├── EnvironmentView.java ├── ExternalData.java ├── ExternalDataD.java ├── ExternalDataThread.java ├── ExternalTableControl.java ├── ExternalTableModel.java ├── ExternalTableView.java ├── GP.java ├── GpfdistRunner.java ├── JobControl.java ├── JobModel.java ├── JobView.java ├── LICENSE.txt ├── Logger.java ├── OSProperties.java ├── Oracle.java ├── OutsourcerControl.java ├── OutsourcerModel.java ├── OutsourcerView.java ├── QueueControl.java ├── QueueModel.java ├── QueueView.java ├── SQLServer.java ├── ScheduleControl.java ├── ScheduleModel.java ├── ScheduleView.java ├── ServerRunnerUI.java ├── UI.java ├── UIConnectionFactory.java ├── UIModel.java ├── UIView.java ├── fi │ └── iki │ │ └── elonen │ │ ├── NanoHTTPD$1$1.class │ │ ├── NanoHTTPD$1.class │ │ ├── NanoHTTPD$AsyncRunner.class │ │ ├── NanoHTTPD$DefaultAsyncRunner.class │ │ ├── NanoHTTPD$DefaultTempFile.class │ │ ├── NanoHTTPD$DefaultTempFileManager.class │ │ ├── NanoHTTPD$DefaultTempFileManagerFactory.class │ │ ├── NanoHTTPD$HTTPSession.class │ │ ├── NanoHTTPD$Method.class │ │ ├── NanoHTTPD$Response$Status.class │ │ ├── NanoHTTPD$Response.class │ │ ├── NanoHTTPD$TempFile.class │ │ ├── NanoHTTPD$TempFileManager.class │ │ ├── NanoHTTPD$TempFileManagerFactory.class │ │ ├── NanoHTTPD.class │ │ ├── ServerRunner.class │ │ ├── SimpleWebServer$1.class │ │ ├── SimpleWebServer$2.class │ │ └── SimpleWebServer.class └── manifest.txt └── yml └── oracle2gp.yml /README.md: -------------------------------------------------------------------------------- 1 | *********************************************************************************** 2 | 作者: 蒋守壮 3 | 4 | 个人邮箱: jiangshouzhuang@163.com 5 | 6 | CSDN博客: http://blog.csdn.net/jiangshouzhuang 7 | *********************************************************************************** 8 | 9 | Oracle2GP能够自动实现数据从Oracle迁移到Greenplum或HAWQ数据库。 10 | 11 | 实现的功能如下： 12 | 13 | 1. 数据库来源为Oracle 14 | 15 | 2. 能够自动将Oracle的DDL转换为Greenplum或HAWQ的语法，并自动在Greenplum中创建指定的schema和table 16 | 17 | 3. 自动实现从Oracle加载数据到Greenplum或HAWQ数据库 18 | 19 | 4. 目前暂时支持的作业类型为： 20 | 21 | * 更新：先truncate表再reload数据 22 | 23 | * 追加: 仅仅从Oracle抽取新增加的数据块 24 | 25 | * 复制：跟踪源数据的改变并应用这些改变(暂未实现) 26 | 27 | * 转换: 通过SQL命令在Greenplum或HAWQ中执行 28 | 29 | 5. 多线程动态调整作业数并行执行 30 | 31 | 6. 作业重复调度 32 | 33 | 7. 自动数据清洗 34 | 35 | 8. 对Greenplum自带工具gpfdist快速加载数据进行优化 36 | 37 | 安装步骤: 38 | 39 | source oracle2gp_path.sh 40 | 41 | bash oracle2gp_compile.sh 42 | 43 | bash oracle2gp_install.sh 44 | 45 | 46 | 目前基于Oracle 11g和Greenplum 4.3.x.x完全测试。 47 | -------------------------------------------------------------------------------- /bin/agentstart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer Agent start script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | ############################################################################################################################ 12 | # 13 | # Make sure Outsourcer Agent hasn't already started 14 | # 15 | ############################################################################################################################ 16 | check=$(ps -ef 2> /dev/null | grep $OSJAR | grep AgentD | grep -v grep | wc -l) 17 | 18 | if [ $check = 1 ]; then 19 | echo Outsourcer Agent has already started! 20 | exit 0 21 | fi 22 | ############################################################################################################################ 23 | # 24 | # Start Outsourcer Agent 25 | # 26 | ############################################################################################################################ 27 | echo Starting Outsourcer Scheduler Agent 28 | 29 | nohup java -classpath $OSAGENTCLASSPATH -Xms16M -Xmx128M AgentD $CONFIG > $AGENTLOG 2>&1 < $AGENTLOG & 30 | echo Outsourcer Agent Scheduler started 31 | -------------------------------------------------------------------------------- /bin/agentstatus: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer Agent status script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | # find the pid for Outsourcer and kill it 12 | pid=$(ps -ef 2> /dev/null | grep $OSJAR | grep AgentD | grep -v grep | awk '{ print $2 }') 13 | 14 | if [ $pid ]; then 15 | echo Up 16 | else 17 | echo Down 18 | fi 19 | -------------------------------------------------------------------------------- /bin/agentstop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer Agent stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | # find the pid for Outsourcer and kill it 12 | pid=$(ps -ef 2> /dev/null | grep $OSJAR | grep AgentD | grep -v grep | awk '{ print $2 }') 13 | 14 | if [ $pid ]; then 15 | echo Killing $pid 16 | kill $pid 17 | sleep 1 18 | fi 19 | 20 | 21 | echo Outsourcer Scheduler Agent has been stopped 22 | -------------------------------------------------------------------------------- /bin/customstart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer get gpfdist port script for custom tables 5 | # 6 | ############################################################################################################################ 7 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 8 | source $PWD/../oracle2gp_path.sh 9 | pid=0 10 | started=0 11 | 12 | # loop through available ports to find an available gpfdist port 13 | 14 | for myport in `seq $OSPORT_CUSTOM_LOWER $OSPORT_CUSTOM_UPPER`; do 15 | used="0" 16 | 17 | p=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v grep | grep $myport | awk -F ' ' '{print $12}' | wc -l` 18 | 19 | if [ "$p" -eq "0" ]; then 20 | mylog=$CUSTOMLOG"_$myport".log 21 | echo "customstart:$$:$myport:start requested" >> $mylog 22 | gpfdist -d $OSHOME -p $myport -t $GPFDISTTIMEOUT -c $YML -m $GPFDISTMAXROW >> $mylog 2>&1 < $mylog & 23 | pid=$! 24 | started="1" 25 | fi 26 | 27 | # check gpfdist process was started 28 | if [ "$pid" -ne "0" ]; then 29 | sleep 0.4 30 | count=`ps -ef 2> /dev/null | grep -v grep | awk -F ' ' '{print $2}' | grep $pid | wc -l` 31 | if [ "$count" -eq "1" ]; then 32 | echo "customstart:$$:$myport:started $pid" >> $mylog 33 | break 34 | else 35 | echo "customstart:$$:$myport:unable to start. Trying next port...." >> $mylog 36 | pid=0 37 | fi 38 | fi 39 | done 40 | # finished loop 41 | 42 | if [ "$pid" -eq "0" ]; then 43 | if [ "$mylog" != "" ]; then 44 | echo "customstart:$$:$myport:Failed to start a gpfdist process" >> $mylog 45 | fi 46 | myport=0 47 | fi 48 | 49 | if [ "$mylog" != "" ]; then 50 | echo "customstart:$$:$myport:done" >> $mylog 51 | fi 52 | 53 | echo $myport 54 | -------------------------------------------------------------------------------- /bin/customstartall: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer start gpfdist for custom tables script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | #stop any orphaned jobs 12 | customstopall 13 | 14 | #loop through custom table to find ports needed to be started 15 | java -classpath $OSCLASSPATH -Xms$XMS -Xmx$XMX CustomSQL $CONFIG start 16 | -------------------------------------------------------------------------------- /bin/customstop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist custom stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | myport=$1 11 | 12 | if [[ $myport -ge $OSPORT_CUSTOM_LOWER && $myport -le $OSPORT_CUSTOM_UPPER ]]; then 13 | mylog=$CUSTOMLOG"_$myport".log 14 | echo "customstop:$$:$myport:stopping" >> $mylog 15 | 16 | k=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v gpfdistcustomstop | grep -v grep | grep "$myport" | awk -F ' ' '{ print $2 }'` 17 | 18 | if [ "$k" != "" ]; then 19 | echo "customstop:$$:$myport:killing $k" >> $mylog 20 | echo "killing $k" 21 | kill $k 22 | exit 23 | else 24 | echo "ERROR: gpfdist job not running on port $myport" 25 | echo "customstop:$$:$myport:ERROR: gpfdist job not running on port $myport" >> $mylog 26 | fi 27 | else 28 | echo "Port number: $myport is outside range of $OSPORT_CUSTOM_LOWER and $OSPORT_CUSTOM_UPPER" 29 | fi 30 | while [ "$k" != "" ]; do 31 | k=`ps -ef 2> /dev/null | grep gpfdist | grep -v grep | grep "$myport" | awk -F ' ' '{ print $2 }'` 32 | echo "customstop:$$:$myport:stopping..." >> $mylog 33 | sleep 0.4 34 | done; 35 | 36 | if [ "$mylog" != "" ]; then 37 | echo "customstop:$$:$myport:done" >> $mylog 38 | fi 39 | 40 | -------------------------------------------------------------------------------- /bin/customstopall: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist custom stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | for i in `ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v grep | awk -F ' ' '{print $12}'`; do 12 | myport=$i 13 | if [[ $myport -ge $OSPORT_CUSTOM_LOWER && $myport -le $OSPORT_CUSTOM_UPPER ]]; then 14 | customstop $myport 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /bin/getdata: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer getdata script. 5 | # 6 | ############################################################################################################################ 7 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 8 | source $PWD/../oracle2gp_path.sh 9 | 10 | LANG="en_US.UTF-8" 11 | ############################################################################################################################ 12 | # External Tables using TRANSFORM with gpfdist 13 | # single parameter is passed to script so this is parsed 14 | ############################################################################################################################ 15 | parm1=`echo $1 | awk -F '+' '{print $1}'` 16 | parm2=`echo $1 | awk -F '+' '{print $2}'` 17 | parm3=`echo $1 | awk -F '+' '{print $3}'` 18 | parm4=`echo $1 | awk -F '+' '{print $4}'` 19 | parm5=`echo $1 | awk -F '+' '{print $5}'` 20 | 21 | if [ "$parm5" != "" ]; then 22 | #Outsourcer built tables have 5 parameters 23 | java -Dfile.encoding=UTF8 -classpath $OSCLASSPATH -Xms$XMS -Xmx$XMX -Djava.security.egd=file:/dev/./urandom ExternalData "$parm1" "$parm2" "$parm3" "$parm4" "$parm5" 24 | else 25 | #User defined custom external tables have 2 parameters 26 | java -Dfile.encoding=UTF8 -classpath $OSCLASSPATH -Xms$XMS -Xmx$XMX -Djava.security.egd=file:/dev/./urandom ExternalData "$parm1" "$parm2" 27 | fi 28 | -------------------------------------------------------------------------------- /bin/gpfdiststart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist start script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | mylog=$GPFDISTLOG 11 | pid=0 12 | 13 | k=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep $OSPORT | grep -v grep | awk -F ' ' '{ print $2 }'` 14 | 15 | if [ "$k" != "" ]; then 16 | echo "gpfdist already running! Stop first with gpfdiststop." 17 | else 18 | echo "gpfdiststart:$$:$OSPORT:Trying to start gpfdist" > $mylog 19 | gpfdist -d $OSHOME -p $OSPORT -t $GPFDISTTIMEOUT -c $YML >> $mylog 2>&1 < $mylog & 20 | pid=$! 21 | 22 | # Check log file to make sure gpfdist started 23 | count=0 24 | if [ "$pid" -ne "0" ]; then 25 | sleep 0.4 26 | count=`ps -ef 2> /dev/null | grep -v grep | awk -F ' ' '{print $2}' | grep $pid | wc -l` 27 | if [ "$count" -eq "1" ]; then 28 | echo "gpfdiststart:$$:$OSPORT:started $pid" >> $mylog 29 | echo "Directory: $OSHOME" 30 | echo "Port: $OSPORT" 31 | echo "YML: $YML" 32 | echo "Log: $GPFDISTLOG" 33 | else 34 | echo "gpfdiststart:$$:$OSPORT:unable to start." >> $mylog 35 | echo "Unable to start gpfdist on port $OSPORT" 36 | fi 37 | else 38 | echo "gpfdiststart:$$:$OSPORT:unable to create background process for gpfdist." >> $mylog 39 | echo "Unable to start background process for gpfdist on port $OSPORT" 40 | fi 41 | fi 42 | -------------------------------------------------------------------------------- /bin/gpfdiststop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | mylog=$GPFDISTLOG 11 | 12 | k=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep $OSPORT | grep -v grep | awk -F ' ' '{ print $2 }'` 13 | 14 | if [ "$k" != "" ]; then 15 | echo "gpfdiststop:$$:$OSPORT:Trying to stop gpfdist" >> $mylog 16 | echo "killing $k" 17 | kill $k 18 | echo "gpfdiststop:$$:$OSPORT:killed $k" >> $mylog 19 | else 20 | echo "gpfdist not running on port $OSPORT" 21 | fi 22 | -------------------------------------------------------------------------------- /bin/jobstart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist start script for Jobs 5 | # 6 | ############################################################################################################################ 7 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 8 | source $PWD/../oracle2gp_path.sh 9 | 10 | pid=0 11 | count=0 12 | myport=$OSPORT_LOWER 13 | mylog="" 14 | 15 | # create a lock file so that we only have one gpfdist processing starting/stopping at a time 16 | lock=false 17 | while [ "$lock" == "false" ]; do 18 | if mkdir $JOBLOCK 2> /dev/null; then 19 | #directory got made 20 | lock="true" 21 | fi 22 | done 23 | 24 | # loop through available ports to find an available gpfdist port 25 | for myport in `seq $OSPORT_LOWER $OSPORT_UPPER`; do 26 | p=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v grep | awk -F ' ' '{print $12}' | grep $myport | wc -l`; 27 | if [ "$p" -eq "0" ]; then 28 | mylog=$JOBLOG"_$myport".log 29 | echo "jobstart:$$:$myport:start requested" >> $mylog 30 | gpfdist -d $OSHOME -p $myport -t $GPFDISTTIMEOUT -c $YML -m $GPFDISTMAXROW >> $mylog 2>&1 < $mylog & 31 | pid=$! 32 | fi 33 | 34 | # check gpfdist process was started 35 | if [ "$pid" -ne "0" ]; then 36 | sleep 0.4 37 | count=`ps -ef 2> /dev/null | grep -v grep | awk -F ' ' '{print $2}' | grep $pid | wc -l` 38 | if [ "$count" -eq "1" ]; then 39 | echo "jobstart:$$:$myport:started $pid" >> $mylog 40 | break 41 | else 42 | echo "jobstart:$$:$myport:unable to start. Trying next port...." >> $mylog 43 | pid=0 44 | fi 45 | fi 46 | done 47 | # finished loop. 48 | 49 | if [ "$pid" -eq "0" ]; then 50 | if [ "$mylog" != "" ]; then 51 | echo "jobstart:$$:$myport:Failed to start a gpfdist process" >> $mylog 52 | fi 53 | myport=0 54 | fi 55 | if [ "$mylog" != "" ]; then 56 | echo "jobstart:$$:$myport:done" >> $mylog 57 | fi 58 | echo $myport 59 | 60 | rm -rf $JOBLOCK 61 | -------------------------------------------------------------------------------- /bin/jobstop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist job stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | myport=$1 11 | k="" 12 | mylog="" 13 | 14 | # create a lock file so that we only have one gpfdist processing starting/stopping at a time 15 | lock=false 16 | while [ "$lock" == "false" ]; do 17 | if mkdir $JOBLOCK 2> /dev/null; then 18 | #directory got made 19 | lock="true" 20 | fi 21 | done 22 | 23 | if [[ "$myport" -ge "$OSPORT_LOWER" && "$myport" -le "$OSPORT_UPPER" ]]; then 24 | mylog=$JOBLOG"_$myport".log 25 | echo "jobstop:$$:$myport:stopping" >> $mylog 26 | 27 | k=`ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v gpfdistjobstop | grep -v grep | grep "$myport" | awk -F ' ' '{ print $2 }'` 28 | 29 | if [ "$k" != "" ]; then 30 | echo "jobstop:$$:$myport:killing $k" >> $mylog 31 | echo "Killing $k" 32 | kill $k 33 | else 34 | echo "ERROR: gpfdist job not running on port $myport" 35 | echo "jobstop:$$:$myport:ERROR: gpfdist job not running on port $myport" >> $mylog 36 | fi 37 | else 38 | echo "ERROR: Port $myport is out of range of $OSPORT_LOWER and $OSPORT_UPPER" 39 | fi 40 | 41 | while [ "$k" != "" ]; do 42 | k=`ps -ef 2> /dev/null | grep gpfdist | grep -v grep | grep "$myport" | awk -F ' ' '{ print $2 }'` 43 | echo "jobstop:$$:$myport:stopping..." >> $mylog 44 | sleep 0.4 45 | done; 46 | 47 | if [ "$mylog" != "" ]; then 48 | echo "jobstop:$$:$myport:remove lock" >> $mylog 49 | fi 50 | 51 | rm -rf $JOBLOCK 52 | 53 | if [ "$mylog" != "" ]; then 54 | echo "jobstop:$$:$myport:done" >> $mylog 55 | fi 56 | -------------------------------------------------------------------------------- /bin/jobstopall: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer gpfdist custom stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | for i in `ps -ef 2> /dev/null | grep "$OSHOME" | grep gpfdist | grep -v grep | awk -F ' ' '{print $2 "," $12}'`; do 12 | mypid=`echo $i | awk -F ',' '{print $1}'` 13 | myport=`echo $i | awk -F ',' '{print $2}'` 14 | if [[ $myport -ge $OSPORT_LOWER && $myport -le $OSPORT_UPPER ]]; then 15 | echo "killing $mypid" 16 | kill $mypid 17 | fi 18 | done 19 | -------------------------------------------------------------------------------- /bin/osstart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer start script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | ############################################################################################################################ 12 | # 13 | # Make sure Outsourcer hasn't already started 14 | # 15 | ############################################################################################################################ 16 | check=$(ps -ef 2> /dev/null | grep $OSJAR | grep ExternalDataD | grep -v grep | wc -l) 17 | 18 | if [ $check = 1 ]; then 19 | echo "Outsourcer has already started!" 20 | exit 0 21 | fi 22 | 23 | echo "Starting Outsourcer..." 24 | 25 | # cleanup orphaned lock file 26 | echo "Removing lock file if present..." 27 | rm -rf $JOBLOCK 28 | 29 | echo "Removing old log files..." 30 | # remove old job files 31 | rm -f job $JOBLOG* 32 | 33 | ############################################################################################################################ 34 | # 35 | # Start Outsourcer 36 | # 37 | ############################################################################################################################ 38 | echo "Starting Outsourcer Queue Daemon..." 39 | 40 | nohup java -classpath $OSCLASSPATH -Xms$XMS -Xmx$XMX ExternalDataD $CONFIG start > $OSLOG 2>&1 < $OSLOG & 41 | echo "Outsourcer Queue Daemon started." 42 | echo "Starting gpfdist for Custom SQL Tables" 43 | customstartall 44 | -------------------------------------------------------------------------------- /bin/osstatus: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer Queue Daemon status script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | # find the pid for Outsourcer and kill it 12 | pid=$(ps -ef 2> /dev/null | grep $OSJAR | grep ExternalDataD | grep -v grep | awk '{ print $2 }') 13 | 14 | if [ $pid ]; then 15 | echo Up 16 | else 17 | echo Down 18 | fi 19 | -------------------------------------------------------------------------------- /bin/osstop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer stop script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | echo "Stopping Outsourcer..." 12 | java -classpath $OSCLASSPATH -Xms$XMS -Xmx$XMX ExternalDataD $CONFIG stop >> $OSLOG 13 | 14 | # find the pid for Outsourcer and kill it 15 | pid=$(ps -ef 2> /dev/null | grep $OSJAR | grep ExternalDataD | grep -v grep | awk '{ print $2 }') 16 | 17 | if [ $pid ]; then 18 | echo Killing $pid 19 | kill $pid 20 | sleep 1 21 | fi 22 | 23 | # cleanup orphaned lock file 24 | rm -rf $JOBLOCK 25 | 26 | echo "Outsourcer Queue Daemon has been stopped" 27 | 28 | # remove orphaned gpfdist processes from jobs 29 | 30 | echo "Stopping orphaned gpfdist processes from jobs" 31 | jobstopall 32 | 33 | echo "Stopping gpfdist processes for custom tables" 34 | customstopall 35 | -------------------------------------------------------------------------------- /bin/start_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer start all script 5 | # 6 | ############################################################################################################################ 7 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 8 | source $PWD/../oracle2gp_path.sh 9 | 10 | echo "############################################################################################################################" 11 | echo "#" 12 | echo "# Start UI" 13 | echo "#" 14 | echo "############################################################################################################################" 15 | uistart 16 | 17 | echo "############################################################################################################################" 18 | echo "#" 19 | echo "# Start Queue Daemon" 20 | echo "#" 21 | echo "############################################################################################################################" 22 | osstart 23 | 24 | echo "############################################################################################################################" 25 | echo "#" 26 | echo "# Start Scheduler Daemon" 27 | echo "#" 28 | echo "############################################################################################################################" 29 | agentstart 30 | 31 | -------------------------------------------------------------------------------- /bin/stop_all: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer stop all script 5 | # 6 | ############################################################################################################################ 7 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 8 | source $PWD/../oracle2gp_path.sh 9 | 10 | echo "############################################################################################################################" 11 | echo "#" 12 | echo "# Stop Scheduler Daemon" 13 | echo "#" 14 | echo "############################################################################################################################" 15 | agentstop 16 | 17 | echo "############################################################################################################################" 18 | echo "#" 19 | echo "# Stop Queue Daemon" 20 | echo "#" 21 | echo "############################################################################################################################" 22 | osstop 23 | 24 | echo "############################################################################################################################" 25 | echo "#" 26 | echo "# Stop UI" 27 | echo "#" 28 | echo "############################################################################################################################" 29 | uistop 30 | 31 | -------------------------------------------------------------------------------- /bin/uistart: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer UI start script. 5 | # 6 | ############################################################################################################################ 7 | set -e 8 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 9 | source $PWD/../oracle2gp_path.sh 10 | 11 | ############################################################################################################################ 12 | # 13 | # Make sure Outsourcer UI hasn't already started 14 | # 15 | ############################################################################################################################ 16 | check=$(ps -ef 2> /dev/null | grep $OSJAR | grep UI | grep -v grep | wc -l) 17 | 18 | if [ $check = 1 ]; then 19 | echo "Outsourcer UI has already started!" 20 | exit 0 21 | fi 22 | ############################################################################################################################ 23 | # 24 | # Reset the os.sessions table 25 | # 26 | ############################################################################################################################ 27 | 28 | if [ -f $SESSIONS ]; then 29 | rm $SESSIONS 30 | fi 31 | touch $SESSIONS 32 | 33 | echo "Removed old sessions" 34 | ############################################################################################################################ 35 | # 36 | # Start Outsourcer UI 37 | # 38 | ############################################################################################################################ 39 | echo "Starting Outsourcer UI..." 40 | 41 | nohup java -classpath $OSUICLASSPATH -Xms16M -Xmx128M UI $UIPORT $SESSIONS $CONFIG > $UILOG 2>&1 < $UILOG & 42 | 43 | gpfdiststart 44 | -------------------------------------------------------------------------------- /bin/uistop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ############################################################################################################################ 3 | # 4 | # Outsourcer UI stop script. 5 | # Be sure to have osconfig.sh in your path and set correctly. 6 | # 7 | ############################################################################################################################ 8 | 9 | set -e 10 | PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 11 | source $PWD/../oracle2gp_path.sh 12 | 13 | # find the pid for Outsourcer and kill it 14 | pid=$(ps -ef 2> /dev/null | grep $OSJAR | grep UI | grep -v grep | awk '{ print $2 }') 15 | 16 | if [ $pid ]; then 17 | echo Killing $pid 18 | kill $pid 19 | fi 20 | 21 | echo "Outsourcer UI has been stopped" 22 | 23 | gpfdiststop 24 | -------------------------------------------------------------------------------- /config.properties: -------------------------------------------------------------------------------- 1 | oshome=/home/gpadmin/Oracle2GP 2 | osport=38999 3 | osserver=cdha 4 | gpserver=cdha 5 | gpdatabase=zhangyun_db 6 | gpport=5432 7 | gpusername=zhangyun 8 | gppassword=zhangyun 9 | -------------------------------------------------------------------------------- /jar/gpdb.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/jar/gpdb.jar -------------------------------------------------------------------------------- /jar/nanohttpd.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/jar/nanohttpd.jar -------------------------------------------------------------------------------- /jar/ojdbc6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/jar/ojdbc6.jar -------------------------------------------------------------------------------- /oracle2gp_compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | #JAVA_HOME 5 | java_home=`echo $JAVA_HOME` 6 | if [ -z ${java_home} ] 7 | then 8 | echo "Please install Java Development Kit(JDK), for example: 1.7.0_67 or 1.7.0_80 version" 9 | exit 10 | else 11 | java -version > jdk_version_file 2>&1 12 | jdk_version=`cat jdk_version_file | grep -m 1 "java version" | awk -F\" '{print $2}'` 13 | rm -f jdk_version_file 14 | fi 15 | 16 | echo "Manifest-Version: 1.0" > manifest.txt 17 | echo "Main-Class: ExternalData" >> manifest.txt 18 | echo "Specification-Title: \"Outsourcer\"" >> manifest.txt 19 | echo "Specification-Version: \"1.0\"" >> manifest.txt 20 | echo "Created-By: ${jdk_version}" >> manifest.txt 21 | d=`date +"%Y-%m-%d %H:%M:%S"` 22 | echo "Build-Date: $d" >> manifest.txt 23 | 24 | chmod -R +x * 25 | 26 | cd src 27 | #javac -cp .:jar/gpdb.jar *.java 28 | javac -Xbootclasspath:${java_home}/jre/lib/rt.jar -source 1.7 -target 1.7 -cp .:../jar/gpdb.jar *.java 29 | jar cfm ../jar/Oracle2GP.jar manifest.txt Logger.class CommonDB.class Oracle.class SQLServer.class ExternalData.class CustomSQL.class GP.class ExternalDataD.class ExternalDataThread.class OSProperties.class GpfdistRunner.class 30 | jar cfm ../jar/Oracle2GPUI.jar manifest.txt *Model.class *View.class *Control.class UI*.class ServerRunnerUI.class 31 | jar cfm ../jar/Oracle2GPScheduler.jar manifest.txt AgentD.class 32 | 33 | cd .. 34 | #bin/stop_all 35 | #sleep 2 36 | #bin/start_all 37 | -------------------------------------------------------------------------------- /oracle2gp_path.sh: -------------------------------------------------------------------------------- 1 | ################################################################################################ 2 | #Oracle2GP的配置信息 3 | ################################################################################################ 4 | 5 | #Oracle2GP安装的家目录 6 | OSHOME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) 7 | export OSHOME 8 | 9 | #Oracle2GP图形化界面端口号 10 | export UIPORT=38090 11 | 12 | #gpfdist的端口号，如果在部署后再次修改该端口号，需要重新执行os_install.sh程序 13 | export OSPORT=38999 14 | 15 | #Oracle2GP的jobs端口号范围(确保此端口号没有被占用) 16 | export OSPORT_LOWER=31850 17 | export OSPORT_UPPER=31900 18 | 19 | #Oracle2GP的custom tables的端口号范围(确保此端口号没有被占用) 20 | export OSPORT_CUSTOM_LOWER=32000 21 | export OSPORT_CUSTOM_UPPER=33000 22 | 23 | #gpfdist的最大行长度，如果程序执行过程中遇到"line too long"错误那么请增加此值 24 | GPFDISTMAXROW=32768 25 | 26 | #Oracle2GP的lock文件路径 27 | export JOBLOCK=/tmp/oracle2gp-jobstart.lock 28 | 29 | #gpfdist程序的超时时间 30 | export GPFDISTTIMEOUT=30 31 | 32 | #Greenplum的yml文件路径 33 | export YML=$OSHOME/yml/oracle2gp.yml 34 | 35 | #Greenplum数据库的配置信息 36 | export CONFIG=$OSHOME/config.properties 37 | 38 | ################################################################################################ 39 | #Oracle2GP相关日志文件配置 40 | ################################################################################################ 41 | #Oracle2GP主程序的日志文件 42 | export OSLOG=$OSHOME/log/Oracle2GP.log 43 | 44 | #Oracle2GP的UI图形化日志文件 45 | export UILOG=$OSHOME/log/Oracle2GP-UI.log 46 | 47 | #Oracle2GP的Agent的日志文件 48 | export AGENTLOG=$OSHOME/log/Oracle2GP-Agent.log 49 | 50 | #Oracle2GP的gpfdist日志文件 51 | export GPFDISTLOG=$OSHOME/log/Oracle2GP-gpfdist.log 52 | 53 | #gpfdist作业的日志的前缀 54 | export JOBLOG=$OSHOME/log/job 55 | 56 | #自定义的gpfdist日志文件的前缀 57 | export CUSTOMLOG=$OSHOME/log/custom 58 | 59 | #Oracle2GP UI Sessions log file 60 | export SESSIONS=$OSHOME/log/Oracle2GP-sessions.log 61 | 62 | ################################################################################################ 63 | #Java内存配置 64 | ################################################################################################ 65 | #Oracle2GP的最小内存 66 | export XMS=256m 67 | 68 | #Oracle2GP的最大内存 69 | export XMX=768m 70 | 71 | ################################################################################################ 72 | #Oracle2GP所需要的Jar文件 73 | ################################################################################################ 74 | #Oracle2GP的Jar包 75 | OSJAR=$OSHOME/jar/Oracle2GP.jar 76 | 77 | #Oracle2GPScheduler的Jar包 78 | export OSAGENTJAR=$OSHOME/jar/Oracle2GPScheduler.jar 79 | 80 | #Oracle2GPUI的Jar包 81 | export OSUIJAR=$OSHOME/jar/Oracle2GPUI.jar 82 | 83 | #GPDB的Jar包 84 | export GPDBJAR=$OSHOME/jar/gpdb.jar 85 | 86 | #Microsoft JDBC的Jar包 87 | export MSJAR=$OSHOME/jar/sqljdbc4.jar 88 | 89 | #Oracle JDBC的Jar包 90 | export OJAR=$OSHOME/jar/ojdbc6.jar 91 | 92 | #Nano的Jar包 93 | export NANOJAR=$OSHOME/jar/nanohttpd.jar 94 | 95 | ################################################################################################ 96 | #Oracle2GP的PATH配置 97 | ################################################################################################ 98 | #Oracle2GP启动的Classpath 99 | export OSCLASSPATH=$OSJAR\:$MSJAR\:$OJAR\:$GPDBJAR 100 | 101 | #Oracle2GP UI启动的Classpath 102 | export OSUICLASSPATH=$OSJAR\:$OSUIJAR\:$MSJAR\:$OJAR\:$GPDBJAR\:$NANOJAR 103 | 104 | #Oracle2GP Agent启动的Classpath 105 | export OSAGENTCLASSPATH=$OSJAR\:$OSAGENTJAR\:$GPDBJAR 106 | 107 | #设置PATH 108 | export PATH=$OSHOME/bin:$PATH 109 | -------------------------------------------------------------------------------- /sql/01_create_ext_schema_extinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA ext; 2 | -------------------------------------------------------------------------------- /sql/02_create_os_schema_osinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA os; 2 | -------------------------------------------------------------------------------- /sql/03_create_os_table_schedule_osinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE os.ao_schedule 2 | (description text NOT NULL, 3 | interval_trunc text NOT NULL, 4 | interval_quantity text NOT NULL, 5 | deleted boolean NOT NULL DEFAULT FALSE, 6 | insert_id serial NOT NULL 7 | ) 8 | WITH (appendonly=true) 9 | :DISTRIBUTED_BY; 10 | --DISTRIBUTED BY (description); 11 | 12 | CREATE VIEW os.schedule AS 13 | SELECT description, interval_trunc, interval_quantity 14 | FROM ( 15 | SELECT description, interval_trunc, interval_quantity, row_number() OVER (PARTITION BY description ORDER BY insert_id DESC) AS rownum, deleted 16 | FROM os.ao_schedule 17 | ) AS sub 18 | WHERE rownum = 1 AND NOT deleted; 19 | 20 | INSERT INTO os.ao_schedule (description, interval_trunc, interval_quantity) VALUES ('Hourly', 'hour', '1 hour'); 21 | INSERT INTO os.ao_schedule (description, interval_trunc, interval_quantity) VALUES ('Daily', 'day', '1 day 4 hours'); 22 | INSERT INTO os.ao_schedule (description, interval_trunc, interval_quantity) VALUES ('Weekly', 'week', '1 week 4 hours'); 23 | INSERT INTO os.ao_schedule (description, interval_trunc, interval_quantity) VALUES ('Monthly', 'month', '1 month 4 hours'); 24 | INSERT INTO os.ao_schedule (description, interval_trunc, interval_quantity) VALUES ('5 Minutes', 'minute', '5 minutes'); 25 | -------------------------------------------------------------------------------- /sql/04_create_os_table_job_osinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE os.ao_job 2 | ( id serial NOT NULL, 3 | refresh_type text NOT NULL, 4 | target_schema_name text, 5 | target_table_name text, 6 | target_append_only boolean NOT NULL, 7 | target_compressed boolean NOT NULL, 8 | target_row_orientation boolean NOT NULL, 9 | source_type text, 10 | source_server_name text, 11 | source_instance_name text, 12 | source_port int, 13 | source_database_name text, 14 | source_schema_name text, 15 | source_table_name text, 16 | source_user_name text, 17 | source_pass text, 18 | column_name text, 19 | sql_text text, 20 | snapshot boolean NOT NULL, 21 | schedule_desc text, 22 | schedule_next timestamp, 23 | schedule_change boolean DEFAULT FALSE, 24 | 25 | deleted boolean NOT NULL DEFAULT FALSE, 26 | insert_id serial NOT NULL) 27 | WITH (appendonly=true) 28 | :DISTRIBUTED_BY; 29 | --DISTRIBUTED BY (id); 30 | 31 | CREATE VIEW os.job AS 32 | SELECT id, refresh_type, target_schema_name, target_table_name, target_append_only, 33 | target_compressed, target_row_orientation, source_type, source_server_name, 34 | source_instance_name, source_port, source_database_name, source_schema_name, 35 | source_table_name, source_user_name, source_pass, column_name, 36 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change 37 | FROM ( 38 | SELECT id, refresh_type, target_schema_name, target_table_name, target_append_only, 39 | target_compressed, target_row_orientation, source_type, source_server_name, 40 | source_instance_name, source_port, source_database_name, source_schema_name, 41 | source_table_name, source_user_name, source_pass, column_name, 42 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change, 43 | row_number() OVER (PARTITION BY id ORDER BY insert_id DESC) AS rownum, deleted 44 | FROM os.ao_job 45 | ) AS sub 46 | WHERE rownum = 1 AND NOT deleted; 47 | 48 | ALTER TABLE os.ao_job 49 | ADD CONSTRAINT job_check 50 | CHECK ((refresh_type = 'refresh'::text AND column_name IS NULL 51 | AND source_type IN ('oracle'::text, 'sqlserver'::text) AND source_server_name IS NOT NULL 52 | AND source_database_name IS NOT NULL AND source_schema_name IS NOT NULL 53 | AND source_table_name IS NOT NULL AND source_user_name IS NOT NULL AND source_pass IS NOT NULL 54 | AND target_schema_name IS NOT NULL AND target_table_name IS NOT NULL 55 | ) OR 56 | (refresh_type = 'append'::text AND column_name IS NOT NULL 57 | AND source_type IN ('oracle'::text, 'sqlserver'::text) AND source_server_name IS NOT NULL 58 | AND source_database_name IS NOT NULL AND source_schema_name IS NOT NULL 59 | AND source_table_name IS NOT NULL AND source_user_name IS NOT NULL AND source_pass IS NOT NULL 60 | AND target_schema_name IS NOT NULL AND target_table_name IS NOT NULL 61 | ) 62 | OR 63 | (refresh_type = 'replication'::text AND column_name IS NOT NULL 64 | AND source_type IN ('oracle'::text, 'sqlserver'::text) AND source_server_name IS NOT NULL 65 | AND source_database_name IS NOT NULL AND source_schema_name IS NOT NULL 66 | AND source_table_name IS NOT NULL AND source_user_name IS NOT NULL AND source_pass IS NOT NULL 67 | AND target_schema_name IS NOT NULL AND target_table_name IS NOT NULL 68 | ) OR 69 | (refresh_type = 'transform' AND source_type IS NULL AND source_server_name IS NULL 70 | AND source_instance_name IS NULL AND source_port IS NULL 71 | AND source_database_name IS NULL AND source_schema_name IS NULL AND source_table_name IS NULL AND source_user_name IS NULL 72 | AND source_pass IS NULL AND column_name IS NULL AND sql_text IS NOT NULL 73 | AND target_schema_name IS NULL AND target_table_name IS NULL 74 | ) OR 75 | (refresh_type = 'ddl'::text AND column_name IS NULL AND sql_text IS NULL 76 | AND source_type IN ('oracle'::text, 'sqlserver'::text) AND source_server_name IS NOT NULL 77 | AND source_database_name IS NOT NULL AND source_schema_name IS NOT NULL 78 | AND source_table_name IS NOT NULL AND source_user_name IS NOT NULL AND source_pass IS NOT NULL 79 | AND target_schema_name IS NOT NULL AND target_table_name IS NOT NULL 80 | ) 81 | ); 82 | 83 | ALTER TABLE os.ao_job ADD CONSTRAINT job_check_port 84 | CHECK ((source_port > 0 AND source_type = 'oracle'::text) OR (source_type <> 'oracle'::text AND source_port IS NULL)); 85 | 86 | ALTER TABLE os.ao_job ADD CONSTRAINT job_refresh_type 87 | CHECK (refresh_type in ('append', 'refresh', 'replication', 'transform', 'ddl')); 88 | -------------------------------------------------------------------------------- /sql/05_create_os_table_queue_osinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE os.ao_queue 2 | ( 3 | queue_id serial NOT NULL, 4 | status text NOT NULL check(status in ('queued', 'success', 'failed', 'processing')), 5 | queue_date timestamp without time zone NOT NULL, 6 | start_date timestamp without time zone, 7 | end_date timestamp, 8 | error_message text, 9 | num_rows int not null default 0, 10 | id int NOT NULL, 11 | refresh_type text NOT NULL, 12 | 13 | target_schema_name text, 14 | target_table_name text, 15 | target_append_only boolean NOT NULL, 16 | target_compressed boolean NOT NULL, 17 | target_row_orientation boolean NOT NULL, 18 | 19 | source_type text, 20 | source_server_name text, 21 | source_instance_name text, 22 | source_port int, 23 | source_database_name text, 24 | source_schema_name text, 25 | source_table_name text, 26 | source_user_name text, 27 | source_pass text, 28 | 29 | column_name text, 30 | sql_text text, 31 | snapshot boolean NOT NULL, 32 | deleted boolean NOT NULL DEFAULT FALSE, 33 | insert_id serial NOT NULL) 34 | WITH (appendonly=true) 35 | :DISTRIBUTED_BY; 36 | --DISTRIBUTED BY (queue_id); 37 | 38 | CREATE VIEW os.queue AS 39 | SELECT queue_id, status, queue_date, start_date, end_date, error_message, 40 | num_rows, id, refresh_type, target_schema_name, target_table_name, 41 | target_append_only, target_compressed, target_row_orientation, 42 | source_type, source_server_name, source_instance_name, source_port, 43 | source_database_name, source_schema_name, source_table_name, 44 | source_user_name, source_pass, column_name, sql_text, snapshot 45 | FROM ( 46 | SELECT queue_id, status, queue_date, start_date, end_date, error_message, 47 | num_rows, id, refresh_type, target_schema_name, target_table_name, 48 | target_append_only, target_compressed, target_row_orientation, 49 | source_type, source_server_name, source_instance_name, source_port, 50 | source_database_name, source_schema_name, source_table_name, 51 | source_user_name, source_pass, column_name, sql_text, snapshot, 52 | row_number() OVER (PARTITION BY queue_id ORDER BY insert_id DESC) AS rownum, deleted 53 | FROM os.ao_queue 54 | ) AS sub 55 | WHERE rownum = 1 AND NOT deleted; 56 | -------------------------------------------------------------------------------- /sql/06_create_variables.variables.sql: -------------------------------------------------------------------------------- 1 | DROP VIEW IF EXISTS os.variables; 2 | DROP TABLE IF EXISTS os.ao_variables; 3 | 4 | CREATE TABLE os.ao_variables 5 | ( name text NOT NULL, 6 | value text, 7 | restart boolean default true NOT NULL, 8 | deleted boolean NOT NULL DEFAULT FALSE, 9 | insert_id serial NOT NULL 10 | ) 11 | WITH (appendonly=true) 12 | :DISTRIBUTED_BY; 13 | --DISTRIBUTED BY (name); 14 | 15 | INSERT INTO os.ao_variables (name, value, restart) VALUES ('max_jobs', '4', false); 16 | INSERT INTO os.ao_variables (name, value, restart) VALUES ('oFetchSize', '2000', false); 17 | 18 | CREATE VIEW os.variables AS 19 | SELECT name, value, restart 20 | FROM ( 21 | SELECT name, value, restart, 22 | row_number() OVER (PARTITION BY name ORDER BY insert_id DESC) AS rownum, deleted 23 | FROM os.ao_variables 24 | ) AS sub 25 | WHERE rownum = 1 AND NOT deleted; 26 | -------------------------------------------------------------------------------- /sql/07_create_os_table_ext_connection_osinstaller.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE os.ao_ext_connection 2 | ( id serial NOT NULL, 3 | source_type text, 4 | source_server_name text NOT NULL, 5 | source_instance_name text, --optional by SQL Server 6 | source_port int, --used by Oracle 7 | source_database_name text, --used by Oracle 8 | source_user_name text NOT NULL, 9 | source_pass text NOT NULL, 10 | deleted boolean NOT NULL DEFAULT FALSE, 11 | insert_id serial NOT NULL) 12 | WITH (appendonly=true) 13 | :DISTRIBUTED_BY; 14 | --DISTRIBUTED BY (id); 15 | 16 | CREATE VIEW os.ext_connection AS 17 | SELECT id, source_type, source_server_name, source_instance_name, source_port, 18 | source_database_name, source_user_name, source_pass 19 | FROM ( 20 | SELECT id, source_type, source_server_name, source_instance_name, source_port, 21 | source_database_name, source_user_name, source_pass, 22 | row_number() OVER (PARTITION BY id ORDER BY insert_id DESC) AS rownum, deleted 23 | FROM os.ao_ext_connection 24 | ) AS sub 25 | WHERE rownum = 1 AND NOT deleted; 26 | 27 | ALTER TABLE os.ao_ext_connection 28 | ADD CONSTRAINT ext_check 29 | CHECK ( (source_type = 'oracle'::text AND source_database_name IS NOT NULL AND source_instance_name IS NULL) OR 30 | (source_type = 'sqlserver' AND source_database_name IS NULL) ); 31 | 32 | ALTER TABLE os.ao_ext_connection ADD CONSTRAINT ext_check_port 33 | CHECK ((source_port > 0 AND source_type = 'oracle'::text) OR (source_type = 'sqlserver'::text AND source_port IS NULL)); 34 | -------------------------------------------------------------------------------- /sql/08_create_osstop.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.osstop; 2 | 3 | CREATE EXTERNAL TABLE os.osstop 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/09_create_osstart.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.osstart; 2 | 3 | CREATE EXTERNAL TABLE os.osstart 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/10_create_osstatus.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.osstatus; 2 | 3 | CREATE EXTERNAL TABLE os.osstatus 4 | (status text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/11_create_agentstop.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.agentstop; 2 | 3 | CREATE EXTERNAL TABLE os.agentstop 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/12_create_agentstart.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.agentstart; 2 | 3 | CREATE EXTERNAL TABLE os.agentstart 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/13_create_agentstatus.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.agentstatus; 2 | 3 | CREATE EXTERNAL TABLE os.agentstatus 4 | (status text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/14_create_os_table_sessions_osinstaller.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS os.ao_sessions; 2 | -------------------------------------------------------------------------------- /sql/15_create_fn_get_variable.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_get_variable(p_name text) 2 | RETURNS text AS 3 | $$ 4 | SELECT value FROM os.variables WHERE name = $1; 5 | $$ 6 | LANGUAGE sql; 7 | -------------------------------------------------------------------------------- /sql/16_create_fn_update_status.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_update_status() 2 | RETURNS os.queue AS 3 | $$ 4 | DECLARE 5 | /* used to update status to processing */ 6 | v_function_name character varying := 'os.fn_update_status'; 7 | v_location int; 8 | 9 | v_max int := 0; 10 | v_count int; 11 | 12 | v_rec os.queue%rowtype; 13 | 14 | BEGIN 15 | v_location := 1000; 16 | select max_jobs::int 17 | into v_max 18 | from os.fn_get_variable('max_jobs') as max_jobs; 19 | 20 | v_location := 1500; 21 | SELECT COUNT(*) INTO v_count 22 | FROM os.queue 23 | WHERE status = 'processing'; 24 | 25 | v_location := 2000; 26 | IF v_count < v_max THEN 27 | 28 | v_location := 2100; 29 | SELECT * INTO v_rec 30 | FROM os.queue 31 | WHERE status = 'queued' 32 | AND clock_timestamp()::timestamp > queue_date 33 | ORDER BY queue_date, id LIMIT 1; 34 | 35 | v_location := 2200; 36 | IF v_rec.id IS NOT NULL THEN 37 | v_location := 2300; 38 | v_rec.status = 'processing'; 39 | v_rec.start_date := clock_timestamp(); 40 | v_rec.end_date := null; 41 | v_rec.error_message := null; 42 | v_rec.num_rows := 0; 43 | 44 | v_location := 2400; 45 | INSERT INTO os.ao_queue 46 | (queue_id, status, queue_date, start_date, end_date, error_message, 47 | num_rows, id, refresh_type, target_schema_name, target_table_name, 48 | target_append_only, target_compressed, target_row_orientation, 49 | source_type, source_server_name, source_instance_name, source_port, 50 | source_database_name, source_schema_name, source_table_name, 51 | source_user_name, source_pass, column_name, sql_text, snapshot) 52 | VALUES 53 | (v_rec.queue_id, v_rec.status, v_rec.queue_date, v_rec.start_date, v_rec.end_date, v_rec.error_message, 54 | v_rec.num_rows, v_rec.id, v_rec.refresh_type, v_rec.target_schema_name, v_rec.target_table_name, 55 | v_rec.target_append_only, v_rec.target_compressed, v_rec.target_row_orientation, 56 | v_rec.source_type, v_rec.source_server_name, v_rec.source_instance_name, v_rec.source_port, 57 | v_rec.source_database_name, v_rec.source_schema_name, v_rec.source_table_name, 58 | v_rec.source_user_name, v_rec.source_pass, v_rec.column_name, v_rec.sql_text, v_rec.snapshot); 59 | 60 | END IF; 61 | 62 | END IF; 63 | 64 | RETURN v_rec; 65 | 66 | EXCEPTION 67 | WHEN OTHERS THEN 68 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 69 | END; 70 | $$ 71 | LANGUAGE plpgsql; 72 | -------------------------------------------------------------------------------- /sql/17_create_fn_update_status2.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_update_status(p_queue_id int, p_status text, p_queue_date timestamp, p_start_date timestamp, p_error_message text, 2 | p_num_rows int, p_id int, p_refresh_type text, p_target_schema_name text, p_target_table_name text, 3 | p_target_append_only boolean, p_target_compressed boolean, p_target_row_orientation boolean, 4 | p_source_type text, p_source_server_name text, source_instance_name text, p_source_port int, 5 | p_source_database_name text, p_source_schema_name text, p_source_table_name text, 6 | p_source_user_name text, p_source_pass text, p_column_name text, p_sql_text text, p_snapshot boolean) 7 | RETURNS void AS 8 | $$ 9 | 10 | INSERT INTO os.ao_queue 11 | (queue_id, status, queue_date, start_date, end_date, error_message, 12 | num_rows, id, refresh_type, target_schema_name, target_table_name, 13 | target_append_only, target_compressed, target_row_orientation, 14 | source_type, source_server_name, source_instance_name, source_port, 15 | source_database_name, source_schema_name, source_table_name, 16 | source_user_name, source_pass, column_name, sql_text, snapshot) 17 | VALUES 18 | ($1, $2, $3, $4, clock_timestamp(), $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, 19 | $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25); 20 | 21 | $$ 22 | LANGUAGE sql; 23 | -------------------------------------------------------------------------------- /sql/18_create_fn_queue_all.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_queue_all() 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | v_function_name text := 'os.fn_queue_all'; 6 | v_location int; 7 | BEGIN 8 | v_location := 1000; 9 | --insert only inserts jobs that aren't already in the queue as processing or queued 10 | INSERT INTO os.ao_queue(status, queue_date, id, refresh_type, 11 | target_schema_name, target_table_name, target_append_only, target_compressed, target_row_orientation, 12 | source_type, source_server_name, source_instance_name, source_port, source_database_name, 13 | source_schema_name, source_table_name, source_user_name, source_pass, 14 | column_name, sql_text, snapshot) 15 | SELECT 'queued' as status, clock_timestamp()::timestamp as queue_date, j.id, j.refresh_type, 16 | j.target_schema_name, j.target_table_name, j.target_append_only, j.target_compressed, j.target_row_orientation, 17 | j.source_type, j.source_server_name, j.source_instance_name, j.source_port, j.source_database_name, 18 | j.source_schema_name, j.source_table_name, j.source_user_name, j.source_pass, 19 | j.column_name, j.sql_text, j.snapshot 20 | FROM os.job j left join (SELECT queue_id, 21 | id 22 | FROM os.queue 23 | WHERE status in ('processing', 'queued')) q on j.id = q.id 24 | WHERE q.queue_id IS NULL 25 | ORDER BY j.id; 26 | 27 | EXCEPTION 28 | WHEN OTHERS THEN 29 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 30 | END; 31 | $$ 32 | LANGUAGE plpgsql; 33 | -------------------------------------------------------------------------------- /sql/19_create_fn_queue.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_queue(p_id integer) 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | v_function_name text := 'os.fn_queue'; 6 | v_location int; 7 | BEGIN 8 | v_location := 1000; 9 | INSERT INTO os.ao_queue(status, queue_date, id, refresh_type, 10 | target_schema_name, target_table_name, target_append_only, target_compressed, target_row_orientation, 11 | source_type, source_server_name, source_instance_name, source_port, source_database_name, 12 | source_schema_name, source_table_name, source_user_name, source_pass, 13 | column_name, sql_text, snapshot) 14 | SELECT 'queued' as status, clock_timestamp()::timestamp as queue_date, j.id, j.refresh_type, 15 | j.target_schema_name, j.target_table_name, j.target_append_only, j.target_compressed, j.target_row_orientation, 16 | j.source_type, j.source_server_name, j.source_instance_name, j.source_port, j.source_database_name, 17 | j.source_schema_name, j.source_table_name, j.source_user_name, j.source_pass, 18 | j.column_name, j.sql_text, j.snapshot 19 | FROM os.job j left join (SELECT queue_id, 20 | id 21 | FROM os.queue 22 | WHERE status in ('processing', 'queued')) q on j.id = q.id 23 | WHERE q.queue_id IS NULL 24 | AND j.id = p_id; 25 | EXCEPTION 26 | WHEN OTHERS THEN 27 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 28 | END; 29 | $$ 30 | LANGUAGE plpgsql; 31 | -------------------------------------------------------------------------------- /sql/20_create_fn_replication_setup.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_replication_setup(p_target_schema text, p_target_table text, p_ext_schema text, p_stage_table text, p_arch_table text, p_column_name text) 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | /* This function is created but NOT used with HAWQ. DELETE is not supported by HAWQ */ 6 | v_function text := 'os.fn_replication_setup'; 7 | v_location int; 8 | v_sql text; 9 | v_debug BOOLEAN := true; 10 | 11 | BEGIN 12 | v_location := 1000; 13 | v_sql := 'DROP TABLE IF EXISTS "' || p_ext_schema || '"."' || p_stage_table || '"'; 14 | 15 | IF v_debug THEN 16 | RAISE INFO '%:%', v_location, v_sql; 17 | END IF; 18 | 19 | EXECUTE v_sql; 20 | 21 | v_location := 1500; 22 | v_sql := 'CREATE TABLE "' || p_ext_schema || '"."' || p_stage_table || '" ' || 23 | '("' || LOWER(p_column_name) || '" bigint not null, ' || 24 | 'change_type char(1) NOT NULL, ' || 25 | 'LIKE "' || p_target_schema || '"."' || p_target_table || '") ' || 26 | 'WITH (APPENDONLY=true) ' || 27 | 'DISTRIBUTED BY ("' || LOWER(p_column_name) || '")'; 28 | IF v_debug THEN 29 | RAISE INFO '%:%', v_location, v_sql; 30 | END IF; 31 | 32 | EXECUTE v_sql; 33 | 34 | v_location := 2000; 35 | v_sql := 'DROP TABLE IF EXISTS "' || p_ext_schema || '"."' || p_arch_table || '"'; 36 | 37 | IF v_debug THEN 38 | RAISE INFO '%:%', v_location, v_sql; 39 | END IF; 40 | 41 | EXECUTE v_sql; 42 | 43 | v_location := 2500; 44 | v_sql := 'CREATE TABLE "' || p_ext_schema || '"."' || p_arch_table || '" ' || 45 | '("' || LOWER(p_column_name) || '" bigint not null, ' || 46 | 'change_type char(1) NOT NULL, ' || 47 | 'LIKE "' || p_target_schema || '"."' || p_target_table || '") ' || 48 | 'WITH (APPENDONLY=true) ' || 49 | 'DISTRIBUTED BY ("' || LOWER(p_column_name) || '")'; 50 | IF v_debug THEN 51 | RAISE INFO '%:%', v_location, v_sql; 52 | END IF; 53 | 54 | EXECUTE v_sql; 55 | 56 | EXCEPTION 57 | WHEN OTHERS THEN 58 | RAISE EXCEPTION '(%:%:%)', v_function, v_location, sqlerrm; 59 | END; 60 | $$ 61 | LANGUAGE plpgsql VOLATILE; 62 | -------------------------------------------------------------------------------- /sql/21_create_fn_replication.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_replication(p_target_schema text, p_target_table text, p_stage_schema text, p_stage_table text, p_append_column_name text) 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | /* This function is created but NOT used with HAWQ. DELETE is not supported by HAWQ */ 6 | v_location INT; 7 | v_procedure VARCHAR := 'os.fn_replication'; 8 | 9 | v_debug BOOLEAN := true; 10 | v_rec RECORD; 11 | v_sql TEXT; 12 | v_columns TEXT; 13 | v_pk_columns TEXT; 14 | v_pk_delete text; 15 | i INT := 0; 16 | v_arch_table TEXT; 17 | v_stage_table TEXT; 18 | 19 | BEGIN 20 | v_location := 1000; 21 | --take the columns from the primary key and create the join statement for the DELETE below 22 | 23 | FOR v_rec IN ( SELECT a.attname::text AS column_name 24 | FROM ( 25 | SELECT generate_series(1, array_upper(sub.attrnums,1)) as x, sub.attrnums, sub.localoid 26 | FROM ( 27 | SELECT g.attrnums, g.localoid 28 | FROM gp_distribution_policy g 29 | JOIN pg_class c on g.localoid = c.oid 30 | JOIN pg_namespace n on c.relnamespace = n.oid 31 | WHERE n.nspname = p_target_schema 32 | AND c.relname = p_target_table 33 | ) AS sub 34 | ) AS sub2 35 | JOIN pg_attribute a ON sub2.localoid = a.attrelid 36 | AND attrnums[X] = a.attnum) LOOP 37 | 38 | i := i + 1; 39 | 40 | IF i = 1 THEN 41 | v_pk_delete := 'x.' || v_rec.column_name || ' = y.' || v_rec.column_name; 42 | v_pk_columns := v_rec.column_name; 43 | ELSE 44 | v_pk_delete := v_pk_delete || ' AND x.' || v_rec.column_name || ' = y.' || v_rec.column_name; 45 | v_pk_columns := v_pk_columns || ', ' || v_rec.column_name; 46 | END IF; 47 | 48 | END LOOP; 49 | 50 | IF v_debug THEN 51 | RAISE INFO '%:%', v_location, v_pk_delete; 52 | END IF; 53 | 54 | --create the temp table that is distributed the same as the target table 55 | v_location := 2000; 56 | v_stage_table := p_stage_table || '_stage'; 57 | v_sql := 'CREATE TEMPORARY TABLE temp_' || p_stage_schema || '_' || p_stage_table || ' ON COMMIT DROP AS ' || 58 | 'SELECT * FROM "' || p_stage_schema || '"."' || p_stage_table || '" ' || 59 | 'DISTRIBUTED BY (' || v_pk_columns || ')'; 60 | 61 | v_location := 2200; 62 | IF v_debug THEN 63 | RAISE INFO '%:%', v_location, v_sql; 64 | END IF; 65 | EXECUTE v_sql; 66 | 67 | --delete the Update and Delete records in the Target Table 68 | v_location := 3500; 69 | v_sql := 'DELETE FROM ' || p_target_schema || '.' || p_target_table || ' x USING temp_' || p_stage_schema || '_' || p_stage_table || ' y ' || 70 | 'WHERE ' || v_pk_delete || ' AND change_type IN (''U'', ''D'')'; 71 | 72 | v_location := 3600; 73 | IF v_debug THEN 74 | RAISE INFO '%:%', v_location, v_sql; 75 | END IF; 76 | EXECUTE v_sql; 77 | 78 | --get the columns from the target table 79 | v_location := 3800; 80 | i := 0; 81 | FOR v_rec in SELECT column_name --, CASE WHEN data_type = 'character' THEN 'text' ELSE data_type END as data_type 82 | FROM information_schema.columns 83 | WHERE table_schema = p_target_schema 84 | AND table_name = p_target_table 85 | ORDER BY ordinal_position LOOP 86 | 87 | i := i + 1; 88 | IF i = 1 THEN 89 | v_columns := v_rec.column_name; 90 | ELSE 91 | v_columns := v_columns || ', ' || v_rec.column_name; 92 | END IF; 93 | 94 | END LOOP; 95 | 96 | IF v_debug THEN 97 | RAISE INFO '%:%', v_location, v_columns; 98 | END IF; 99 | --insert the Update and Insert records 100 | v_location := 4000; 101 | v_sql := 'INSERT INTO "' || p_target_schema || '"."' || p_target_table || '" ( ' || v_columns || ') ' || 102 | 'SELECT ' || v_columns || ' FROM (' || 103 | 'SELECT rank() over (partition by ' || v_pk_columns || ' order by "' || LOWER(p_append_column_name) || '" desc) as rank, ' || v_columns || ', change_type ' || 104 | 'FROM temp_' || p_stage_schema || '_' || p_stage_table || ' ) AS SUB1 ' || 105 | 'WHERE SUB1.rank = 1 AND change_type IN (''U'', ''I'')'; 106 | 107 | v_location := 4100; 108 | IF v_debug THEN 109 | RAISE INFO '%:%', v_location, v_sql; 110 | END IF; 111 | EXECUTE v_sql; 112 | 113 | v_location := 5000; 114 | v_arch_table := p_target_schema || '_' || p_target_table || '_arch'; 115 | 116 | v_sql := 'TRUNCATE TABLE "' || p_stage_schema || '"."' || v_arch_table || '"'; 117 | IF v_debug THEN 118 | RAISE INFO '%:%', v_location, v_sql; 119 | END IF; 120 | EXECUTE v_sql; 121 | 122 | v_location := 6000; 123 | v_sql := 'INSERT INTO "' || p_stage_schema || '"."' || v_arch_table || '" ' || 124 | 'SELECT * FROM "' || 'temp_' || p_stage_schema || '_' || p_stage_table || '"'; 125 | 126 | IF v_debug THEN 127 | RAISE INFO '%:%', v_location, v_sql; 128 | END IF; 129 | EXECUTE v_sql; 130 | 131 | EXCEPTION WHEN OTHERS THEN 132 | RAISE EXCEPTION'(%:%:%)', v_procedure, v_location, sqlerrm; 133 | END; 134 | $$ 135 | LANGUAGE plpgsql; 136 | -------------------------------------------------------------------------------- /sql/22_create_fn_create_ext_table.replace.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS os.fn_create_ext_table(text, text[], integer, text); 2 | DROP FUNCTION IF EXISTS os.fn_create_ext_table(integer); 3 | -------------------------------------------------------------------------------- /sql/23_create_fn_start_schedule.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_start_schedule() 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | v_function_name text := 'os.fn_start_schedule'; 6 | v_location int; 7 | BEGIN 8 | 9 | v_location := 1000; 10 | INSERT INTO os.ao_job 11 | (id, refresh_type, target_schema_name, target_table_name, target_append_only, 12 | target_compressed, target_row_orientation, source_type, source_server_name, 13 | source_instance_name, source_port, source_database_name, source_schema_name, 14 | source_table_name, source_user_name, source_pass, column_name, 15 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change) 16 | SELECT j.id, refresh_type, target_schema_name, target_table_name, target_append_only, 17 | target_compressed, target_row_orientation, source_type, source_server_name, 18 | source_instance_name, source_port, source_database_name, source_schema_name, 19 | source_table_name, source_user_name, source_pass, column_name, 20 | sql_text, snapshot, schedule_desc, sub2.schedule_next, schedule_change 21 | FROM (SELECT j.id, min(exec_time) AS schedule_next 22 | FROM ( 23 | SELECT CASE WHEN interval_trunc = 'minute' THEN date_trunc('hour', now()::timestamp) + (interval_quantity::interval * i) 24 | ELSE date_trunc(interval_trunc, (now()::timestamp - ('1' || interval_trunc)::interval)) + (('1' || interval_trunc)::interval * i) + interval_quantity::interval END AS exec_time, 25 | description 26 | FROM os.schedule, generate_series(0, 60) AS i 27 | ) as sub 28 | JOIN os.job j ON sub.description = j.schedule_desc 29 | WHERE exec_time > now() 30 | GROUP BY j.id) as sub2, 31 | os.job j 32 | WHERE sub2.id = j.id; 33 | 34 | EXCEPTION 35 | WHEN OTHERS THEN 36 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 37 | END; 38 | $$ 39 | LANGUAGE plpgsql; 40 | -------------------------------------------------------------------------------- /sql/24_create_fn_schedule.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_schedule() 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | v_function_name text := 'os.fn_schedule'; 6 | v_location int; 7 | v_rec record; 8 | BEGIN 9 | v_location := 1000; 10 | --insert jobs into the queue that have a schedule_next less than now 11 | FOR v_rec IN (SELECT id FROM os.job WHERE schedule_next < clock_timestamp()::timestamp) LOOP 12 | INSERT INTO os.ao_job 13 | (id, refresh_type, target_schema_name, target_table_name, target_append_only, 14 | target_compressed, target_row_orientation, source_type, source_server_name, 15 | source_instance_name, source_port, source_database_name, source_schema_name, 16 | source_table_name, source_user_name, source_pass, column_name, 17 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change) 18 | SELECT id, refresh_type, target_schema_name, target_table_name, target_append_only, 19 | target_compressed, target_row_orientation, source_type, source_server_name, 20 | source_instance_name, source_port, source_database_name, source_schema_name, 21 | source_table_name, source_user_name, source_pass, column_name, 22 | sql_text, snapshot, schedule_desc, null::timestamp as schedule_next, schedule_change 23 | FROM os.job 24 | WHERE id = v_rec.id; 25 | PERFORM os.fn_queue(v_rec.id); 26 | END LOOP; 27 | 28 | v_location := 2000; 29 | --update the schedule_next for jobs that have completed 30 | INSERT INTO os.ao_job 31 | (id, refresh_type, target_schema_name, target_table_name, target_append_only, 32 | target_compressed, target_row_orientation, source_type, source_server_name, 33 | source_instance_name, source_port, source_database_name, source_schema_name, 34 | source_table_name, source_user_name, source_pass, column_name, 35 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change) 36 | SELECT j.id, refresh_type, target_schema_name, target_table_name, target_append_only, 37 | target_compressed, target_row_orientation, source_type, source_server_name, 38 | source_instance_name, source_port, source_database_name, source_schema_name, 39 | source_table_name, source_user_name, source_pass, column_name, 40 | sql_text, snapshot, schedule_desc, sub2.schedule_next, schedule_change 41 | FROM os.job j, 42 | (SELECT j.id, min(exec_time) AS schedule_next 43 | FROM ( 44 | SELECT CASE WHEN interval_trunc = 'minute' THEN date_trunc('hour', now()::timestamp) + (interval_quantity::interval * i) 45 | ELSE date_trunc(interval_trunc, (now()::timestamp - ('1' || interval_trunc)::interval)) + (('1' || interval_trunc)::interval * i) + interval_quantity::interval END AS exec_time, 46 | description 47 | FROM os.schedule, generate_series(0, 60) AS i 48 | ) as sub 49 | JOIN os.job j ON sub.description = j.schedule_desc 50 | WHERE exec_time > now() 51 | GROUP BY j.id) as sub2 52 | WHERE sub2.id = j.id 53 | AND NOT EXISTS (SELECT NULL FROM os.queue q WHERE j.id = q.id AND status IN ('processing', 'queued')) 54 | AND j.schedule_next IS NULL; 55 | 56 | v_location := 3000; 57 | --update the schedule_next for jobs that have changed 58 | INSERT INTO os.ao_job 59 | (id, refresh_type, target_schema_name, target_table_name, target_append_only, 60 | target_compressed, target_row_orientation, source_type, source_server_name, 61 | source_instance_name, source_port, source_database_name, source_schema_name, 62 | source_table_name, source_user_name, source_pass, column_name, 63 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change) 64 | SELECT j.id, refresh_type, target_schema_name, target_table_name, target_append_only, 65 | target_compressed, target_row_orientation, source_type, source_server_name, 66 | source_instance_name, source_port, source_database_name, source_schema_name, 67 | source_table_name, source_user_name, source_pass, column_name, 68 | sql_text, snapshot, schedule_desc, sub2.schedule_next, false as schedule_change 69 | FROM os.job j, 70 | (SELECT j.id, min(exec_time) AS schedule_next 71 | FROM ( 72 | SELECT CASE WHEN interval_trunc = 'minute' THEN date_trunc('hour', now()::timestamp) + (interval_quantity::interval * i) 73 | ELSE date_trunc(interval_trunc, (now()::timestamp - ('1' || interval_trunc)::interval)) + (('1' || interval_trunc)::interval * i) + interval_quantity::interval END AS exec_time, 74 | description 75 | FROM os.schedule, generate_series(0, 60) AS i 76 | ) as sub 77 | JOIN os.job j ON sub.description = j.schedule_desc 78 | WHERE exec_time > now() 79 | GROUP BY j.id) AS sub2 80 | WHERE sub2.id = j.id 81 | AND NOT EXISTS (SELECT NULL FROM os.queue q WHERE j.id = q.id AND status IN ('processing', 'queued')) 82 | AND j.schedule_next IS NULL; 83 | 84 | v_location := 4000; 85 | --remove schedule_next if the schedule_desc is removed 86 | INSERT INTO os.ao_job 87 | (id, refresh_type, target_schema_name, target_table_name, target_append_only, 88 | target_compressed, target_row_orientation, source_type, source_server_name, 89 | source_instance_name, source_port, source_database_name, source_schema_name, 90 | source_table_name, source_user_name, source_pass, column_name, 91 | sql_text, snapshot, schedule_desc, schedule_next, schedule_change) 92 | SELECT id, refresh_type, target_schema_name, target_table_name, target_append_only, 93 | target_compressed, target_row_orientation, source_type, source_server_name, 94 | source_instance_name, source_port, source_database_name, source_schema_name, 95 | source_table_name, source_user_name, source_pass, column_name, 96 | sql_text, snapshot, schedule_desc, null::timestamp as schedule_next, schedule_change 97 | FROM os.job j 98 | WHERE schedule_desc IS NULL AND schedule_next IS NOT NULL; 99 | 100 | EXCEPTION 101 | WHEN OTHERS THEN 102 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 103 | END; 104 | $$ 105 | LANGUAGE plpgsql VOLATILE; 106 | -------------------------------------------------------------------------------- /sql/25_migrate.old_job.upgrade.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO os.ao_job 2 | (id, refresh_type, 3 | target_schema_name, target_table_name, target_append_only, target_compressed, target_row_orientation, 4 | source_type, source_server_name, source_instance_name, source_port, source_database_name, 5 | source_schema_name, source_table_name, source_user_name, source_pass, 6 | column_name, sql_text, snapshot) 7 | SELECT id, refresh_type, 8 | (target).schema_name, (target).table_name, false AS append_only, false AS target_compressed, true AS target_row_orientation, 9 | (source).type, (source).server_name, (source).instance_name, (source).port, (source).database_name, 10 | (source).schema_name, (source).table_name, (source).user_name, (source).pass, 11 | column_name, sql_text, coalesce(snapshot, false) AS snapshot 12 | FROM :os_backup.job 13 | WHERE refresh_type in ('append', 'refresh', 'replication', 'transform', 'ddl'); 14 | -------------------------------------------------------------------------------- /sql/26_migrate.new_job.upgrade.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO os.ao_job 2 | (id, refresh_type, 3 | target_schema_name, target_table_name, target_append_only, target_compressed, target_row_orientation, 4 | source_type, source_server_name, source_instance_name, source_port, source_database_name, 5 | source_schema_name, source_table_name, source_user_name, source_pass, 6 | column_name, sql_text, snapshot, 7 | schedule_desc) 8 | SELECT id, refresh_type, 9 | (target).schema_name, (target).table_name, false AS append_only, false AS target_compressed, true AS target_row_orientation, 10 | (source).type, (source).server_name, (source).instance_name, (source).port, (source).database_name, 11 | (source).schema_name, (source).table_name, (source).user_name, (source).pass, 12 | column_name, sql_text, coalesce(snapshot, false) AS snapshot, 13 | schedule_desc 14 | FROM :os_backup.job 15 | WHERE refresh_type in ('append', 'refresh', 'replication', 'transform', 'ddl'); 16 | -------------------------------------------------------------------------------- /sql/27_migrate.remaining.upgrade.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO os.ao_queue 2 | (queue_id, status, queue_date, start_date, end_date, error_message, num_rows, 3 | id, refresh_type, 4 | target_schema_name, target_table_name, target_append_only, target_compressed, target_row_orientation, 5 | source_type, source_server_name, source_instance_name, source_port, source_database_name, 6 | source_schema_name, source_table_name, source_user_name, source_pass, 7 | column_name, sql_text, snapshot) 8 | SELECT queue_id, status, queue_date, start_date, end_date, error_message, num_rows, 9 | id, refresh_type, 10 | (target).schema_name, (target).table_name, false as append_only, false as target_compressed, true as target_row_orientation, 11 | (source).type, (source).server_name, (source).instance_name, (source).port, (source).database_name, 12 | (source).schema_name, (source).table_name, (source).user_name, (source).pass, 13 | column_name, sql_text, coalesce(snapshot, false) AS snapshot 14 | FROM :os_backup.queue; 15 | 16 | INSERT INTO os.ao_ext_connection 17 | SELECT * FROM :os_backup.ext_connection; 18 | 19 | SELECT setval('os.ao_job_id_seq', i, true) 20 | FROM (SELECT max(id) AS i FROM :os_backup.job) AS sub; 21 | 22 | SELECT setval('os.ao_ext_connection_id_seq', i, true) 23 | FROM (SELECT max(id) AS i FROM :os_backup.ext_connection) AS sub; 24 | 25 | SELECT setval('os.ao_queue_queue_id_seq', i, true) 26 | FROM (SELECT max(queue_id) AS i FROM :os_backup.queue) AS sub; 27 | -------------------------------------------------------------------------------- /sql/28_create_sessions.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.sessions; 2 | 3 | CREATE EXTERNAL TABLE os.sessions 4 | ( 5 | session_id integer, 6 | expire_date timestamp without time zone 7 | ) 8 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 9 | -------------------------------------------------------------------------------- /sql/29_create_uistop.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.uistop; 2 | 3 | CREATE EXTERNAL TABLE os.uistop 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/30_create_uistart.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.uistart; 2 | 3 | CREATE EXTERNAL TABLE os.uistart 4 | (foo text) 5 | LOCATION (:LOCATION) FORMAT 'text' (DELIMITER '|'); 6 | -------------------------------------------------------------------------------- /sql/31_create_fn_cancel_job.replace.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION os.fn_cancel_job(p_id integer) 2 | RETURNS void AS 3 | $$ 4 | DECLARE 5 | v_function_name text := 'os.fn_cancel_job'; 6 | v_location int; 7 | v_procpid int; 8 | v_found boolean := FALSE; 9 | BEGIN 10 | v_location := 1000; 11 | --check for External Tables loading data 12 | WITH q as (SELECT 'INSERT INTO "' || target_schema_name || '"."' || target_table_name || '"' AS current_query 13 | FROM os.queue 14 | WHERE status = 'processing' 15 | AND id = p_id) 16 | SELECT p.procpid 17 | INTO v_procpid 18 | FROM pg_stat_activity p, q 19 | WHERE p.datname = current_database() 20 | AND p.current_query LIKE q.current_query || '%'; 21 | 22 | WHILE v_procpid IS NOT NULL LOOP 23 | v_found := TRUE; 24 | 25 | PERFORM pg_cancel_backend(v_procpid); 26 | 27 | PERFORM pg_sleep(1); 28 | 29 | WITH q as (SELECT 'INSERT INTO "' || target_schema_name || '"."' || target_table_name || '"' AS current_query 30 | FROM os.queue 31 | WHERE status = 'processing' 32 | AND id = p_id) 33 | SELECT p.procpid 34 | INTO v_procpid 35 | FROM pg_stat_activity p, q 36 | WHERE p.datname = current_database() 37 | AND p.current_query LIKE q.current_query || '%'; 38 | 39 | END LOOP; 40 | 41 | IF v_found = FALSE THEN 42 | v_location := 2000; 43 | --job not found so check for other types of processes like Transform steps 44 | WITH q AS (SELECT coalesce(replace(sql_text, ';', ''), target_table_name) as current_query 45 | FROM os.queue 46 | WHERE status = 'processing' 47 | AND id = p_id) 48 | SELECT p.procpid 49 | INTO v_procpid 50 | FROM pg_stat_activity p, q 51 | WHERE p.datname = current_database() 52 | AND POSITION(p.current_query IN q.current_query) > 0; 53 | 54 | WHILE v_procpid IS NOT NULL LOOP 55 | 56 | v_found := TRUE; 57 | 58 | PERFORM pg_cancel_backend(v_procpid); 59 | 60 | PERFORM pg_sleep(1); 61 | 62 | WITH q AS (SELECT coalesce(replace(sql_text, ';', ''), target_table_name) as current_query 63 | FROM os.queue 64 | WHERE status = 'processing' 65 | AND id = p_id) 66 | SELECT p.procpid 67 | INTO v_procpid 68 | FROM pg_stat_activity p, q 69 | WHERE p.datname = current_database() 70 | AND p.current_query LIKE '%' || q.current_query || '%'; 71 | END LOOP; 72 | END IF; 73 | 74 | IF v_found = FALSE THEN 75 | v_location := 3000; 76 | --process for job not found at all so mark as an orphaned job 77 | INSERT INTO os.ao_queue 78 | (queue_id, status, queue_date, start_date, end_date, error_message, 79 | num_rows, id, refresh_type, target_schema_name, target_table_name, 80 | target_append_only, target_compressed, target_row_orientation, 81 | source_type, source_server_name, source_instance_name, source_port, 82 | source_database_name, source_schema_name, source_table_name, 83 | source_user_name, source_pass, column_name, sql_text, snapshot) 84 | SELECT queue_id, 'failed' as status, queue_date, start_date, clock_timestamp() AS end_date, 85 | 'Database session for job not found!' AS error_message, 86 | 0 AS num_rows, id, refresh_type, target_schema_name, target_table_name, 87 | target_append_only, target_compressed, target_row_orientation, 88 | source_type, source_server_name, source_instance_name, source_port, 89 | source_database_name, source_schema_name, source_table_name, 90 | source_user_name, source_pass, column_name, sql_text, snapshot 91 | FROM os.queue q 92 | WHERE q.status = 'processing' 93 | AND q.id = p_id; 94 | 95 | END IF; 96 | 97 | EXCEPTION 98 | WHEN OTHERS THEN 99 | RAISE EXCEPTION '(%:%:%)', v_function_name, v_location, sqlerrm; 100 | END; 101 | $$ 102 | LANGUAGE plpgsql; 103 | 104 | -------------------------------------------------------------------------------- /sql/32_create_custom_sql.install_os5.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE os.ao_custom_sql 2 | ( id serial NOT NULL, 3 | table_name text NOT NULL, 4 | columns text[] NOT NULL, 5 | column_datatypes text[] NOT NULL, 6 | sql_text text NOT NULL, 7 | source_type text, 8 | source_server_name text NOT NULL, 9 | source_instance_name text, --optional by SQL Server 10 | source_port int, --used by Oracle 11 | source_database_name text, --used by Oracle 12 | source_user_name text NOT NULL, 13 | source_pass text NOT NULL, 14 | gpfdist_port int NOT NULL, 15 | deleted boolean NOT NULL DEFAULT FALSE, 16 | insert_id serial NOT NULL) 17 | WITH (appendonly=true) 18 | :DISTRIBUTED_BY; 19 | --DISTRIBUTED BY (id); 20 | 21 | CREATE VIEW os.custom_sql AS 22 | SELECT id, table_name, columns, column_datatypes, sql_text, source_type, source_server_name, source_instance_name, source_port, 23 | source_database_name, source_user_name, source_pass, gpfdist_port 24 | FROM ( 25 | SELECT id, table_name, columns, column_datatypes, sql_text, source_type, source_server_name, source_instance_name, source_port, 26 | source_database_name, source_user_name, source_pass, gpfdist_port, 27 | row_number() OVER (PARTITION BY id ORDER BY insert_id DESC) AS rownum, deleted 28 | FROM os.ao_custom_sql 29 | ) AS sub 30 | WHERE rownum = 1 AND NOT deleted; 31 | 32 | ALTER TABLE os.ao_custom_sql 33 | ADD CONSTRAINT custom_check 34 | CHECK ( (source_type = 'oracle'::text AND source_database_name IS NOT NULL AND source_instance_name IS NULL) OR 35 | (source_type = 'sqlserver' AND source_database_name IS NULL) ); 36 | 37 | ALTER TABLE os.ao_custom_sql ADD CONSTRAINT custom_check_port 38 | CHECK ((source_port > 0 AND source_type = 'oracle'::text) OR (source_type = 'sqlserver'::text AND source_port IS NULL)); 39 | -------------------------------------------------------------------------------- /sql/33_create_jobstart.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.jobstart; 2 | -------------------------------------------------------------------------------- /sql/34_create_jobstop.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.jobstop; 2 | -------------------------------------------------------------------------------- /sql/35_create_fn_jobstop.replace.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS os.fn_jobstop(text,int,int); 2 | -------------------------------------------------------------------------------- /sql/36_create_customstart.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.customstart; 2 | -------------------------------------------------------------------------------- /sql/37_create_customstop.gpfdist.sql: -------------------------------------------------------------------------------- 1 | DROP EXTERNAL TABLE IF EXISTS os.customstop; 2 | -------------------------------------------------------------------------------- /sql/38_create_fn_custom_stop.replace.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS os.fn_customstop(text, integer, integer); 2 | -------------------------------------------------------------------------------- /sql/39_create_fn_custom_startall.replace.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS os.fn_custom_startall(text, integer); 2 | -------------------------------------------------------------------------------- /sql/40_fix_custom_sql.fix_os5.sql: -------------------------------------------------------------------------------- 1 | DROP VIEW os.custom_sql; 2 | 3 | ALTER TABLE os.ao_custom_sql RENAME TO ao_custom_sql_backup; 4 | 5 | CREATE TABLE os.ao_custom_sql 6 | ( id serial NOT NULL, 7 | table_name text NOT NULL, 8 | columns text[] NOT NULL, 9 | column_datatypes text[] NOT NULL, 10 | sql_text text NOT NULL, 11 | source_type text, 12 | source_server_name text NOT NULL, 13 | source_instance_name text, --optional by SQL Server 14 | source_port int, --used by Oracle 15 | source_database_name text, --used by Oracle 16 | source_user_name text NOT NULL, 17 | source_pass text NOT NULL, 18 | gpfdist_port int NOT NULL, 19 | deleted boolean NOT NULL DEFAULT FALSE, 20 | insert_id serial NOT NULL) 21 | WITH (appendonly=true) 22 | :DISTRIBUTED_BY; 23 | --DISTRIBUTED BY (id); 24 | 25 | CREATE VIEW os.custom_sql AS 26 | SELECT id, table_name, columns, column_datatypes, sql_text, source_type, source_server_name, source_instance_name, source_port, 27 | source_database_name, source_user_name, source_pass, gpfdist_port 28 | FROM ( 29 | SELECT id, table_name, columns, column_datatypes, sql_text, source_type, source_server_name, source_instance_name, source_port, 30 | source_database_name, source_user_name, source_pass, gpfdist_port, 31 | row_number() OVER (PARTITION BY id ORDER BY insert_id DESC) AS rownum, deleted 32 | FROM os.ao_custom_sql 33 | ) AS sub 34 | WHERE rownum = 1 AND NOT deleted; 35 | 36 | ALTER TABLE os.ao_custom_sql 37 | ADD CONSTRAINT custom_check 38 | CHECK ( (source_type = 'oracle'::text AND source_database_name IS NOT NULL AND source_instance_name IS NULL) OR 39 | (source_type = 'sqlserver' AND source_database_name IS NULL) ); 40 | 41 | ALTER TABLE os.ao_custom_sql ADD CONSTRAINT custom_check_port 42 | CHECK ((source_port > 0 AND source_type = 'oracle'::text) OR (source_type = 'sqlserver'::text AND source_port IS NULL)); 43 | 44 | INSERT INTO os.ao_custom_sql 45 | (table_name, columns, column_datatypes, sql_text, source_type, 46 | source_server_name, source_instance_name, source_port, source_database_name, 47 | source_user_name, source_pass, gpfdist_port) 48 | SELECT table_name, columns, column_datatypes, sql_text, source_type, 49 | source_server_name, source_instance_name, source_port, source_database_name, 50 | source_user_name, source_pass, (:osport_custom_lower - 1 + row_number() over ())::int as gpfdist_port 51 | FROM os.ao_custom_sql_backup 52 | WHERE DELETED = FALSE; 53 | 54 | DROP TABLE os.ao_custom_sql_backup; 55 | -------------------------------------------------------------------------------- /sql/41_create_fn_create_ext_table.replace.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS os.fn_create_ext_table(integer, character varying); 2 | -------------------------------------------------------------------------------- /sql/distribution.txt: -------------------------------------------------------------------------------- 1 | 1|ao_schedule|description 2 | 2|ao_job|id 3 | 3|ao_queue|queue_id 4 | 4|ao_variables|name 5 | 5|ao_ext_connection|id 6 | 6|ao_custom_sql|id 7 | -------------------------------------------------------------------------------- /src/AgentD.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.net.*; 3 | import java.io.*; 4 | 5 | public class AgentD 6 | { 7 | private static String myclass = "AgentD"; 8 | public static boolean debug = false; 9 | 10 | public static void main(String[] args) throws Exception 11 | { 12 | 13 | String method = "main"; 14 | int location = 1000; 15 | 16 | String configFile = args[0]; 17 | Connection conn = null; 18 | 19 | location = 2000; 20 | try 21 | { 22 | location = 3000; 23 | conn = CommonDB.connectGP(configFile); 24 | 25 | location = 3100; 26 | Statement stmt = conn.createStatement(); 27 | 28 | location = 3200; 29 | String strSQL = "VACUUM os.job"; 30 | 31 | location = 3300; 32 | stmt.executeUpdate(strSQL); 33 | 34 | location = 3400; 35 | strSQL = "SELECT os.fn_start_schedule()"; 36 | 37 | location = 3500; 38 | stmt.executeQuery(strSQL); 39 | 40 | location = 3600; 41 | conn.close(); 42 | 43 | if (debug) 44 | Logger.printMsg("Start to loop...."); 45 | loadLoop(configFile); 46 | 47 | } 48 | catch (SQLException ex) 49 | { 50 | throw new SQLException("(" + myclass + ":" + method + ":" + location + ":" + ex.getMessage() + ")"); 51 | } 52 | finally 53 | { 54 | if (conn != null) 55 | conn.close(); 56 | } 57 | } 58 | 59 | private static void loadLoop(String configFile) throws Exception 60 | { 61 | 62 | String method = "loadLoop"; 63 | int location = 1000; 64 | 65 | boolean loop = true; 66 | Connection conn = null; 67 | 68 | while (loop) 69 | { 70 | try 71 | { 72 | location = 2000; 73 | conn = CommonDB.connectGP(configFile); 74 | 75 | location = 2100; 76 | Statement stmt = conn.createStatement(); 77 | 78 | location = 2200; 79 | String strSQL = "SELECT os.fn_schedule()"; 80 | 81 | location = 2300; 82 | stmt.executeQuery(strSQL); 83 | 84 | location = 2400; 85 | conn.close(); 86 | 87 | location = 3000; 88 | Thread.sleep(10000); 89 | 90 | } 91 | catch (SQLException ex) 92 | { 93 | Logger.printMsg("(" + method + ":" + location + ":" + ex.getMessage() + ")"); 94 | Thread.sleep(10000); 95 | } 96 | finally 97 | { 98 | if (conn != null) 99 | conn.close(); 100 | } 101 | } 102 | 103 | } 104 | } 105 | 106 | -------------------------------------------------------------------------------- /src/CustomSQL.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.net.*; 3 | import java.io.*; 4 | 5 | public class CustomSQL 6 | { 7 | private static String myclass = "CustomSQL"; 8 | public static boolean debug = true; 9 | public static String configFile = ""; 10 | 11 | public static void main(String[] args) throws Exception 12 | { 13 | 14 | String method = "main"; 15 | int location = 1000; 16 | int argsCount = args.length; 17 | 18 | configFile = args[0]; 19 | String action = args[1]; 20 | Connection conn = null; 21 | 22 | location = 2000; 23 | try 24 | { 25 | location = 3000; 26 | if (action.equals("start")) 27 | { 28 | conn = CommonDB.connectGP(configFile); 29 | startAll(conn); 30 | conn.close(); 31 | } 32 | } 33 | catch (SQLException ex) 34 | { 35 | throw new SQLException("(" + myclass + ":" + method + ":" + location + ":" + ex.getMessage() + ")"); 36 | } 37 | finally 38 | { 39 | if (conn != null) 40 | conn.close(); 41 | } 42 | } 43 | 44 | private static void startAll(Connection conn) throws SQLException 45 | { 46 | String id = ""; 47 | String tableName = ""; 48 | String columns = ""; 49 | String columnDataTypes = ""; 50 | String sqlText = ""; 51 | String sourceType = ""; 52 | String sourceServerName = ""; 53 | String sourceInstanceName = ""; 54 | String sourcePort = ""; 55 | String sourceDatabaseName = ""; 56 | String sourceUserName = ""; 57 | String sourcePass = ""; 58 | int gpfdistPort = 0; 59 | int columnCount = 0; 60 | 61 | try 62 | { 63 | String strSQL = "SELECT id, table_name, columns, column_datatypes, sql_text,\n"; 64 | strSQL += "source_type, source_server_name, source_instance_name, source_port, source_database_name, source_user_name, source_pass,\n"; 65 | strSQL += "array_upper(columns, 1) as column_count\n"; 66 | strSQL += "FROM os.custom_sql"; 67 | 68 | Statement stmt = conn.createStatement(); 69 | ResultSet rs = stmt.executeQuery(strSQL); 70 | while (rs.next()) 71 | { 72 | id = rs.getString(1); 73 | tableName = rs.getString(2); 74 | columns = rs.getString(3); 75 | columnDataTypes = rs.getString(4); 76 | sqlText = rs.getString(5); 77 | sourceType = rs.getString(6); 78 | sourceServerName = rs.getString(7); 79 | sourceInstanceName = rs.getString(8); 80 | sourcePort = rs.getString(9); 81 | sourceDatabaseName = rs.getString(10); 82 | sourceUserName = rs.getString(11); 83 | sourcePass = rs.getString(12); 84 | columnCount = rs.getInt(13); 85 | 86 | if (sourceType == null) 87 | sourceType = ""; 88 | 89 | if (sourceInstanceName == null) 90 | sourceInstanceName = ""; 91 | 92 | if (sourcePort == null) 93 | sourcePort = ""; 94 | 95 | if (sourceDatabaseName == null) 96 | sourceDatabaseName = ""; 97 | 98 | updateTable(conn, id, tableName, columns, columnDataTypes, sqlText, sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceUserName, sourcePass); 99 | } 100 | 101 | } 102 | catch (SQLException ex) 103 | { 104 | throw new SQLException(ex.getMessage()); 105 | } 106 | } 107 | 108 | private static void updateTable(Connection conn, String id, String tableName, String columns, String columnDataTypes, String sqlText, String sourceType, String sourceServerName, String sourceInstanceName, String sourcePort, String sourceDatabaseName, String sourceUserName, String sourcePass) throws SQLException 109 | { 110 | columns = "'" + columns + "'"; 111 | columnDataTypes = "'" + columnDataTypes + "'"; 112 | sqlText = setSQLString(sqlText); 113 | sourceType = setSQLString(sourceType); 114 | sourceServerName = setSQLString(sourceServerName); 115 | sourceInstanceName = setSQLString(sourceInstanceName); 116 | sourcePort = setSQLInt(sourcePort); 117 | sourceDatabaseName = setSQLString(sourceDatabaseName); 118 | sourceUserName = setSQLString(sourceUserName); 119 | sourcePass = setSQLString(sourcePass); 120 | 121 | try 122 | { 123 | //stop the gpfdist process for this job 124 | customStop(conn, id); 125 | 126 | //drop the old version of the table 127 | tableName = tableName.toLowerCase(); 128 | dropExtTable(conn, tableName); 129 | 130 | //add the quotes for the table name so the insert works properly 131 | tableName = setSQLString(tableName); 132 | 133 | //dynamically get the gpfdist port for the custom table 134 | int gpfdistPort = GpfdistRunner.customStart(OSProperties.osHome); 135 | System.out.println("Starting gpfdist on port " + gpfdistPort); 136 | 137 | //build the insert statement 138 | String strSQL = "INSERT INTO os.ao_custom_sql\n"; 139 | strSQL += "(id, table_name, columns, column_datatypes, sql_text, source_type, source_server_name, source_instance_name, source_port, source_database_name, source_user_name, source_pass, gpfdist_port)\n"; 140 | strSQL += "VALUES\n"; 141 | strSQL += " (" + id + ", \n"; 142 | strSQL += " " + tableName + ", \n"; 143 | strSQL += " " + columns + ", \n"; 144 | strSQL += " " + columnDataTypes + ", \n"; 145 | strSQL += " " + sqlText + ", \n"; 146 | strSQL += " " + sourceType + ", \n"; 147 | strSQL += " " + sourceServerName + ", \n"; 148 | strSQL += " " + sourceInstanceName + ", \n"; 149 | strSQL += " " + sourcePort + ", \n"; 150 | strSQL += " " + sourceDatabaseName + ", \n"; 151 | strSQL += " " + sourceUserName + ", \n"; 152 | strSQL += " " + sourcePass + ", \n"; 153 | strSQL += " " + gpfdistPort + ")\n"; 154 | updateTable(conn, strSQL); 155 | 156 | //create the external table 157 | createExtTable(conn, id); 158 | } 159 | catch (SQLException ex) 160 | { 161 | throw new SQLException(ex.getMessage()); 162 | } 163 | } 164 | 165 | private static String setSQLString(String columnValue) 166 | { 167 | columnValue = columnValue.replace("'", "\\'"); 168 | 169 | if (columnValue.equals("")) 170 | columnValue = "null"; 171 | else 172 | columnValue = "'" + columnValue + "'"; 173 | 174 | return columnValue; 175 | } 176 | 177 | private static String setSQLInt(String columnValue) 178 | { 179 | if (columnValue.equals("")) 180 | columnValue = "null"; 181 | 182 | return columnValue; 183 | } 184 | 185 | private static void updateTable(Connection conn, String strSQL) throws SQLException 186 | { 187 | try 188 | { 189 | Statement stmt = conn.createStatement(); 190 | stmt.executeUpdate(strSQL); 191 | } 192 | catch (SQLException ex) 193 | { 194 | throw new SQLException(ex.getMessage()); 195 | } 196 | } 197 | 198 | private static void customStop(Connection conn, String id) throws SQLException 199 | { 200 | int myGpfdistPort = 0; 201 | try 202 | { 203 | //get the existing gpfdist_port value 204 | //but only if the port isn't in use by another 205 | //external table. 206 | //two tables should never use the same port 207 | //unless there is a problem 208 | String strSQL = "SELECT c1.gpfdist_port\n"; 209 | strSQL += "FROM os.custom_sql AS c1\n"; 210 | strSQL += "WHERE c1.id = " + id + "\n"; 211 | strSQL += "AND NOT EXISTS (SELECT NULL\n"; 212 | strSQL += " FROM os.custom_sql AS c2\n"; 213 | strSQL += " WHERE c1.id <> c2.id\n"; 214 | strSQL += " AND c1.gpfdist_port = c2.gpfdist_port)"; 215 | 216 | Statement stmt = conn.createStatement(); 217 | ResultSet rs = stmt.executeQuery(strSQL); 218 | while (rs.next()) 219 | { 220 | myGpfdistPort = rs.getInt(1); 221 | GpfdistRunner.customStop(OSProperties.osHome, myGpfdistPort); 222 | } 223 | 224 | } 225 | catch (SQLException ex) 226 | { 227 | throw new SQLException(ex.getMessage()); 228 | } 229 | } 230 | 231 | private static void dropExtTable(Connection conn, String tableName) throws SQLException 232 | { 233 | try 234 | { 235 | String strSQL = "DROP EXTERNAL TABLE IF EXISTS " + tableName; 236 | updateTable(conn, strSQL); 237 | } 238 | catch (SQLException ex) 239 | { 240 | throw new SQLException(ex.getMessage()); 241 | } 242 | } 243 | 244 | private static void createExtTable(Connection conn, String id) throws SQLException 245 | { 246 | try 247 | { 248 | String strSQL = "SELECT LOWER(table_name) as table_name,\n"; 249 | strSQL += "unnest(columns) AS column_name,\n"; 250 | strSQL += "unnest(column_datatypes) AS data_type,\n"; 251 | strSQL += "gpfdist_port\n"; 252 | strSQL += "FROM os.custom_sql\n"; 253 | strSQL += "WHERE id = " + id; 254 | 255 | String strSQLCreateTable = ""; 256 | int i = 0; 257 | int gpfdistPort = 0; 258 | 259 | Statement stmt = conn.createStatement(); 260 | ResultSet rs = stmt.executeQuery(strSQL); 261 | while (rs.next()) 262 | { 263 | i++; 264 | if (i == 1) 265 | { 266 | gpfdistPort = rs.getInt(4); 267 | strSQLCreateTable = "CREATE EXTERNAL TABLE " + rs.getString(1) + "\n"; 268 | strSQLCreateTable += "(" + rs.getString(2) + " " + rs.getString(3); 269 | } 270 | else 271 | { 272 | strSQLCreateTable += ",\n" + "\"" + rs.getString(2) + "\" " + rs.getString(3); 273 | } 274 | } 275 | 276 | strSQLCreateTable += ")\n"; 277 | 278 | strSQLCreateTable += "LOCATION ('gpfdist://" + OSProperties.osServer + ":" + gpfdistPort + "/config.properties+" + id + "#transform=externaldata')\n"; 279 | strSQLCreateTable += "FORMAT 'TEXT' (delimiter '|' null 'null')"; 280 | 281 | updateTable(conn, strSQLCreateTable); 282 | } 283 | catch (SQLException ex) 284 | { 285 | throw new SQLException(ex.getMessage()); 286 | } 287 | } 288 | 289 | } 290 | -------------------------------------------------------------------------------- /src/CustomSQLControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | 6 | public class CustomSQLControl 7 | { 8 | public static String buildPage(Map parms) 9 | { 10 | ResultSet rs = null; 11 | String search = parms.get("search"); 12 | String limit = parms.get("limit"); 13 | String offset = parms.get("offset"); 14 | String actionType = parms.get("action_type"); 15 | String sortBy = parms.get("sort_by"); 16 | String sort = parms.get("sort"); 17 | String id = parms.get("id"); 18 | String tableName = parms.get("table_name"); 19 | String sqlText = parms.get("sql_text"); 20 | String sourceType = parms.get("source_type"); 21 | String sourceServerName = parms.get("source_server_name"); 22 | String sourceInstanceName = parms.get("source_instance_name"); 23 | String sourcePort = parms.get("source_port"); 24 | String sourceDatabaseName = parms.get("source_database_name"); 25 | String sourceUserName = parms.get("source_user_name"); 26 | String sourcePass = parms.get("source_pass"); 27 | String submit = parms.get("submit_form"); 28 | String schema = parms.get("schema"); 29 | 30 | ArrayList extConnectionIdList = new ArrayList(); 31 | String extConnectionId = parms.get("ext_connection_id"); 32 | ArrayList dataTypes = new ArrayList(); 33 | ArrayList columns = new ArrayList(); 34 | ArrayList columnDataTypes = new ArrayList(); 35 | 36 | String strColumnName = ""; //name of the html form field column1, column2... 37 | String strColumn = ""; //value from column1, column2... 38 | 39 | String strColumnDataTypeName = ""; //name of the html form field data_type1, data_type2... 40 | String strColumnDataType = ""; //value from the data_type1, data_type2... html field 41 | 42 | for (int i=0; i <= CustomSQLModel.maxColumns; i++) 43 | { 44 | //get the name of the column being posted 45 | strColumnName = "column" + i; 46 | strColumn = parms.get(strColumnName); 47 | if (strColumn != null && !strColumn.equals("")) 48 | { 49 | //strColumn = "'" + strColumn + "'"; 50 | columns.add(strColumn); 51 | 52 | //get the data type 53 | strColumnDataTypeName = "data_type" + i; 54 | strColumnDataType = parms.get(strColumnDataTypeName); 55 | if (strColumnDataType != null && !strColumnDataType.equals("")) 56 | { 57 | //strColumnDataType = "'" + strColumnDataType + "'"; 58 | columnDataTypes.add(strColumnDataType); 59 | } 60 | } 61 | } 62 | 63 | if (search == null) 64 | search = ""; 65 | 66 | if (limit == null) 67 | limit = "10"; 68 | 69 | if (offset == null) 70 | offset = "0"; 71 | 72 | if (sort == null || sort.equals("")) 73 | sort = "asc"; 74 | 75 | if (sortBy == null || sortBy.equals("")) 76 | sortBy = "id"; 77 | 78 | if (id == null) 79 | id = ""; 80 | 81 | if (extConnectionId == null) 82 | extConnectionId = ""; 83 | 84 | if (tableName == null) 85 | tableName = ""; 86 | 87 | if (sqlText == null) 88 | sqlText = ""; 89 | 90 | if (sourceType == null) 91 | sourceType = ""; 92 | 93 | if (sourceServerName == null) 94 | sourceServerName = ""; 95 | 96 | if (sourceInstanceName == null) 97 | sourceInstanceName = ""; 98 | 99 | if (sourcePort == null) 100 | sourcePort = ""; 101 | 102 | if (sourceDatabaseName == null) 103 | sourceDatabaseName = ""; 104 | 105 | if (sourceUserName == null) 106 | sourceUserName = ""; 107 | 108 | if (sourcePass == null) 109 | sourcePass = ""; 110 | 111 | if (actionType == null || actionType.equals("")) 112 | actionType = "view"; 113 | 114 | if (submit == null || submit.equals("")) 115 | submit = "0"; 116 | 117 | if (schema == null) 118 | schema = ""; 119 | 120 | String msg = ""; 121 | 122 | if (actionType.equals("view")) 123 | { 124 | try 125 | { 126 | rs = CustomSQLModel.getList(search, limit, offset, sortBy, sort); 127 | msg = CustomSQLView.viewList(search, rs, limit, offset, sortBy, sort); 128 | } 129 | catch (Exception ex) 130 | { 131 | msg += ex.getMessage(); 132 | } 133 | } 134 | else if (actionType.equals("update")) 135 | { 136 | if (submit.equals("0") || tableName.equals("") || columns.isEmpty() || columnDataTypes.isEmpty() || sqlText.equals("") || sourceServerName.equals("") || sourceUserName.equals("") || sourcePass.equals("")) 137 | { 138 | dataTypes = CustomSQLModel.getDataTypes(); 139 | CustomSQLModel e = CustomSQLModel.getModel(id); 140 | msg = CustomSQLView.viewUpdate(e.id, e.tableName, e.columns, e.columnDataTypes, e.sqlText, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceUserName, e.sourcePass, dataTypes); 141 | } 142 | else 143 | { 144 | try 145 | { 146 | CustomSQLModel.updateTable(id, tableName, columns, columnDataTypes, sqlText, sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceUserName, sourcePass); 147 | rs = CustomSQLModel.getList(search, limit, offset, sortBy, sort); 148 | msg = CustomSQLView.viewList(search, rs, limit, offset, sortBy, sort); 149 | } 150 | catch (Exception ex) 151 | { 152 | msg = ex.getMessage(); 153 | } 154 | } 155 | } 156 | else if (actionType.equals("delete")) 157 | { 158 | if (submit.equals("0")) 159 | { 160 | dataTypes = CustomSQLModel.getDataTypes(); 161 | CustomSQLModel e = CustomSQLModel.getModel(id); 162 | msg = CustomSQLView.viewDelete(e.id, e.tableName, e.columns, e.columnDataTypes, e.sqlText, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceUserName, e.sourcePass, dataTypes); 163 | } 164 | else 165 | { 166 | try 167 | { 168 | CustomSQLModel.deleteTable(id, tableName); 169 | rs = CustomSQLModel.getList(search, limit, offset, sortBy, sort); 170 | msg = CustomSQLView.viewList(search, rs, limit, offset, sortBy, sort); 171 | } 172 | catch (Exception ex) 173 | { 174 | msg = ex.getMessage(); 175 | } 176 | } 177 | } 178 | else if (actionType.equals("create")) 179 | { 180 | try 181 | { 182 | if (submit.equals("1") && !tableName.equals("") && !columns.isEmpty() && !columnDataTypes.isEmpty() && !sqlText.equals("") && !extConnectionId.equals("")) 183 | { 184 | submit = "1"; 185 | } 186 | else 187 | { 188 | submit = "0"; 189 | } 190 | if (submit.equals("0")) 191 | { 192 | dataTypes = CustomSQLModel.getDataTypes(); 193 | extConnectionIdList = ExternalTableModel.getExtConnectionIds(); 194 | msg = CustomSQLView.viewCreate(tableName, columns, columnDataTypes, dataTypes, sqlText, extConnectionIdList); 195 | } 196 | else 197 | { 198 | CustomSQLModel.insertTable(tableName, columns, columnDataTypes, sqlText, extConnectionId); 199 | rs = CustomSQLModel.getList(search, limit, offset, sortBy, sort); 200 | msg = CustomSQLView.viewList(search, rs, limit, offset, sortBy, sort); 201 | } 202 | } 203 | catch (Exception ex) 204 | { 205 | msg = ex.getMessage(); 206 | } 207 | } 208 | return msg; 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/EnvironmentControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | 4 | public class EnvironmentControl 5 | { 6 | public static String buildPage(Map parms) 7 | { 8 | ResultSet rs = null; 9 | String search = parms.get("search"); 10 | String limit = parms.get("limit"); 11 | String offset = parms.get("offset"); 12 | String actionType = parms.get("action_type"); 13 | String sortBy = parms.get("sort_by"); 14 | String sort = parms.get("sort"); 15 | String maxJobs = parms.get("max_jobs"); 16 | String oFetchSize = parms.get("oFetchSize"); 17 | String submit = parms.get("submit_form"); 18 | 19 | if (search == null) 20 | search = ""; 21 | 22 | if (limit == null) 23 | limit = "10"; 24 | 25 | if (offset == null) 26 | offset = "0"; 27 | 28 | if (actionType == null || actionType.equals("")) 29 | actionType = "view"; 30 | 31 | if (sortBy == null || sortBy.equals("")) 32 | sortBy = "name"; 33 | 34 | if (sort == null || sort.equals("")) 35 | sort = "asc"; 36 | 37 | if (submit == null || submit.equals("")) 38 | submit = "0"; 39 | 40 | if (maxJobs == null || maxJobs.equals("")) 41 | maxJobs = ""; 42 | 43 | if (oFetchSize == null || oFetchSize.equals("")) 44 | oFetchSize = ""; 45 | 46 | String msg = ""; 47 | 48 | if (actionType.equals("view")) 49 | { 50 | try 51 | { 52 | rs = EnvironmentModel.getList(search, limit, offset, sortBy, sort); 53 | msg = EnvironmentView.viewList(search, rs, limit, offset, sortBy, sort); 54 | } 55 | catch (Exception ex) 56 | { 57 | msg += ex.getMessage(); 58 | } 59 | } 60 | else if (actionType.equals("Daemon")) 61 | { 62 | String status = ""; 63 | try 64 | { 65 | status = EnvironmentModel.getStatus(); 66 | if (submit.equals("0")) 67 | { 68 | msg = EnvironmentView.viewStartStop(status); 69 | } 70 | else 71 | { 72 | EnvironmentModel.setStatus(status); 73 | rs = EnvironmentModel.getList(search, limit, offset, sortBy, sort); 74 | msg = EnvironmentView.viewList(search, rs, limit, offset, sortBy, sort); 75 | } 76 | } 77 | catch (Exception ex) 78 | { 79 | msg += ex.getMessage(); 80 | } 81 | } 82 | else if (actionType.equals("Agent")) 83 | { 84 | String status = ""; 85 | try 86 | { 87 | status = EnvironmentModel.getAgentStatus(); 88 | if (submit.equals("0")) 89 | { 90 | msg = EnvironmentView.viewAgentStartStop(status); 91 | } 92 | else 93 | { 94 | EnvironmentModel.setAgentStatus(status); 95 | rs = EnvironmentModel.getList(search, limit, offset, sortBy, sort); 96 | msg = EnvironmentView.viewList(search, rs, limit, offset, sortBy, sort); 97 | } 98 | } 99 | catch (Exception ex) 100 | { 101 | msg += ex.getMessage(); 102 | } 103 | } 104 | else if (actionType.equals("max_jobs")) 105 | { 106 | try 107 | { 108 | if (submit.equals("0")) 109 | { 110 | maxJobs = EnvironmentModel.getVariable("max_jobs"); 111 | msg = EnvironmentView.viewMaxJobs(maxJobs); 112 | } 113 | else 114 | { 115 | EnvironmentModel.setVariable("max_jobs", maxJobs); 116 | rs = EnvironmentModel.getList(search, limit, offset, sortBy, sort); 117 | msg = EnvironmentView.viewList(search, rs, limit, offset, sortBy, sort); 118 | } 119 | } 120 | catch (Exception ex) 121 | { 122 | msg += ex.getMessage(); 123 | } 124 | } 125 | else if (actionType.equals("oFetchSize")) 126 | { 127 | try 128 | { 129 | if (submit.equals("0")) 130 | { 131 | oFetchSize = EnvironmentModel.getVariable("oFetchSize"); 132 | msg = EnvironmentView.viewFetchSize(oFetchSize); 133 | } 134 | else 135 | { 136 | EnvironmentModel.setVariable("oFetchSize", oFetchSize); 137 | rs = EnvironmentModel.getList(search, limit, offset, sortBy, sort); 138 | msg = EnvironmentView.viewList(search, rs, limit, offset, sortBy, sort); 139 | } 140 | } 141 | catch (Exception ex) 142 | { 143 | msg += ex.getMessage(); 144 | } 145 | } 146 | return msg; 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/EnvironmentModel.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.net.*; 4 | import java.io.*; 5 | 6 | public class EnvironmentModel 7 | { 8 | public static ResultSet getList(String search, String limit, String offset, String sortBy, String sort) throws SQLException 9 | { 10 | String strSQL = "SELECT CASE WHEN restart IS FALSE\n"; 11 | strSQL += " THEN '' || description || ' (Dynamic)'\n"; 12 | strSQL += " ELSE name || ' (Re-run os_install.sh to change this value)'\n"; 13 | strSQL += "END as name,\n"; 14 | strSQL += "value\n"; 15 | strSQL += "FROM (SELECT name, name as description, value, restart FROM os.variables\n"; 16 | strSQL += "UNION\n"; 17 | strSQL += "SELECT 'Daemon' as name, 'Queue Daemon' as description,\n"; 18 | strSQL += "status as value,\n"; 19 | strSQL += "false as restart\n"; 20 | strSQL += "FROM os.osstatus\n"; 21 | strSQL += "UNION\n"; 22 | strSQL += "SELECT 'Agent' as name, 'Scheduler Daemon' as description,\n"; 23 | strSQL += "status as value,\n"; 24 | strSQL += "false as restart\n"; 25 | strSQL += "FROM os.agentstatus\n"; 26 | strSQL += ") as sub\n"; 27 | 28 | if (!search.equals("")) 29 | { 30 | strSQL += "WHERE LOWER(name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 31 | strSQL += "OR LOWER(value) LIKE '%' || LOWER('" + search + "') || '%'\n"; 32 | } 33 | 34 | sortBy = sortBy.toLowerCase(); 35 | if (sortBy.equals("name") || sortBy.equals("value") || sortBy.equals("restart")) 36 | strSQL += "ORDER BY " + sortBy + " " + sort + "\n"; 37 | else 38 | strSQL += "ORDER BY name asc\n"; 39 | 40 | if (!limit.equals("")) 41 | strSQL += "LIMIT " + limit + " "; 42 | 43 | if (!offset.equals("")) 44 | strSQL += "OFFSET " + offset; 45 | 46 | try 47 | { 48 | ResultSet rs = OutsourcerModel.getResults(strSQL); 49 | return rs; 50 | } 51 | catch (SQLException ex) 52 | { 53 | throw new SQLException(ex.getMessage()); 54 | } 55 | } 56 | 57 | public static String getStatus() throws SQLException 58 | { 59 | String strSQL = "SELECT status FROM os.osstatus"; 60 | String status = "Down"; 61 | try 62 | { 63 | ResultSet rs = OutsourcerModel.getResults(strSQL); 64 | while (rs.next()) 65 | { 66 | status = rs.getString(1); 67 | } 68 | return status; 69 | } 70 | catch (SQLException ex) 71 | { 72 | throw new SQLException(ex.getMessage()); 73 | } 74 | } 75 | 76 | public static String getAgentStatus() throws SQLException 77 | { 78 | String strSQL = "SELECT status FROM os.agentstatus"; 79 | String status = "Down"; 80 | try 81 | { 82 | ResultSet rs = OutsourcerModel.getResults(strSQL); 83 | while (rs.next()) 84 | { 85 | status = rs.getString(1); 86 | } 87 | return status; 88 | } 89 | catch (SQLException ex) 90 | { 91 | throw new SQLException(ex.getMessage()); 92 | } 93 | } 94 | 95 | public static String getVariable(String name) throws SQLException 96 | { 97 | String strSQL = "SELECT value FROM os.variables WHERE name = '" + name + "'"; 98 | String value = ""; 99 | try 100 | { 101 | ResultSet rs = OutsourcerModel.getResults(strSQL); 102 | while (rs.next()) 103 | { 104 | value = rs.getString(1); 105 | } 106 | return value; 107 | } 108 | catch (SQLException ex) 109 | { 110 | throw new SQLException(ex.getMessage()); 111 | } 112 | 113 | } 114 | 115 | public static void setVariable(String name, String value) throws SQLException 116 | { 117 | name = OutsourcerModel.setSQLString(name); 118 | value = OutsourcerModel.setSQLString(value); 119 | 120 | String strSQL = "INSERT INTO os.ao_variables\n"; 121 | strSQL += "(name, value, restart)\n"; 122 | strSQL += "SELECT name, " + value + " as value, restart\n"; 123 | strSQL += "FROM os.variables\n"; 124 | strSQL += "WHERE name = " + name; 125 | 126 | try 127 | { 128 | OutsourcerModel.updateTable(strSQL); 129 | } 130 | catch (SQLException ex) 131 | { 132 | throw new SQLException(ex.getMessage()); 133 | } 134 | } 135 | 136 | public static void setStatus(String status) throws SQLException 137 | { 138 | String strSQL = ""; 139 | if (status.equals("Down")) 140 | { 141 | strSQL = "SELECT * from os.osstart"; 142 | } 143 | else if (status.equals("Up")) 144 | { 145 | strSQL = "SELECT * from os.osstop"; 146 | } 147 | 148 | try 149 | { 150 | OutsourcerModel.getResults(strSQL); 151 | } 152 | catch (SQLException ex) 153 | { 154 | throw new SQLException(ex.getMessage()); 155 | } 156 | } 157 | 158 | public static void setAgentStatus(String status) throws SQLException 159 | { 160 | String strSQL = ""; 161 | if (status.equals("Down")) 162 | { 163 | strSQL = "SELECT * from os.agentstart"; 164 | } 165 | else if (status.equals("Up")) 166 | { 167 | strSQL = "SELECT * from os.agentstop"; 168 | } 169 | 170 | try 171 | { 172 | OutsourcerModel.getResults(strSQL); 173 | } 174 | catch (SQLException ex) 175 | { 176 | throw new SQLException(ex.getMessage()); 177 | } 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /src/EnvironmentView.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | 4 | public class EnvironmentView 5 | { 6 | public static String action = "environment"; 7 | 8 | private static String getHead() 9 | { 10 | String myScript = "function formSubmit(offset)\n"; 11 | myScript += "{\n"; 12 | myScript += " document.getElementById(\"offset\").value = offset;\n"; 13 | myScript += " document.getElementById(\"myForm\").submit();\n"; 14 | myScript += "}\n"; 15 | myScript += "function sortRS(sortBy, sort)\n"; 16 | myScript += "{\n"; 17 | myScript += " document.getElementById(\"offset\").value = 0;\n"; 18 | myScript += " document.getElementById(\"sort_by\").value = sortBy;\n"; 19 | myScript += " document.getElementById(\"sort\").value = sort;\n"; 20 | myScript += " document.getElementById(\"myForm\").submit();\n"; 21 | myScript += "}\n"; 22 | 23 | return myScript; 24 | } 25 | 26 | public static String viewList(String search, ResultSet rs, String limit, String offset, String sortBy, String sort) 27 | { 28 | String myScript = getHead(); 29 | String msg = OutsourcerView.viewSearch(action, search, limit, offset, sortBy, sort, myScript); 30 | msg += getHeader(sortBy, sort); 31 | 32 | try 33 | { 34 | msg += OutsourcerView.viewResults(limit, offset, rs); 35 | } 36 | catch (SQLException ex) 37 | { 38 | msg += ex.getMessage(); 39 | } 40 | return msg; 41 | } 42 | 43 | private static String getHeader(String sortBy, String sort) 44 | { 45 | 46 | String downArrow = "↓"; 47 | String upArrow = "↑"; 48 | String defaultSort = "asc"; 49 | 50 | String nameHeader = "Name\n"; 83 | valueHeader += valueFocus + "onclick=\"sortRS('value', '" + valueSort + "')\">" + valueArrow + "\n"; 84 | 85 | String msg = "\n"; 86 | msg += "\n"; 87 | msg += nameHeader + valueHeader; 88 | msg += "\n"; 89 | 90 | return msg; 91 | } 92 | 93 | public static String viewStartStop(String status) 94 | { 95 | String msg = OutsourcerView.viewHeader("", "", action); 96 | msg += "\n"; 97 | msg += "
\n"; 98 | msg += "\n"; 99 | msg += "\n"; 100 | msg += "\n"; 101 | msg += "\n"; 102 | msg += "\n"; 103 | msg += "\n"; 104 | msg += "
Queue Daemon Status" + status + "
Start/Stop
\n"; 105 | msg += "\n"; 106 | 107 | return msg; 108 | } 109 | 110 | public static String viewAgentStartStop(String status) 111 | { 112 | String msg = OutsourcerView.viewHeader("", "", action); 113 | msg += "
\n"; 114 | msg += "\n"; 115 | msg += "\n"; 116 | msg += "\n"; 117 | msg += "\n"; 118 | msg += "\n"; 119 | msg += "\n"; 120 | msg += "\n"; 121 | msg += "
Scheduler Daemon Status" + status + "
Start/Stop
\n"; 122 | msg += "
\n"; 123 | 124 | return msg; 125 | } 126 | 127 | public static String viewMaxJobs(String maxJobs) 128 | { 129 | String msg = OutsourcerView.viewHeader("", "", action); 130 | msg += "
\n"; 131 | msg += "\n"; 132 | msg += "\n"; 133 | msg += "\n"; 134 | msg += "\n"; 135 | msg += "\n"; 136 | msg += "
Max Jobs
\n"; 137 | msg += "
\n"; 138 | 139 | return msg; 140 | } 141 | 142 | public static String viewFetchSize(String oFetchSize) 143 | { 144 | String msg = OutsourcerView.viewHeader("", "", action); 145 | msg += "
\n"; 146 | msg += "\n"; 147 | msg += "\n"; 148 | msg += "\n"; 149 | msg += "\n"; 150 | msg += "\n"; 151 | msg += "
Oracle Fetch Size
\n"; 152 | msg += "
\n"; 153 | 154 | return msg; 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/ExternalData.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.net.*; 3 | import java.io.*; 4 | 5 | public class ExternalData 6 | { 7 | 8 | private static String myclass = "ExternalData"; 9 | public static boolean debug = false; 10 | 11 | public static void main(String[] args) throws Exception 12 | { 13 | String method = "main"; 14 | int location = 1000; 15 | int argsCount = args.length; 16 | 17 | String configFile = ""; 18 | int queueId = 0; 19 | String appendColumnMax = "0"; 20 | String refreshType = ""; 21 | String sourceTable = ""; 22 | int customSQLId = 0; 23 | 24 | location = 2000; 25 | 26 | if (argsCount == 5) 27 | { 28 | location = 3100; 29 | //this is an external table Outsourcer defines 30 | configFile = args[0]; 31 | queueId = Integer.parseInt(args[1]); 32 | appendColumnMax = args[2]; 33 | refreshType = args[3]; 34 | sourceTable = args[4]; 35 | executeOS(configFile, queueId, appendColumnMax, refreshType, sourceTable); 36 | } 37 | else if (argsCount == 2) 38 | { 39 | location = 3200; 40 | //this is an external table that a user defines 41 | configFile = args[0]; 42 | customSQLId = Integer.parseInt(args[1]); 43 | executeExt(configFile, customSQLId); 44 | } 45 | } 46 | 47 | private static void executeOS(String configFile, int queueId, String appendColumnMax, String refreshType, String sourceTable) throws Exception 48 | { 49 | String method = "executeOS"; 50 | int location = 1000; 51 | 52 | String sourceType = ""; 53 | String sourceServer = ""; 54 | String sourceInstance = ""; 55 | int sourcePort = 0; 56 | String sourceDatabase = ""; 57 | String sourceSchema = ""; 58 | String sourceUser = ""; 59 | String sourcePass = ""; 60 | String appendColumnName = ""; 61 | String strSQL = ""; 62 | Connection conn = null; 63 | Connection gpConn = null; 64 | 65 | try 66 | { 67 | location = 3000; 68 | 69 | ResultSet rs; 70 | Statement stmt; 71 | 72 | gpConn = CommonDB.connectGP(configFile); 73 | 74 | location = 3010; 75 | strSQL = GP.getQueueDetails(gpConn, queueId); 76 | 77 | location = 3020; 78 | stmt = gpConn.createStatement(); 79 | 80 | location = 3040; 81 | rs = stmt.executeQuery(strSQL); 82 | 83 | while (rs.next()) 84 | { 85 | //handled with a parameter so Outsourcer can do replications jobs 86 | //refreshType = rs.getString(1); 87 | sourceType = rs.getString(2); 88 | sourceServer = rs.getString(3); 89 | sourceInstance = rs.getString(4); 90 | sourcePort = rs.getInt(5); 91 | sourceDatabase = rs.getString(6); 92 | sourceSchema = rs.getString(7); 93 | //handled with a parameter so Outsourcer can do replications jobs 94 | //sourceTable = rs.getString(8); 95 | sourceUser = rs.getString(9); 96 | sourcePass = rs.getString(10); 97 | appendColumnName = rs.getString(11); 98 | } 99 | 100 | location = 3090; 101 | 102 | appendColumnMax = appendColumnMax.replace("SPACE", " "); 103 | 104 | if (sourceType.equals("sqlserver")) 105 | { 106 | location = 3100; 107 | gpConn.close(); 108 | 109 | location = 3150; 110 | conn = CommonDB.connectSQLServer(sourceServer, sourceInstance, sourceUser, sourcePass); 111 | 112 | location = 3200; 113 | //create SQL statement for selecting data 114 | strSQL = SQLServer.getSQLForData(conn, sourceDatabase, sourceSchema, sourceTable, refreshType, appendColumnName, appendColumnMax); 115 | 116 | location = 3300; 117 | //execute the SQL Statement 118 | CommonDB.outputData(conn, strSQL); 119 | 120 | location = 3400; 121 | conn.close(); 122 | 123 | } 124 | else if (sourceType.equals("oracle")) 125 | { 126 | location = 4000; 127 | int fetchSize = Integer.parseInt(GP.getVariable(gpConn, "oFetchSize")); 128 | 129 | location = 4050; 130 | gpConn.close(); 131 | 132 | location = 4100; 133 | conn = CommonDB.connectOracle(sourceServer, sourceDatabase, sourcePort, sourceUser, sourcePass, fetchSize); 134 | 135 | location = 4200; 136 | //execute the SQL Statement 137 | strSQL = Oracle.getSQLForData(conn, sourceSchema, sourceTable, refreshType, appendColumnName, appendColumnMax); 138 | 139 | location = 4300; 140 | //execute the SQL Statement 141 | CommonDB.outputData(conn, strSQL); 142 | 143 | location = 4400; 144 | conn.close(); 145 | 146 | } 147 | 148 | } 149 | catch (SQLException ex) 150 | { 151 | throw new SQLException("(" + myclass + ":" + method + ":" + location + ":" + ex.getMessage() + ")"); 152 | } 153 | finally 154 | { 155 | if (conn != null) 156 | conn.close(); 157 | if (gpConn != null) 158 | gpConn.close(); 159 | } 160 | } 161 | 162 | private static void executeExt(String configFile, int customSQLId) throws Exception 163 | { 164 | String method = "executeExt"; 165 | int location = 1000; 166 | 167 | String sourceType = ""; 168 | String sourceServer = ""; 169 | String sourceInstance = ""; 170 | int sourcePort = 0; 171 | String sourceDatabase = ""; 172 | String sourceUser = ""; 173 | String sourcePass = ""; 174 | String selectSQL = ""; 175 | String strSQL = ""; 176 | Connection conn = null; 177 | Connection gpConn = null; 178 | 179 | try 180 | { 181 | location = 3000; 182 | gpConn = CommonDB.connectGP(configFile); 183 | 184 | ResultSet rs; 185 | Statement stmt; 186 | 187 | location = 3010; 188 | strSQL = GP.getCustomSQLDetails(gpConn, customSQLId); 189 | 190 | location = 3020; 191 | stmt = gpConn.createStatement(); 192 | 193 | location = 3030; 194 | rs = stmt.executeQuery(strSQL); 195 | 196 | while (rs.next()) 197 | { 198 | sourceType = rs.getString(1); 199 | sourceServer = rs.getString(2); 200 | sourceInstance = rs.getString(3); 201 | sourcePort = rs.getInt(4); 202 | sourceDatabase = rs.getString(5); 203 | sourceUser = rs.getString(6); 204 | sourcePass = rs.getString(7); 205 | selectSQL = rs.getString(8); 206 | } 207 | 208 | location = 3090; 209 | if (sourceType.equals("sqlserver")) 210 | { 211 | 212 | location = 3100; 213 | gpConn.close(); 214 | 215 | location = 3200; 216 | conn = CommonDB.connectSQLServer(sourceServer, sourceInstance, sourceUser, sourcePass); 217 | 218 | location = 3300; 219 | //execute the SQL Statement 220 | CommonDB.outputData(conn, selectSQL); 221 | 222 | location = 3400; 223 | conn.close(); 224 | 225 | } 226 | else if (sourceType.equals("oracle")) 227 | { 228 | location = 4000; 229 | int fetchSize = Integer.parseInt(GP.getVariable(gpConn, "oFetchSize")); 230 | 231 | location = 4100; 232 | gpConn.close(); 233 | 234 | location = 4200; 235 | conn = CommonDB.connectOracle(sourceServer, sourceDatabase, sourcePort, sourceUser, sourcePass, fetchSize); 236 | 237 | location = 4300; 238 | //execute the SQL Statement 239 | CommonDB.outputData(conn, selectSQL); 240 | 241 | location = 4400; 242 | conn.close(); 243 | 244 | } 245 | 246 | } 247 | catch (SQLException ex) 248 | { 249 | throw new SQLException("(" + myclass + ":" + method + ":" + location + ":" + ex.getMessage() + ")"); 250 | } 251 | finally 252 | { 253 | if (conn != null) 254 | conn.close(); 255 | if (gpConn != null) 256 | gpConn.close(); 257 | } 258 | } 259 | 260 | } 261 | -------------------------------------------------------------------------------- /src/ExternalDataD.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.net.*; 3 | import java.io.*; 4 | 5 | public class ExternalDataD 6 | { 7 | private static String myclass = "ExternalDataD"; 8 | public static boolean debug = true; 9 | public static String configFile = ""; 10 | public static String dbVersion = "GPDB"; 11 | 12 | public static void main(String[] args) throws Exception 13 | { 14 | 15 | String method = "main"; 16 | int location = 1000; 17 | int argsCount = args.length; 18 | Connection conn = null; 19 | 20 | configFile = args[0]; 21 | String action = args[1]; 22 | 23 | location = 2000; 24 | try 25 | { 26 | location = 3000; 27 | if (action.equals("start")) 28 | { 29 | conn = CommonDB.connectGP(configFile); 30 | GP.cancelJobs(conn); 31 | dbVersion = GP.getVersion(conn); 32 | conn.close(); 33 | loadLoop(); 34 | } 35 | else if (action.equals("stop")) 36 | { 37 | conn = CommonDB.connectGP(configFile); 38 | GP.failJobs(conn); 39 | GP.cancelJobs(conn); 40 | conn.close(); 41 | } 42 | } 43 | catch (SQLException ex) 44 | { 45 | throw new SQLException("(" + myclass + ":" + method + ":" + location + ":" + ex.getMessage() + ")"); 46 | } 47 | finally 48 | { 49 | if (conn != null) 50 | conn.close(); 51 | } 52 | } 53 | 54 | private static void loadLoop() throws Exception 55 | { 56 | 57 | String method = "loadLoop"; 58 | int location = 1000; 59 | Connection conn = null; 60 | 61 | boolean loop = true; 62 | 63 | while (loop) 64 | { 65 | try 66 | { 67 | location = 2000; 68 | ResultSet rs; 69 | Statement stmt; 70 | String strSQL = ""; 71 | 72 | int queueId = 0; 73 | Timestamp queueDate = new Timestamp((new java.util.Date()).getTime()); 74 | Timestamp startDate = new Timestamp((new java.util.Date()).getTime()); 75 | int numRows = 0; 76 | int id = 0; 77 | String refreshType = ""; 78 | String targetSchema = ""; 79 | String targetTable = ""; 80 | boolean targetAppendOnly = false; 81 | boolean targetCompressed = false; 82 | boolean targetRowOrientation = true; 83 | String sourceType = ""; 84 | String sourceServer = ""; 85 | String sourceInstance = ""; 86 | int sourcePort = 0; 87 | String sourceDatabase = ""; 88 | String sourceSchema = ""; 89 | String sourceTable = ""; 90 | String sourceUser = ""; 91 | String sourcePass = ""; 92 | String columnName = ""; 93 | String sqlText = ""; 94 | boolean snapshot = false; 95 | 96 | location = 3000; 97 | conn = CommonDB.connectGP(configFile); 98 | 99 | location = 3100; 100 | strSQL = "SELECT queue_id, queue_date, start_date,\n"; 101 | strSQL += " id, refresh_type, LOWER(target_schema_name) as target_schema_name, LOWER(target_table_name) as target_table_name,\n"; 102 | strSQL += " target_append_only, target_compressed, target_row_orientation,\n"; 103 | strSQL += " source_type, source_server_name, source_instance_name, source_port, source_database_name, source_schema_name,\n"; 104 | strSQL += " source_table_name, source_user_name, source_pass, column_name,\n"; 105 | strSQL += " sql_text, snapshot\n"; 106 | strSQL += "FROM os.fn_update_status()"; 107 | 108 | location = 3200; 109 | stmt = conn.createStatement(); 110 | 111 | location = 3300; 112 | rs = stmt.executeQuery(strSQL); 113 | 114 | //query only returns one record 115 | while (rs.next()) 116 | { 117 | location = 4000; 118 | queueId = rs.getInt(1); 119 | queueDate = rs.getTimestamp(2); 120 | startDate = rs.getTimestamp(3); 121 | id = rs.getInt(4); 122 | refreshType = rs.getString(5); 123 | targetSchema = rs.getString(6); 124 | targetTable = rs.getString(7); 125 | targetAppendOnly = rs.getBoolean(8); 126 | targetCompressed = rs.getBoolean(9); 127 | targetRowOrientation = rs.getBoolean(10); 128 | sourceType = rs.getString(11); 129 | sourceServer = rs.getString(12); 130 | sourceInstance = rs.getString(13); 131 | sourcePort = rs.getInt(14); 132 | sourceDatabase = rs.getString(15); 133 | sourceSchema = rs.getString(16); 134 | sourceTable = rs.getString(17); 135 | sourceUser = rs.getString(18); 136 | sourcePass = rs.getString(19); 137 | columnName = rs.getString(20); 138 | sqlText = rs.getString(21); 139 | snapshot = rs.getBoolean(22); 140 | } 141 | 142 | location = 4100; 143 | if (conn != null) 144 | conn.close(); 145 | 146 | location = 4200; 147 | if (id != 0) 148 | { 149 | location = 5000; 150 | if (debug) 151 | Logger.printMsg("Thread working on ID: " + id); 152 | 153 | ExternalDataThread edt = new ExternalDataThread(queueId, queueDate, startDate, id, refreshType, targetSchema, targetTable, targetAppendOnly, targetCompressed, targetRowOrientation, sourceType, sourceServer, sourceInstance, sourcePort, sourceDatabase, sourceSchema, sourceTable, sourceUser, sourcePass, columnName, sqlText, snapshot); 154 | 155 | location = 5100; 156 | if (debug) 157 | Logger.printMsg("Thread is initialized"); 158 | 159 | Thread t = new Thread(edt); 160 | 161 | location = 5200; 162 | if (debug) 163 | Logger.printMsg("Thread started"); 164 | t.start(); 165 | } 166 | else 167 | { 168 | location = 6000; 169 | Thread.sleep(5000); 170 | } 171 | 172 | } 173 | catch (SQLException ex) 174 | { 175 | Logger.printMsg("(" + method + ":" + location + ":" + ex.getMessage() + ")"); 176 | Thread.sleep(5000); 177 | } 178 | finally 179 | { 180 | if (conn != null) 181 | conn.close(); 182 | } 183 | } 184 | 185 | } 186 | } 187 | 188 | -------------------------------------------------------------------------------- /src/ExternalTableControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.util.ArrayList; 4 | 5 | public class ExternalTableControl 6 | { 7 | public static String buildPage(Map parms) 8 | { 9 | ResultSet rs = null; 10 | String search = parms.get("search"); 11 | String limit = parms.get("limit"); 12 | String offset = parms.get("offset"); 13 | String actionType = parms.get("action_type"); 14 | String sortBy = parms.get("sort_by"); 15 | String sort = parms.get("sort"); 16 | String id = parms.get("id"); 17 | String sourceType = parms.get("source_type"); 18 | String sourceServerName = parms.get("source_server_name"); 19 | String sourceInstanceName = parms.get("source_instance_name"); 20 | String sourcePort = parms.get("source_port"); 21 | String sourceDatabaseName = parms.get("source_database_name"); 22 | String sourceUserName = parms.get("source_user_name"); 23 | String sourcePass = parms.get("source_pass"); 24 | String submit = parms.get("submit_form"); 25 | String validate = parms.get("validate"); 26 | String validateMsg = ""; 27 | String sourceSchema = parms.get("source_schema"); 28 | String targetSchema = parms.get("target_schema"); 29 | String strTargetAppendOnly = parms.get("target_append_only"); 30 | boolean targetAppendOnly = Boolean.valueOf(parms.get("target_append_only")); 31 | boolean targetCompressed = Boolean.valueOf(parms.get("target_compressed")); 32 | boolean targetRowOrientation = Boolean.valueOf(parms.get("target_row_orientation")); 33 | String refreshType = parms.get("refresh_type"); 34 | 35 | String scheduleDesc = parms.get("schedule_desc"); 36 | ArrayList scheduleList = new ArrayList(); 37 | 38 | if (search == null) 39 | search = ""; 40 | 41 | if (limit == null) 42 | limit = "10"; 43 | 44 | if (offset == null) 45 | offset = "0"; 46 | 47 | if (sortBy == null || sortBy.equals("")) 48 | sortBy = "id"; 49 | 50 | if (sort == null || sort.equals("")) 51 | sort = "asc"; 52 | 53 | if (id == null) 54 | id = ""; 55 | 56 | if (sourceType == null) 57 | sourceType = ""; 58 | 59 | if (sourceServerName == null) 60 | sourceServerName = ""; 61 | 62 | if (sourceInstanceName == null) 63 | sourceInstanceName = ""; 64 | 65 | if (sourcePort == null) 66 | sourcePort = ""; 67 | 68 | if (sourceDatabaseName == null) 69 | sourceDatabaseName = ""; 70 | 71 | if (sourceUserName == null) 72 | sourceUserName = ""; 73 | 74 | if (sourcePass == null) 75 | sourcePass = ""; 76 | 77 | if (actionType == null || actionType.equals("")) 78 | actionType = "view"; 79 | 80 | if (submit == null || submit.equals("")) 81 | submit = "0"; 82 | 83 | if (validate == null) 84 | validate = "0"; 85 | 86 | if (sourceSchema == null) 87 | sourceSchema = ""; 88 | 89 | if (targetSchema == null) 90 | targetSchema = ""; 91 | 92 | if (refreshType == null) 93 | refreshType = ""; 94 | 95 | if (scheduleDesc == null) 96 | scheduleDesc = ""; 97 | 98 | if (strTargetAppendOnly == null) 99 | { 100 | targetAppendOnly = true; 101 | targetCompressed = false; 102 | targetRowOrientation = true; 103 | } 104 | 105 | //need the parsing of parms into local variables to call the right views 106 | 107 | String msg = ""; 108 | 109 | if (actionType.equals("view")) 110 | { 111 | try 112 | { 113 | rs = ExternalTableModel.getList(search, limit, offset, sortBy, sort); 114 | msg = ExternalTableView.viewList(search, rs, limit, offset, sortBy, sort); 115 | } 116 | catch (Exception ex) 117 | { 118 | msg += ex.getMessage(); 119 | } 120 | } 121 | else if (actionType.equals("update")) 122 | { 123 | if (submit.equals("0")) 124 | { 125 | ExternalTableModel e = ExternalTableModel.getModel(id); 126 | msg = ExternalTableView.viewUpdate(e.id, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceUserName, e.sourcePass, ""); 127 | } 128 | else 129 | { 130 | if (validate.equals("0")) 131 | { 132 | try 133 | { 134 | ExternalTableModel.insertTable(id, sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceUserName, sourcePass); 135 | rs = ExternalTableModel.getList(search, limit, offset, sortBy, sort); 136 | msg = ExternalTableView.viewList(search, rs, limit, offset, sortBy, sort); 137 | } 138 | catch (Exception ex) 139 | { 140 | msg = ex.getMessage(); 141 | } 142 | } 143 | else 144 | { 145 | if ( ( sourceType.equals("oracle") && (sourceServerName.equals("") || sourcePort.equals("") || sourceDatabaseName.equals("") || sourceUserName.equals("") || sourcePass.equals("")) ) || (sourceType.equals("sqlserver") && (sourceServerName.equals("") || sourceUserName.equals("") || sourcePass.equals("")) ) ) 146 | validateMsg = "Please provide correct values for connection and try again."; 147 | else 148 | validateMsg = ExternalTableModel.validate(sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceUserName, sourcePass); 149 | 150 | msg = ExternalTableView.viewUpdate(id, sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceUserName, sourcePass, validateMsg); 151 | 152 | } 153 | } 154 | 155 | } 156 | else if (actionType.equals("delete")) 157 | { 158 | if (submit.equals("0")) 159 | { 160 | ExternalTableModel e = ExternalTableModel.getModel(id); 161 | msg = ExternalTableView.viewDelete(e.id, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceUserName, e.sourcePass); 162 | } 163 | else 164 | { 165 | try 166 | { 167 | ExternalTableModel.deleteTable(id); 168 | rs = ExternalTableModel.getList(search, limit, offset, sortBy, sort); 169 | msg = ExternalTableView.viewList(search, rs, limit, offset, sortBy, sort); 170 | } 171 | catch (Exception ex) 172 | { 173 | msg = ex.getMessage(); 174 | } 175 | } 176 | 177 | } 178 | //creating jobs based on an schema from the source 179 | else if (actionType.equals("create")) 180 | { 181 | ResultSet databaseList = null; 182 | ResultSet schemaList = null; 183 | Connection conn = null; 184 | 185 | //make sure valid values are set for submitting, else refresh the page. 186 | if (submit.equals("1") && (id.equals("") || sourceSchema.equals("") || targetSchema.equals("") || refreshType.equals("")) ) 187 | submit = "0"; 188 | 189 | if (submit.equals("0")) 190 | { 191 | ExternalTableModel e = ExternalTableModel.getModel(id); 192 | try 193 | { 194 | if (e.sourceType.equals("sqlserver")) 195 | { 196 | conn = CommonDB.connectSQLServer(e.sourceServerName, e.sourceInstanceName, e.sourceUserName, e.sourcePass); 197 | //get list of databases 198 | databaseList = SQLServer.getDatabaseList(conn); 199 | 200 | if (!(sourceDatabaseName.equals(""))) 201 | { 202 | //database picked, now get the schemas in that database 203 | //note that for SQLServer, it is sourceDatabaseName and not e.sourceDatabaseName 204 | schemaList = SQLServer.getSchemaList(conn, sourceDatabaseName); 205 | } 206 | } 207 | else if (e.sourceType.equals("oracle")) 208 | { 209 | //note that for Oracle, it is e.sourceDatabaseName 210 | sourceDatabaseName = e.sourceDatabaseName; 211 | int intSourcePort = Integer.parseInt(e.sourcePort); 212 | conn = CommonDB.connectOracle(e.sourceServerName, e.sourceDatabaseName, intSourcePort, e.sourceUserName, e.sourcePass, 10); 213 | schemaList = Oracle.getSchemaList(conn); 214 | } 215 | 216 | scheduleList = ScheduleModel.getDescriptions(); 217 | msg = ExternalTableView.viewCreate(databaseList, schemaList, e.id, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, sourceDatabaseName, e.sourceUserName, targetSchema, targetAppendOnly, targetCompressed, targetRowOrientation, refreshType, scheduleList); 218 | if (conn != null) 219 | conn.close(); 220 | } 221 | catch (Exception ex) 222 | { 223 | msg = ex.getMessage(); 224 | } 225 | } 226 | else 227 | { 228 | try 229 | { 230 | int count = ExternalTableModel.createJobs(id, sourceDatabaseName, sourceSchema, targetSchema, targetAppendOnly, targetCompressed, targetRowOrientation, refreshType, scheduleDesc); 231 | msg = ExternalTableView.viewResults(count); 232 | } 233 | catch (Exception ex) 234 | { 235 | msg = ex.getMessage(); 236 | } 237 | } 238 | } 239 | 240 | return msg; 241 | } 242 | } 243 | -------------------------------------------------------------------------------- /src/GpfdistRunner.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.io.*; 3 | 4 | public class GpfdistRunner 5 | { 6 | public static void main(String[] args) throws Exception 7 | { 8 | String osHome = args[0]; 9 | int myPort = jobStart(osHome); 10 | System.out.println("my port: " + myPort); 11 | 12 | String myMsg = jobStop(osHome, myPort); 13 | System.out.println("my stop message: " + myMsg); 14 | 15 | myPort = customStart(osHome); 16 | System.out.println("my port: " + myPort); 17 | 18 | 19 | myMsg = customStop(osHome, myPort); 20 | System.out.println("my stop message: " + myMsg); 21 | 22 | } 23 | 24 | public static int jobStart(String osHome) throws SQLException 25 | { 26 | String myCommand = "jobstart"; 27 | int port = 0; 28 | 29 | try 30 | { 31 | port = start(osHome, myCommand); 32 | } 33 | catch (Exception ex) 34 | { 35 | throw new SQLException(ex.getMessage()); 36 | } 37 | return port; 38 | } 39 | 40 | public static String jobStop(String osHome, int port) throws SQLException 41 | { 42 | String myCommand = "jobstop"; 43 | String myReturn = ""; 44 | 45 | try 46 | { 47 | myReturn = stop(osHome, myCommand, port); 48 | } 49 | catch (Exception ex) 50 | { 51 | throw new SQLException(ex.getMessage()); 52 | } 53 | return myReturn; 54 | } 55 | 56 | public static int customStart(String osHome) throws SQLException 57 | { 58 | String myCommand = "customstart"; 59 | int port = 0; 60 | 61 | try 62 | { 63 | port = start(osHome, myCommand); 64 | } 65 | catch (Exception ex) 66 | { 67 | throw new SQLException(ex.getMessage()); 68 | } 69 | return port; 70 | } 71 | 72 | public static String customStop(String osHome, int port) throws SQLException 73 | { 74 | String myCommand = "customstop"; 75 | String myReturn = ""; 76 | 77 | try 78 | { 79 | myReturn = stop(osHome, myCommand, port); 80 | } 81 | catch (Exception ex) 82 | { 83 | throw new SQLException(ex.getMessage()); 84 | } 85 | return myReturn; 86 | } 87 | 88 | private static int start(String osHome, String myCommand) throws SQLException 89 | { 90 | int port = 0; 91 | 92 | try 93 | { 94 | String shellCommand = osHome + "/bin/" + myCommand; 95 | Process p = Runtime.getRuntime().exec(shellCommand); 96 | BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); 97 | String line = null; 98 | while ((line = in.readLine()) != null) 99 | { 100 | port = Integer.parseInt(line); 101 | } 102 | if (port == 0) 103 | { 104 | //unable to get a gpfdist process so fail 105 | throw new SQLException("ERROR: Unable to acquire a gpfdist process"); 106 | } 107 | } 108 | catch (Exception ex) 109 | { 110 | throw new SQLException(ex.getMessage()); 111 | } 112 | return port; 113 | } 114 | 115 | private static String stop(String osHome, String myCommand, int port) throws SQLException 116 | { 117 | String myReturn = ""; 118 | 119 | try 120 | { 121 | String shellCommand = osHome + "/bin/" + myCommand; 122 | shellCommand += " " + Integer.toString(port); 123 | Process p = Runtime.getRuntime().exec(shellCommand); 124 | BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); 125 | String line = null; 126 | while ((line = in.readLine()) != null) 127 | { 128 | myReturn += line + " "; 129 | //System.out.println(line); 130 | } 131 | return myReturn; 132 | } 133 | catch (Exception ex) 134 | { 135 | throw new SQLException(ex.getMessage()); 136 | } 137 | 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/JobControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.util.ArrayList; 4 | 5 | public class JobControl 6 | { 7 | public static String buildPage(Map parms) 8 | { 9 | ResultSet rs = null; 10 | String search = parms.get("search"); 11 | String limit = parms.get("limit"); 12 | String offset = parms.get("offset"); 13 | String actionType = parms.get("action_type"); 14 | String sortBy = parms.get("sort_by"); 15 | String sort = parms.get("sort"); 16 | String id = parms.get("id"); 17 | String refreshType = parms.get("refresh_type"); 18 | String targetSchemaName = parms.get("target_schema_name"); 19 | String targetTableName = parms.get("target_table_name"); 20 | boolean targetAppendOnly = Boolean.valueOf(parms.get("target_append_only")); 21 | boolean targetCompressed = Boolean.valueOf(parms.get("target_compressed")); 22 | boolean targetRowOrientation = Boolean.valueOf(parms.get("target_row_orientation")); 23 | String sourceType = parms.get("source_type"); 24 | String sourceServerName = parms.get("source_server_name"); 25 | String sourceInstanceName = parms.get("source_instance_name"); 26 | String sourcePort = parms.get("source_port"); 27 | String sourceDatabaseName = parms.get("source_database_name"); 28 | String sourceSchemaName = parms.get("source_schema_name"); 29 | String sourceTableName = parms.get("source_table_name"); 30 | String sourceUserName = parms.get("source_user_name"); 31 | String sourcePass = parms.get("source_pass"); 32 | String columnName = parms.get("column_name"); 33 | String sqlText = parms.get("sql_text"); 34 | boolean snapshot = Boolean.valueOf(parms.get("snapshot")); 35 | String submit = parms.get("submit_form"); 36 | String queueAction = parms.get("queue_action"); 37 | 38 | String scheduleDesc = parms.get("schedule_desc"); 39 | ArrayList scheduleList = new ArrayList(); 40 | 41 | if (search == null) 42 | search = ""; 43 | 44 | if (limit == null) 45 | limit = "10"; 46 | 47 | if (offset == null) 48 | offset = "0"; 49 | 50 | if (sort == null || sort.equals("")) 51 | sort = "asc"; 52 | 53 | if (sortBy == null || sortBy.equals("")) 54 | sortBy = "id"; 55 | 56 | if (id == null) 57 | id = ""; 58 | 59 | if (refreshType == null) 60 | refreshType = ""; 61 | 62 | if (targetSchemaName == null) 63 | targetSchemaName = ""; 64 | 65 | if (targetTableName == null) 66 | targetTableName = ""; 67 | 68 | if (sourceType == null) 69 | sourceType = ""; 70 | 71 | if (sourceServerName == null) 72 | sourceServerName = ""; 73 | 74 | if (sourceInstanceName == null) 75 | sourceInstanceName = ""; 76 | 77 | if (sourcePort == null) 78 | sourcePort = ""; 79 | 80 | if (sourceDatabaseName == null) 81 | sourceDatabaseName = ""; 82 | 83 | if (sourceSchemaName == null) 84 | sourceSchemaName = ""; 85 | 86 | if (sourceTableName == null) 87 | sourceTableName = ""; 88 | 89 | if (sourceUserName == null) 90 | sourceUserName = ""; 91 | 92 | if (sourcePass == null) 93 | sourcePass = ""; 94 | 95 | if (columnName == null) 96 | columnName = ""; 97 | 98 | if (sqlText == null) 99 | sqlText = ""; 100 | 101 | //if (snapshot == null) 102 | // snapshot = false; 103 | 104 | if (actionType == null || actionType.equals("")) 105 | actionType = "view"; 106 | 107 | if (submit == null || submit.equals("")) 108 | submit = "0"; 109 | 110 | if (queueAction == null) 111 | queueAction = ""; 112 | 113 | if (scheduleDesc == null) 114 | scheduleDesc = ""; 115 | 116 | //need the parsing of parms into local variables to call the right views 117 | 118 | String msg = ""; 119 | 120 | try 121 | { 122 | if (actionType.equals("view")) 123 | { 124 | rs = JobModel.getList(search, limit, offset, sortBy, sort); 125 | msg = JobView.viewList(search, rs, limit, offset, sortBy, sort); 126 | } 127 | else if (actionType.equals("update")) 128 | { 129 | if (submit.equals("0")) 130 | { 131 | JobModel e = JobModel.getModel(id); 132 | scheduleList = ScheduleModel.getDescriptions(); 133 | msg = JobView.viewUpdate(e.id, e.refreshType, e.targetSchemaName, e.targetTableName, e.targetAppendOnly, e.targetCompressed, e.targetRowOrientation, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceSchemaName, e.sourceTableName, e.sourceUserName, e.sourcePass, e.columnName, e.sqlText, e.snapshot, e.scheduleDesc, scheduleList); 134 | } 135 | else 136 | { 137 | JobModel.insertTable(id, refreshType, targetSchemaName, targetTableName, targetAppendOnly, targetCompressed, targetRowOrientation, sourceType, sourceServerName, sourceInstanceName, sourcePort, sourceDatabaseName, sourceSchemaName, sourceTableName, sourceUserName, sourcePass, columnName, sqlText, snapshot, scheduleDesc); 138 | rs = JobModel.getList(search, limit, offset, sortBy, sort); 139 | msg = JobView.viewList(search, rs, limit, offset, sortBy, sort); 140 | } 141 | } 142 | else if (actionType.equals("delete")) 143 | { 144 | if (submit.equals("0")) 145 | { 146 | JobModel e = JobModel.getModel(id); 147 | scheduleList = ScheduleModel.getDescriptions(); 148 | msg = JobView.viewDelete(e.id, e.refreshType, e.targetSchemaName, e.targetTableName, e.targetAppendOnly, e.targetCompressed, e.targetRowOrientation, e.sourceType, e.sourceServerName, e.sourceInstanceName, e.sourcePort, e.sourceDatabaseName, e.sourceSchemaName, e.sourceTableName, e.sourceUserName, e.sourcePass, e.columnName, e.sqlText, e.snapshot, e.scheduleDesc, scheduleList); 149 | } 150 | else 151 | { 152 | JobModel.deleteTable(id); 153 | rs = JobModel.getList(search, limit, offset, sortBy, sort); 154 | msg = JobView.viewList(search, rs, limit, offset, sortBy, sort); 155 | } 156 | 157 | } 158 | else if (actionType.equals("queue")) 159 | { 160 | if (queueAction.equals("insert")) 161 | QueueModel.insertTable(id); 162 | else if (queueAction.equals("update")) 163 | QueueModel.updateTable(id); 164 | 165 | rs = JobModel.getList(search, limit, offset, sortBy, sort); 166 | msg = JobView.viewList(search, rs, limit, offset, sortBy, sort); 167 | } 168 | else if (actionType.equals("delete_all")) 169 | { 170 | if (submit.equals("0")) 171 | { 172 | msg = JobView.viewDelete(); 173 | } 174 | else 175 | { 176 | JobModel.deleteTable(); 177 | rs = JobModel.getList(search, limit, offset, sortBy, sort); 178 | msg = JobView.viewList(search, rs, limit, offset, sortBy, sort); 179 | } 180 | } 181 | } 182 | catch (Exception ex) 183 | { 184 | msg = ex.getMessage(); 185 | } 186 | return msg; 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Pivotal Software, Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of Pivotal Software, Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from this 15 | software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | 29 | 30 | ================================================================= 31 | TABLE OF CONTENTS 32 | ================================================================= 33 | 34 | Pivotal Outsourcer: 35 | 36 | The following is a listing of the open source components detailed in this document. 37 | This list is provided for your convenience; please read further if you wish to 38 | review the copyright notice(s) and the full text of the license associated 39 | with each component. 40 | 41 | 42 | 43 | 44 | SECTION 1: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES 45 | 46 | >>> nanohttpd-1.25 47 | 48 | 49 | --------------- SECTION 1: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES ---------- 50 | 51 | 52 | BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES are applicable to the following component(s). 53 | 54 | 55 | >>> nanohttpd-1.25 56 | 57 | Copyright (C) 2001,2005-2011 by Jarno Elonen 58 | and Copyright (C) 2010 by Konstantinos Togias 59 | 60 | Redistribution and use in source and binary forms, with or without 61 | modification, are permitted provided that the following conditions 62 | are met: 63 | 64 | Redistributions of source code must retain the above copyright notice, 65 | this list of conditions and the following disclaimer. Redistributions in 66 | binary form must reproduce the above copyright notice, this list of 67 | conditions and the following disclaimer in the documentation and/or other 68 | materials provided with the distribution. The name of the author may not 69 | be used to endorse or promote products derived from this software without 70 | specific prior written permission. 71 | 72 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 73 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 74 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 75 | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 76 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 77 | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 78 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 79 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 80 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 81 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 82 | 83 | 84 | =========================================================================== 85 | 86 | To the extent any open source components are licensed under the 87 | GPL and/or LGPL, or other similar licenses that require the 88 | source code and/or modifications to source code to be made 89 | available (as would be noted above), you may obtain a copy of 90 | the source code corresponding to the binaries for such open 91 | source components and modifications thereto, if any, (the 92 | "Source Files"), by downloading the Source Files from Pivotal's website at 93 | http://www.gopivotal.com/open-source, or by sending a request, 94 | with your name and address to: Pivotal Software, Inc., 1900 S. Norfolk Street #125, 95 | San Mateo, CA 94403, Attention: General Counsel. All such requests should 96 | clearly specify: OPEN SOURCE FILES REQUEST, 97 | Attention General Counsel. Pivotal shall mail a copy of the 98 | Source Files to you on a CD or equivalent physical medium. This 99 | offer to obtain a copy of the Source Files is valid for three 100 | years from the date you acquired this Software product. 101 | Alternatively, the Source Files may accompany the Pivotal product. 102 | 103 | [OUTSOURCER401GASS123113] 104 | -------------------------------------------------------------------------------- /src/Logger.java: -------------------------------------------------------------------------------- 1 | import java.util.Calendar; 2 | 3 | public class Logger 4 | { 5 | public static void main(String[] args) 6 | { 7 | String myMsg = args[0]; 8 | 9 | printMsg(myMsg); 10 | 11 | } 12 | 13 | public static void printMsg(String myMsg) 14 | { 15 | Calendar now = Calendar.getInstance(); 16 | 17 | System.out.println(now.getTime() + ";" + myMsg); 18 | 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/OSProperties.java: -------------------------------------------------------------------------------- 1 | import java.io.FileNotFoundException; 2 | import java.io.IOException; 3 | import java.io.InputStream; 4 | import java.io.FileInputStream; 5 | import java.util.Properties; 6 | 7 | public class OSProperties 8 | { 9 | public static String osServer = ""; 10 | public static int osPort = 0; 11 | public static String osHome = ""; 12 | public static String gpServer = ""; 13 | public static String gpDatabase = ""; 14 | public static int gpPort = 0; 15 | public static String gpUserName = ""; 16 | public static String gpPassword = ""; 17 | 18 | public static void main(String[] args) throws Exception 19 | { 20 | String configFile = args[0]; 21 | getPropValues(configFile); 22 | 23 | System.out.println("osServer: " + osServer); 24 | System.out.println("osPort: " + osPort); 25 | System.out.println("osHome: " + osHome); 26 | System.out.println("gpServer: " + gpServer); 27 | System.out.println("gpDatabase: " + gpDatabase); 28 | System.out.println("gpPort: " + gpPort); 29 | System.out.println("gpUserName: " + gpUserName); 30 | System.out.println("gpPassword: " + gpPassword); 31 | 32 | } 33 | 34 | public static void getPropValues(String configFile) throws IOException 35 | { 36 | try 37 | { 38 | 39 | Properties prop = new Properties(); 40 | InputStream inputStream = new FileInputStream(configFile); 41 | 42 | if (inputStream != null) 43 | { 44 | prop.load(inputStream); 45 | } 46 | else 47 | { 48 | throw new FileNotFoundException(configFile + " not found!"); 49 | } 50 | 51 | // set the property values 52 | osServer = prop.getProperty("osserver"); 53 | String strOsPort = prop.getProperty("osport"); 54 | if (strOsPort != null) 55 | osPort = Integer.parseInt(strOsPort); 56 | osHome = prop.getProperty("oshome"); 57 | gpServer = prop.getProperty("gpserver"); 58 | gpDatabase = prop.getProperty("gpdatabase"); 59 | String strGpPort = prop.getProperty("gpport"); 60 | if (strGpPort != null) 61 | gpPort = Integer.parseInt(strGpPort); 62 | gpUserName = prop.getProperty("gpusername"); 63 | gpPassword = prop.getProperty("gppassword"); 64 | 65 | } 66 | catch (IOException iex) 67 | { 68 | System.out.println("Unable to read config.properties located here: " + configFile); 69 | System.out.println(iex.getMessage()); 70 | } 71 | } 72 | } 73 | 74 | -------------------------------------------------------------------------------- /src/OutsourcerControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | 3 | public class OutsourcerControl 4 | { 5 | public static String buildPage(String uri, Map parms) 6 | { 7 | String msg = ""; 8 | 9 | if (uri.equals("/")) 10 | //Build navigation page 11 | { 12 | msg += OutsourcerView.viewMain(); 13 | } 14 | else if (uri.equals("/queue")) 15 | { 16 | msg += QueueControl.buildPage(parms); 17 | } 18 | else if (uri.equals("/jobs")) 19 | { 20 | msg += JobControl.buildPage(parms); 21 | } 22 | else if (uri.equals("/environment")) 23 | { 24 | msg += EnvironmentControl.buildPage(parms); 25 | } 26 | else if (uri.equals("/external")) 27 | { 28 | msg += ExternalTableControl.buildPage(parms); 29 | } 30 | else if (uri.equals("/schedule")) 31 | { 32 | msg += ScheduleControl.buildPage(parms); 33 | } 34 | else if (uri.equals("/custom")) 35 | { 36 | msg += CustomSQLControl.buildPage(parms); 37 | } 38 | else 39 | { 40 | msg += OutsourcerView.viewPageNotFound(); 41 | } 42 | msg += OutsourcerView.viewFooter(); 43 | 44 | return msg; 45 | 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/OutsourcerModel.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import java.net.*; 3 | import java.io.*; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | 7 | public class OutsourcerModel 8 | { 9 | 10 | public static ResultSet getResults(String strSQL) throws SQLException 11 | { 12 | try 13 | { 14 | Connection conn = UIConnectionFactory.getConnection(); 15 | Statement stmt = conn.createStatement(); 16 | ResultSet rs = stmt.executeQuery(strSQL); 17 | conn.close(); 18 | return rs; 19 | } 20 | catch (SQLException ex) 21 | { 22 | throw new SQLException(ex.getMessage()); 23 | } 24 | } 25 | 26 | public static ArrayList getStringArray(String strSQL) throws SQLException 27 | { 28 | 29 | ArrayList stringArray = new ArrayList(); 30 | try 31 | { 32 | String output = ""; 33 | String columnValue = ""; 34 | Connection conn = UIConnectionFactory.getConnection(); 35 | Statement stmt = conn.createStatement(); 36 | ResultSet rs = stmt.executeQuery(strSQL); 37 | ResultSetMetaData rsmd = rs.getMetaData(); 38 | int numberOfColumns = rsmd.getColumnCount(); 39 | while (rs.next()) 40 | { 41 | output = ""; 42 | for (int i=1; i gpArrayToJavaArray(String gpArray) 63 | { 64 | //Array in Greenplum is formatted as a string with { } 65 | gpArray = gpArray.substring(1, gpArray.length() - 1); 66 | 67 | ArrayList stringArray = new ArrayList(Arrays.asList(gpArray.split(","))); 68 | return stringArray; 69 | } 70 | 71 | public static String javaArrayToString(ArrayList stringArray) 72 | { 73 | int i = 0; 74 | String output = "{"; 75 | for (String s : stringArray) 76 | { 77 | 78 | s = "'" + s.replaceAll("[^a-zA-Z0-9]", "") + "'"; 79 | s = s.toLowerCase(); 80 | 81 | i++; 82 | if (i == 1) 83 | output = "ARRAY[" + s; 84 | else 85 | output += "," + s; 86 | } 87 | output += "]"; 88 | 89 | return output; 90 | } 91 | 92 | public static void updateTable(String strSQL) throws SQLException 93 | { 94 | try 95 | { 96 | Connection conn = UIConnectionFactory.getConnection(); 97 | Statement stmt = conn.createStatement(); 98 | stmt.executeUpdate(strSQL); 99 | conn.close(); 100 | } 101 | catch (SQLException ex) 102 | { 103 | throw new SQLException(ex.getMessage()); 104 | } 105 | } 106 | public static String setSQLString(boolean columnValue) 107 | { 108 | String strColumnValue = ""; 109 | 110 | if (columnValue) 111 | strColumnValue = "true"; 112 | else 113 | strColumnValue = "false"; 114 | 115 | return strColumnValue; 116 | } 117 | public static String setSQLString(String columnValue) 118 | { 119 | columnValue = columnValue.replace("'", "\\'"); 120 | 121 | if (columnValue.equals("")) 122 | columnValue = "null"; 123 | else 124 | columnValue = "'" + columnValue + "'"; 125 | 126 | return columnValue; 127 | } 128 | public static String setSQLString(int columnValue) 129 | { 130 | String strColumnValue = Integer.toString(columnValue); 131 | 132 | if (strColumnValue.equals("")) 133 | strColumnValue = "null"; 134 | 135 | return strColumnValue; 136 | } 137 | public static String setSQLInt(String columnValue) 138 | { 139 | if (columnValue.equals("")) 140 | columnValue = "null"; 141 | 142 | return columnValue; 143 | } 144 | public static String setSQLTimestamp(Timestamp columnValue) 145 | { 146 | String strColumnValue = columnValue.toString(); 147 | 148 | return strColumnValue; 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/QueueControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | 4 | public class QueueControl 5 | { 6 | public static String buildPage(Map parms) 7 | { 8 | ResultSet rs = null; 9 | String search = parms.get("search"); 10 | String limit = parms.get("limit"); 11 | String offset = parms.get("offset"); 12 | String actionType = parms.get("action_type"); 13 | String sortBy = parms.get("sort_by"); 14 | String sort = parms.get("sort"); 15 | String queueID = parms.get("queueID"); 16 | String id = parms.get("id"); 17 | 18 | if (search == null) 19 | search = ""; 20 | 21 | if (limit == null) 22 | limit = "10"; 23 | 24 | if (offset == null) 25 | offset = "0"; 26 | 27 | if (sort == null || sort.equals("")) 28 | sort = "asc"; 29 | 30 | if (sortBy == null || sortBy.equals("")) 31 | sortBy = "status"; 32 | 33 | if (queueID == null) 34 | queueID = ""; 35 | 36 | if (id == null) 37 | id = ""; 38 | 39 | if (actionType == null || actionType.equals("")) 40 | actionType = "view"; 41 | 42 | String msg = ""; 43 | 44 | if (actionType.equals("view")) 45 | { 46 | try 47 | { 48 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 49 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 50 | } 51 | catch (Exception ex) 52 | { 53 | msg += ex.getMessage(); 54 | } 55 | 56 | } 57 | else if (actionType.equals("insert")) 58 | { 59 | try 60 | { 61 | QueueModel.insertTable(id); 62 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 63 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 64 | } 65 | catch (Exception ex) 66 | { 67 | msg = ex.getMessage(); 68 | } 69 | } 70 | else if (actionType.equals("insert_all")) 71 | { 72 | try 73 | { 74 | QueueModel.insertTableAll(); 75 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 76 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 77 | } 78 | catch (Exception ex) 79 | { 80 | msg = ex.getMessage(); 81 | } 82 | } 83 | else if (actionType.equals("update")) 84 | { 85 | try 86 | { 87 | QueueModel.updateTable(queueID); 88 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 89 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 90 | } 91 | catch (Exception ex) 92 | { 93 | msg = ex.getMessage(); 94 | } 95 | } 96 | else if (actionType.equals("delete")) 97 | { 98 | try 99 | { 100 | QueueModel.deleteTable(queueID); 101 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 102 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 103 | } 104 | catch (Exception ex) 105 | { 106 | msg = ex.getMessage(); 107 | } 108 | } 109 | else if (actionType.equals("cancel")) 110 | { 111 | try 112 | { 113 | QueueModel.cancelTable(id); 114 | rs = QueueModel.getList(search, limit, offset, sortBy, sort); 115 | msg = QueueView.viewList(search, rs, limit, offset, sortBy, sort); 116 | } 117 | catch (Exception ex) 118 | { 119 | msg = ex.getMessage(); 120 | } 121 | } 122 | return msg; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/QueueModel.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.net.*; 4 | import java.io.*; 5 | 6 | public class QueueModel 7 | { 8 | public static ResultSet getList(String search, String limit, String offset, String sortBy, String sort) throws SQLException 9 | { 10 | String strSQL = "SELECT queue_id,\n"; 11 | strSQL += "initcap(status) || CASE WHEN status = 'failed' THEN \n"; 12 | strSQL += " ' ' ||\n"; 13 | strSQL += " ' '\n"; 14 | strSQL += " WHEN status = 'queued' THEN' '\n"; 15 | strSQL += " WHEN status = 'success' THEN ' '\n"; 16 | strSQL += " WHEN status = 'processing' THEN ' '\n"; 17 | strSQL += " else ''\n"; 18 | strSQL += " END AS my_status,\n"; 19 | strSQL += "DATE_TRUNC('second', queue_date) AS my_queue_date, \n"; 20 | strSQL += "COALESCE((DATE_TRUNC('second', start_date))::text, '') AS my_start_date,\n"; 21 | strSQL += "COALESCE((DATE_TRUNC('second', end_date))::text, '') AS my_end_date,\n"; 22 | strSQL += "COALESCE((COALESCE(DATE_TRUNC('second', end_date), CURRENT_TIMESTAMP(0)::timestamp) - DATE_TRUNC('second', start_date))::text, '') AS duration,\n"; 23 | strSQL += "num_rows,\n"; 24 | strSQL += "'' || id || '',\n"; 25 | strSQL += "CASE WHEN refresh_type = 'transform' THEN SUBSTRING(sql_text, 1, 30)\n"; 26 | strSQL += "ELSE COALESCE((target_schema_name || '.' || target_table_name), '') END AS target_table_name,\n"; 27 | strSQL += "COALESCE(error_message, '') AS my_error_message\n"; 28 | strSQL += "FROM os.queue\n"; 29 | if (!search.equals("")) 30 | { 31 | strSQL += "WHERE LOWER(status) LIKE '%' || LOWER('" + search + "') || '%'\n"; 32 | strSQL += "OR LOWER(refresh_type) LIKE '%' || LOWER('" + search + "') || '%'\n"; 33 | strSQL += "OR LOWER(target_schema_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 34 | strSQL += "OR LOWER(target_table_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 35 | strSQL += "OR LOWER(source_type) LIKE '%' || LOWER('" + search + "') || '%'\n"; 36 | strSQL += "OR LOWER(source_server_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 37 | strSQL += "OR LOWER(source_instance_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 38 | strSQL += "OR LOWER(source_port) LIKE '%' || LOWER('" + search + "') || '%'\n"; 39 | strSQL += "OR LOWER(source_database_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 40 | strSQL += "OR LOWER(source_schema_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 41 | strSQL += "OR LOWER(source_table_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 42 | strSQL += "OR LOWER(source_user_name) LIKE '%' || LOWER('" + search +"') || '%'\n"; 43 | strSQL += "OR LOWER(column_name) LIKE '%' || LOWER('" + search + "') || '%'\n"; 44 | strSQL += "OR LOWER(sql_text) LIKE '%' || LOWER('" + search + "') || '%'\n"; 45 | } 46 | sortBy = sortBy.toLowerCase(); 47 | if (sortBy.equals("queue_id") || sortBy.equals("status") || sortBy.equals("queue_date") || sortBy.equals("start_date") || sortBy.equals("end_date") || sortBy.equals("duration") || sortBy.equals("num_rows") || sortBy.equals("id") || sortBy.equals("target_table_name") || sortBy.equals("error_message")) 48 | strSQL += "ORDER BY " + sortBy + " " + sort + ", start_date DESC\n"; 49 | else 50 | strSQL += "ORDER BY status ASC, queue_id DESC\n"; 51 | 52 | if (!limit.equals("")) 53 | strSQL += "LIMIT " + limit + " "; 54 | 55 | if (!offset.equals("")) 56 | strSQL += "OFFSET " + offset; 57 | 58 | try 59 | { 60 | ResultSet rs = OutsourcerModel.getResults(strSQL); 61 | return rs; 62 | } 63 | catch (SQLException ex) 64 | { 65 | throw new SQLException(ex.getMessage()); 66 | } 67 | } 68 | 69 | public static void updateTable(String queueId) throws SQLException 70 | { 71 | String strSQL = "INSERT INTO os.ao_queue\n"; 72 | strSQL += "(queue_id, status, queue_date, start_date, end_date, error_message,\n"; 73 | strSQL += "num_rows, id, refresh_type, target_schema_name, target_table_name,\n"; 74 | strSQL += "target_append_only, target_compressed, target_row_orientation,\n"; 75 | strSQL += "source_type, source_server_name, source_instance_name, source_port,\n"; 76 | strSQL += "source_database_name, source_schema_name, source_table_name,\n"; 77 | strSQL += "source_user_name, source_pass, column_name, sql_text, snapshot)\n"; 78 | strSQL += "SELECT queue_id, 'queued' as status, queue_date, NULL AS start_date, NULL AS end_date, NULL AS error_message,\n"; 79 | strSQL += "0 AS num_rows, id, refresh_type, target_schema_name, target_table_name,\n"; 80 | strSQL += "target_append_only, target_compressed, target_row_orientation,\n"; 81 | strSQL += "source_type, source_server_name, source_instance_name, source_port,\n"; 82 | strSQL += "source_database_name, source_schema_name, source_table_name,\n"; 83 | strSQL += "source_user_name, source_pass, column_name, sql_text, snapshot\n"; 84 | strSQL += "FROM os.queue\n"; 85 | strSQL += "WHERE queue_id = " + queueId + "\n"; 86 | strSQL += "AND status <> 'processing'"; 87 | 88 | 89 | try 90 | { 91 | OutsourcerModel.updateTable(strSQL); 92 | } 93 | catch (SQLException ex) 94 | { 95 | throw new SQLException(ex.getMessage()); 96 | } 97 | } 98 | 99 | public static void insertTable(String id) throws SQLException 100 | { 101 | String strSQL = "SELECT os.fn_queue(" + id + ")"; 102 | 103 | try 104 | { 105 | OutsourcerModel.getResults(strSQL); 106 | } 107 | catch (SQLException ex) 108 | { 109 | throw new SQLException(ex.getMessage()); 110 | } 111 | } 112 | 113 | public static void insertTableAll() throws SQLException 114 | { 115 | String strSQL = "SELECT os.fn_queue_all()"; 116 | 117 | try 118 | { 119 | OutsourcerModel.getResults(strSQL); 120 | } 121 | catch (SQLException ex) 122 | { 123 | throw new SQLException(ex.getMessage()); 124 | } 125 | } 126 | 127 | public static void deleteTable(String queueID) throws SQLException 128 | { 129 | String strSQL = "INSERT INTO os.ao_queue\n"; 130 | strSQL += "(queue_id, status, queue_date, start_date, end_date, error_message,\n"; 131 | strSQL += "num_rows, id, refresh_type, target_schema_name, target_table_name,\n"; 132 | strSQL += "target_append_only, target_compressed, target_row_orientation,\n"; 133 | strSQL += "source_type, source_server_name, source_instance_name, source_port,\n"; 134 | strSQL += "source_database_name, source_schema_name, source_table_name,\n"; 135 | strSQL += "source_user_name, source_pass, column_name, sql_text, snapshot,\n"; 136 | strSQL += "deleted)\n"; 137 | strSQL += "SELECT queue_id, status, queue_date, start_date, end_date, error_message,\n"; 138 | strSQL += "num_rows, id, refresh_type, target_schema_name, target_table_name,\n"; 139 | strSQL += "target_append_only, target_compressed, target_row_orientation,\n"; 140 | strSQL += "source_type, source_server_name, source_instance_name, source_port,\n"; 141 | strSQL += "source_database_name, source_schema_name, source_table_name,\n"; 142 | strSQL += "source_user_name, source_pass, column_name, sql_text, snapshot,\n"; 143 | strSQL += "TRUE AS deleted\n"; 144 | strSQL += "FROM os.queue\n"; 145 | strSQL += "WHERE queue_id = " + queueID + "\n"; 146 | strSQL += "AND status IN ('queued', 'failed')\n"; 147 | 148 | try 149 | { 150 | OutsourcerModel.updateTable(strSQL); 151 | } 152 | catch (SQLException ex) 153 | { 154 | throw new SQLException(ex.getMessage()); 155 | } 156 | } 157 | 158 | public static void cancelTable(String id) throws SQLException 159 | { 160 | String strSQL = "SELECT os.fn_cancel_job(" + id + ")"; 161 | 162 | try 163 | { 164 | ResultSet rs = OutsourcerModel.getResults(strSQL); 165 | } 166 | catch (SQLException ex) 167 | { 168 | throw new SQLException(ex.getMessage()); 169 | } 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /src/QueueView.java: -------------------------------------------------------------------------------- 1 | import java.sql.ResultSet; 2 | 3 | public class QueueView 4 | { 5 | public static String action = "queue"; 6 | 7 | private static String getHead() 8 | { 9 | String myScript = "function formSubmit(offset)\n"; 10 | myScript += "{\n"; 11 | myScript += " document.getElementById(\"offset\").value = offset;\n"; 12 | myScript += " document.getElementById(\"myForm\").submit();\n"; 13 | myScript += "}\n"; 14 | myScript += "function sortRS(sortBy, sort)\n"; 15 | myScript += "{\n"; 16 | myScript += " document.getElementById(\"offset\").value = 0;\n"; 17 | myScript += " document.getElementById(\"sort_by\").value = sortBy;\n"; 18 | myScript += " document.getElementById(\"sort\").value = sort;\n"; 19 | myScript += " document.getElementById(\"myForm\").submit();\n"; 20 | myScript += "}\n"; 21 | myScript += "function updateQueue(id, myAction)\n"; 22 | myScript += "{\n"; 23 | myScript += " document.myForm.action = 'queue';\n"; 24 | myScript += " document.getElementById(\"action_type\").value = myAction;\n"; 25 | myScript += " document.getElementById(\"offset\").value = 0;\n"; 26 | myScript += " document.getElementById(\"queueID\").value = id;\n"; 27 | myScript += " document.getElementById(\"id\").value = id;\n"; 28 | myScript += " document.getElementById(\"myForm\").submit();\n"; 29 | myScript += "}\n"; 30 | myScript += "function cancelQueue(id, myAction)\n"; 31 | myScript += "{\n"; 32 | myScript += " document.myForm.action = 'queue';\n"; 33 | myScript += " document.getElementById(\"action_type\").value = myAction;\n"; 34 | myScript += " document.getElementById(\"offset\").value = 0;\n"; 35 | myScript += " document.getElementById(\"id\").value = id;\n"; 36 | myScript += " document.getElementById(\"myForm\").submit();\n"; 37 | myScript += "}\n"; 38 | 39 | return myScript; 40 | } 41 | 42 | public static String viewList(String search, ResultSet rs, String limit, String offset, String sortBy, String sort) 43 | { 44 | String myScript = getHead(); 45 | String msg = OutsourcerView.viewSearch(action, search, limit, offset, sortBy, sort, myScript); 46 | msg += getHeader(sortBy, sort); 47 | try 48 | { 49 | msg = msg + OutsourcerView.viewResults(limit, offset, rs); 50 | } 51 | catch (Exception localException) 52 | { 53 | msg = msg + localException.getMessage(); 54 | } 55 | return msg; 56 | } 57 | 58 | private static String getHeader(String sortBy, String sort) 59 | { 60 | 61 | String downArrow = "↓"; 62 | String upArrow = "↑"; 63 | String defaultSort = "asc"; 64 | 65 | String queueIDHeader = "Queue\n"; 227 | statusHeader += statusFocus + "onclick=\"sortRS('status', '" + statusSort + "')\">" + statusArrow + "\n"; 228 | queueDateHeader += queueDateFocus + "onclick=\"sortRS('queue_date', '" + queueDateSort + "')\">" + queueDateArrow + "\n"; 229 | startDateHeader += startDateFocus + "onclick=\"sortRS('start_date', '" + startDateSort + "')\">" + startDateArrow + "\n"; 230 | endDateHeader += endDateFocus + "onclick=\"sortRS('end_date', '" + endDateSort + "')\">" + endDateArrow + "\n"; 231 | durationHeader += durationFocus + "onclick=\"sortRS('duration', '" + durationSort + "')\">" + durationArrow + "\n"; 232 | rowsHeader += rowsFocus + "onclick=\"sortRS('num_rows', '" + rowsSort + "')\">" + rowsArrow + "\n"; 233 | jobIDHeader += jobIDFocus + "onclick=\"sortRS('id', '" + jobIDSort + "')\">" + jobIDArrow + "\n"; 234 | targetHeader += targetFocus + "onclick=\"sortRS('target_table_name', '" + targetSort + "')\">" + targetArrow + "\n"; 235 | errorMsgHeader += errorMsgFocus + "onclick=\"sortRS('error_message', '" + errorMsgSort + "')\">" + errorMsgArrow + "\n"; 236 | 237 | String msg = "\n"; 238 | msg += "\n"; 239 | msg += queueIDHeader + statusHeader + queueDateHeader + startDateHeader + endDateHeader + durationHeader + rowsHeader + jobIDHeader + targetHeader + errorMsgHeader; 240 | msg += "\n"; 241 | 242 | return msg; 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/ScheduleControl.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.util.ArrayList; 4 | 5 | public class ScheduleControl 6 | { 7 | public static String buildPage(Map parms) 8 | { 9 | ResultSet rs = null; 10 | String search = parms.get("search"); 11 | String limit = parms.get("limit"); 12 | String offset = parms.get("offset"); 13 | String actionType = parms.get("action_type"); 14 | String sortBy = parms.get("sort_by"); 15 | String sort = parms.get("sort"); 16 | String description = parms.get("description"); 17 | String intervalTrunc = parms.get("interval_trunc"); 18 | String intervalQuantity = parms.get("interval_quantity"); 19 | String submit = parms.get("submit_form"); 20 | String schema = parms.get("schema"); 21 | ArrayList schemaList = new ArrayList(); 22 | 23 | if (search == null) 24 | search = ""; 25 | 26 | if (limit == null) 27 | limit = "10"; 28 | 29 | if (offset == null) 30 | offset = "0"; 31 | 32 | if (sort == null || sort.equals("")) 33 | sort = "asc"; 34 | 35 | if (sortBy == null || sortBy.equals("")) 36 | sortBy = "description"; 37 | 38 | if (description == null) 39 | description = ""; 40 | 41 | if (intervalTrunc == null) 42 | intervalTrunc = ""; 43 | 44 | if (intervalQuantity == null) 45 | intervalQuantity = ""; 46 | 47 | if (actionType == null || actionType.equals("")) 48 | actionType = "view"; 49 | 50 | if (submit == null || submit.equals("")) 51 | submit = "0"; 52 | 53 | if (schema == null) 54 | schema = ""; 55 | 56 | String msg = ""; 57 | 58 | if (actionType.equals("view")) 59 | { 60 | try 61 | { 62 | rs = ScheduleModel.getList(search, limit, offset, sortBy, sort); 63 | msg = ScheduleView.viewList(search, rs, limit, offset, sortBy, sort); 64 | } 65 | catch (Exception ex) 66 | { 67 | msg += ex.getMessage(); 68 | } 69 | } 70 | else if (actionType.equals("update") || actionType.equals("insert")) 71 | { 72 | if (submit.equals("0")) 73 | { 74 | ScheduleModel e = ScheduleModel.getModel(description); 75 | msg = ScheduleView.viewUpdate(e.description, e.intervalTrunc, e.intervalQuantity); 76 | } 77 | else 78 | { 79 | try 80 | { 81 | ScheduleModel.insertTable(description, intervalTrunc, intervalQuantity); 82 | rs = ScheduleModel.getList(search, limit, offset, sortBy, sort); 83 | msg = ScheduleView.viewList(search, rs, limit, offset, sortBy, sort); 84 | } 85 | catch (Exception ex) 86 | { 87 | msg = ex.getMessage(); 88 | } 89 | } 90 | } 91 | else if (actionType.equals("delete")) 92 | { 93 | if (submit.equals("0")) 94 | { 95 | ScheduleModel e = ScheduleModel.getModel(description); 96 | msg = ScheduleView.viewDelete(e.description, e.intervalTrunc, e.intervalQuantity); 97 | } 98 | else 99 | { 100 | try 101 | { 102 | ScheduleModel.deleteTable(description); 103 | rs = ScheduleModel.getList(search, limit, offset, sortBy, sort); 104 | msg = ScheduleView.viewList(search, rs, limit, offset, sortBy, sort); 105 | } 106 | catch (Exception ex) 107 | { 108 | msg = ex.getMessage(); 109 | } 110 | } 111 | } 112 | else if (actionType.equals("create")) 113 | { 114 | try 115 | { 116 | if (submit.equals("0")) 117 | { 118 | schemaList = JobModel.getSchemas(); 119 | msg = ScheduleView.viewCreate(description, schemaList); 120 | } 121 | else 122 | { 123 | JobModel.updateJobsSchedule(description, schema); 124 | rs = ScheduleModel.getList(search, limit, offset, sortBy, sort); 125 | msg = ScheduleView.viewList(search, rs, limit, offset, sortBy, sort); 126 | } 127 | } 128 | catch (Exception ex) 129 | { 130 | msg = ex.getMessage(); 131 | } 132 | } 133 | return msg; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/ScheduleModel.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.sql.*; 3 | import java.net.*; 4 | import java.io.*; 5 | import java.util.ArrayList; 6 | 7 | public class ScheduleModel 8 | { 9 | String description; 10 | String intervalTrunc; 11 | String intervalQuantity; 12 | 13 | public static ResultSet getList(String search, String limit, String offset, String sortBy, String sort) throws SQLException 14 | { 15 | 16 | String strSQL = "SELECT '' ||\n"; 17 | strSQL += "' ' ||\n"; 18 | strSQL += "' ' AS manage,\n"; 19 | strSQL += "description, initcap(interval_trunc) as interval_trunc, initcap(interval_quantity) as interval_quantity\n"; 20 | strSQL += "FROM os.schedule\n"; 21 | 22 | if (!search.equals("")) 23 | { 24 | strSQL += "WHERE LOWER(description) LIKE '%' || LOWER('" + search + "') || '%'\n"; 25 | strSQL += "OR LOWER(interval_trunc) LIKE '%' || LOWER('" + search + "') || '%'\n"; 26 | strSQL += "OR LOWER(interval_quantity) LIKE '%' || LOWER('" + search + "') || '%'\n"; 27 | } 28 | sortBy = sortBy.toLowerCase(); 29 | if (sortBy.equals("description") || sortBy.equals("interval_trunc") || sortBy.equals("interval_quantity")) 30 | strSQL += "ORDER BY " + sortBy + " " + sort + "\n"; 31 | else 32 | strSQL += "ORDER BY description ASC\n"; 33 | 34 | if (!limit.equals("")) 35 | strSQL += "LIMIT " + limit + " "; 36 | 37 | if (!offset.equals("")) 38 | strSQL += "OFFSET " + offset; 39 | 40 | try 41 | { 42 | ResultSet rs = OutsourcerModel.getResults(strSQL); 43 | return rs; 44 | } 45 | catch (SQLException ex) 46 | { 47 | throw new SQLException(ex.getMessage()); 48 | } 49 | } 50 | 51 | public static void insertTable(String description, String intervalTrunc, String intervalQuantity) throws SQLException 52 | { 53 | description = OutsourcerModel.setSQLString(description); 54 | intervalTrunc = OutsourcerModel.setSQLString(intervalTrunc); 55 | intervalQuantity = OutsourcerModel.setSQLString(intervalQuantity); 56 | 57 | String strSQL = "INSERT INTO os.ao_schedule\n"; 58 | strSQL += "(description, interval_trunc, interval_quantity)\n"; 59 | strSQL += "VALUES (" + description + ",\n"; 60 | strSQL += " " + intervalTrunc + ", " + intervalQuantity + ")"; 61 | 62 | try 63 | { 64 | OutsourcerModel.updateTable(strSQL); 65 | } 66 | catch (SQLException ex) 67 | { 68 | throw new SQLException(ex.getMessage()); 69 | } 70 | } 71 | 72 | public static void deleteTable(String description) throws SQLException 73 | { 74 | description = OutsourcerModel.setSQLString(description); 75 | 76 | try 77 | { 78 | //update jobs.schedule_desc to null if removing a matching schedule 79 | String strSQL = "INSERT INTO os.ao_job\n"; 80 | strSQL += "(id, refresh_type, target_schema_name, target_table_name, target_append_only,\n"; 81 | strSQL += "target_compressed, target_row_orientation, source_type, source_server_name,\n"; 82 | strSQL += "source_instance_name, source_port, source_database_name, source_schema_name,\n"; 83 | strSQL += "source_table_name, source_user_name, source_pass, column_name,\n"; 84 | strSQL += "sql_text, snapshot, schedule_desc, schedule_next, schedule_change)\n"; 85 | strSQL += "SELECT id, refresh_type, target_schema_name, target_table_name, target_append_only,\n"; 86 | strSQL += "target_compressed, target_row_orientation, source_type, source_server_name,\n"; 87 | strSQL += "source_instance_name, source_port, source_database_name, source_schema_name,\n"; 88 | strSQL += "source_table_name, source_user_name, source_pass, column_name,\n"; 89 | strSQL += "sql_text, snapshot, null as schedule_desc, schedule_next, schedule_change\n"; 90 | strSQL += "FROM os.job\n"; 91 | strSQL += "WHERE schedule_desc = " + description; 92 | OutsourcerModel.updateTable(strSQL); 93 | 94 | //remove the schedule 95 | strSQL = "INSERT INTO os.ao_schedule\n"; 96 | strSQL += "(description, interval_trunc, interval_quantity, deleted)\n"; 97 | strSQL += "SELECT description, interval_trunc, interval_quantity, TRUE AS deleted\n"; 98 | strSQL += "FROM os.schedule\n"; 99 | strSQL += "WHERE description = " + description; 100 | OutsourcerModel.updateTable(strSQL); 101 | 102 | } 103 | catch (SQLException ex) 104 | { 105 | throw new SQLException(ex.getMessage()); 106 | } 107 | } 108 | 109 | public ScheduleModel (String aDescription) throws SQLException 110 | { 111 | aDescription = OutsourcerModel.setSQLString(aDescription); 112 | 113 | String strSQL = "SELECT description, interval_trunc, interval_quantity\n"; 114 | strSQL += "FROM os.schedule\n"; 115 | strSQL += "WHERE description = " + aDescription; 116 | 117 | try 118 | { 119 | ResultSet rs = OutsourcerModel.getResults(strSQL); 120 | while (rs.next()) 121 | { 122 | description = rs.getString(1); 123 | intervalTrunc = rs.getString(2); 124 | intervalQuantity = rs.getString(3); 125 | } 126 | } 127 | catch (SQLException ex) 128 | { 129 | //do something?? 130 | } 131 | 132 | } 133 | 134 | public static ScheduleModel getModel(String description) 135 | { 136 | try 137 | { 138 | return new ScheduleModel(description); 139 | } 140 | catch (Exception ex) 141 | { 142 | return null; 143 | } 144 | } 145 | 146 | public static ArrayList getDescriptions() throws SQLException 147 | { 148 | String strSQL = "SELECT description\n"; 149 | strSQL += "FROM os.schedule\n"; 150 | strSQL += "ORDER BY description"; 151 | 152 | ArrayList descriptions = new ArrayList(); 153 | 154 | try 155 | { 156 | descriptions = OutsourcerModel.getStringArray(strSQL); 157 | return descriptions; 158 | } 159 | catch (SQLException ex) 160 | { 161 | throw new SQLException(ex.getMessage()); 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/ServerRunnerUI.java: -------------------------------------------------------------------------------- 1 | import fi.iki.elonen.*; 2 | import java.io.IOException; 3 | 4 | public class ServerRunnerUI 5 | { 6 | public static void run(Class serverClass) 7 | { 8 | try 9 | { 10 | executeInstance((NanoHTTPD) serverClass.newInstance()); 11 | } 12 | catch (Exception e) 13 | { 14 | e.printStackTrace(); 15 | } 16 | } 17 | 18 | public static void executeInstance(NanoHTTPD server) 19 | { 20 | try 21 | { 22 | server.start(); 23 | } 24 | catch (IOException ioe) 25 | { 26 | System.err.println("Couldn't start server:\n" + ioe); 27 | System.exit(-1); 28 | } 29 | 30 | System.out.println("Server started.\n"); 31 | 32 | boolean loop = true; 33 | while (loop) 34 | { 35 | try 36 | { 37 | Thread.sleep(5000); 38 | } 39 | catch (Exception e) 40 | { 41 | server.stop(); 42 | System.out.println("Server stopped.\n"); 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/UI.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import fi.iki.elonen.*; 3 | import java.sql.*; 4 | import java.io.IOException; 5 | 6 | public class UI extends NanoHTTPD 7 | { 8 | public static int webPort = 8080; 9 | public static String gpVersion = "HEAP"; 10 | public static String sessions = ""; 11 | public static String configFile = ""; 12 | 13 | public static void main(String[] args) 14 | { 15 | for (int i = 0; i < args.length; ++i) 16 | { 17 | if (i == 0) 18 | webPort = Integer.parseInt(args[0]); 19 | if (i == 1) 20 | sessions = args[1]; 21 | if (i == 2) 22 | configFile = args[2]; 23 | } 24 | gpVersion = UIModel.getVersion(); 25 | ServerRunnerUI.run(UI.class); 26 | } 27 | 28 | public UI() 29 | { 30 | super(webPort); 31 | } 32 | 33 | @Override 34 | public Response serve(String uri, Method method, Map header, Map parms, Map files) 35 | { 36 | 37 | String username = parms.get("username"); 38 | String password = parms.get("password"); 39 | String submit = parms.get("submit_form"); 40 | String action_type = parms.get("action_type"); 41 | String loginMessage = ""; 42 | boolean alive = false; 43 | 44 | String msg = ""; 45 | boolean auth = false; 46 | 47 | if (submit == null || submit.equals("")) 48 | submit = "0"; 49 | 50 | if (username == null) 51 | username = ""; 52 | 53 | if (password == null) 54 | password = ""; 55 | 56 | if (action_type == null) 57 | action_type = ""; 58 | 59 | String sessionID = UIModel.getSessionID(header); 60 | 61 | if (sessionID == null) 62 | sessionID = "0"; 63 | 64 | if (action_type.equals("logout")) 65 | { 66 | sessionID = "0"; 67 | } 68 | 69 | if (!uri.equals("/favicon.ico")) 70 | { 71 | if (submit.equals("1") && (!username.equals("")) && (!password.equals(""))) 72 | { 73 | try 74 | { 75 | auth = UIModel.authenticate(username, password); 76 | } 77 | catch (SQLException ex) 78 | { 79 | loginMessage = ex.getMessage(); 80 | } 81 | if (auth) 82 | { 83 | sessionID = UIModel.setSessionID(header); 84 | try 85 | { 86 | UIModel.insertSession(sessionID); 87 | } 88 | catch (SQLException ex) 89 | { 90 | loginMessage = ex.getMessage(); 91 | } 92 | } 93 | else if (loginMessage.equals("")) 94 | { 95 | loginMessage = "Failed to login. Must use a valid Greenplum database account that is a SuperUser."; 96 | } 97 | } 98 | 99 | if (!(sessionID.equals("0"))) 100 | { 101 | try 102 | { 103 | alive = UIModel.keepAlive(sessionID); 104 | } 105 | catch (SQLException ex) 106 | { 107 | loginMessage = ex.getMessage(); 108 | } 109 | } 110 | if (alive) 111 | msg = OutsourcerControl.buildPage(uri, parms); 112 | else 113 | sessionID = "0"; 114 | 115 | if (sessionID.equals("0")) 116 | { 117 | msg = UIView.viewLogin(loginMessage); 118 | } 119 | } 120 | 121 | NanoHTTPD.Response out = new NanoHTTPD.Response(msg); 122 | 123 | if (!uri.equals("/favicon.ico")) 124 | { 125 | out.addHeader("Set-Cookie", "OutsourcerSessionID=" + sessionID + ";"); 126 | UIModel.logger(uri, sessionID); 127 | } 128 | return out; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/UIConnectionFactory.java: -------------------------------------------------------------------------------- 1 | import java.sql.*; 2 | import org.postgresql.ds.*; 3 | import java.io.IOException; 4 | 5 | public class UIConnectionFactory 6 | { 7 | 8 | // example to use: 9 | // Connection conn = UIConnectionFactory.getConnection(); 10 | 11 | private static PGPoolingDataSource ds; 12 | 13 | private UIConnectionFactory() 14 | { 15 | } 16 | 17 | public static synchronized PGPoolingDataSource getDataSource() throws SQLException 18 | { 19 | if (ds == null) 20 | { 21 | try 22 | { 23 | System.out.println("Creating new connection pool"); 24 | OSProperties.getPropValues(UI.configFile); 25 | ds = new PGPoolingDataSource(); 26 | ds.setDataSourceName("OutsourcerUIPool"); 27 | ds.setServerName(OSProperties.gpServer); 28 | ds.setDatabaseName(OSProperties.gpDatabase); 29 | ds.setPortNumber(OSProperties.gpPort); 30 | ds.setUser(OSProperties.gpUserName); 31 | ds.setPassword(OSProperties.gpPassword); 32 | ds.setMaxConnections(10); 33 | } 34 | catch (IOException iox) 35 | { 36 | System.out.println("Unable to load config file. Check environment variables and try again."); 37 | throw new SQLException(iox.getMessage()); 38 | } 39 | } 40 | return ds; 41 | } 42 | 43 | public static Connection getConnection() throws SQLException 44 | { 45 | try 46 | { 47 | return getDataSource().getConnection(); 48 | } 49 | catch (SQLException ex) 50 | { 51 | System.out.println(ex.getMessage()); 52 | System.out.println("gpServer from properties: " + OSProperties.gpServer); 53 | System.out.println("gpDatabase from properties: " + OSProperties.gpDatabase); 54 | System.out.println("gpPort from properties: " + OSProperties.gpPort); 55 | System.out.println("gpUsername from properties: " + OSProperties.gpUserName); 56 | System.out.println("gpPassword from properties: " + OSProperties.gpPassword); 57 | throw new SQLException(ex.getMessage()); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/UIModel.java: -------------------------------------------------------------------------------- 1 | import java.util.Map; 2 | import java.util.Random; 3 | import java.sql.*; 4 | import java.util.Date; 5 | import java.util.TimeZone; 6 | import java.text.SimpleDateFormat; 7 | import java.io.File; 8 | import java.io.FileWriter; 9 | import java.io.IOException; 10 | 11 | public class UIModel 12 | { 13 | public static void logger(String uri, String sessionID) 14 | { 15 | Date now = new Date(); 16 | System.out.println(now.toString() + "|" + sessionID + "|" + uri); 17 | } 18 | 19 | public static String getSessionID(Map header) 20 | { 21 | 22 | String sessionID = "0"; 23 | String cookies = header.get("cookie"); 24 | if (cookies != null) 25 | { 26 | String[] cookieValues = cookies.split(";"); 27 | for (int i = 0; i < cookieValues.length; i++) 28 | { 29 | String[] split = cookieValues[i].trim().split("="); 30 | if (split.length == 2) 31 | { 32 | if (split[0].equals("OutsourcerSessionID")) 33 | { 34 | sessionID = split[1]; 35 | } 36 | } 37 | } 38 | } 39 | return sessionID; 40 | } 41 | 42 | public static void insertSession(String sessionID) throws SQLException 43 | { 44 | try 45 | { 46 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 47 | sdf.setTimeZone(TimeZone.getTimeZone("UTC")); 48 | String expireDate = sdf.format(new Date(System.currentTimeMillis()+15*60*1000)); 49 | 50 | File file = new File(UI.sessions); 51 | if(!file.exists()) 52 | { 53 | file.createNewFile(); 54 | } 55 | FileWriter writer = new FileWriter(file, true); 56 | writer.write(sessionID + "|'" + expireDate + "'\n"); 57 | writer.flush(); 58 | writer.close(); 59 | 60 | } 61 | catch (IOException ex) 62 | { 63 | throw new SQLException(ex.getMessage()); 64 | } 65 | } 66 | 67 | public static boolean authenticate(String username, String password) throws SQLException 68 | { 69 | boolean authenticated = false; 70 | username = username.toLowerCase(); 71 | try 72 | { 73 | //don't use the pool to authenticate 74 | Connection conn = CommonDB.connectGP(UI.configFile, username, password); 75 | 76 | String strSQL = "SELECT rolname FROM pg_roles WHERE rolsuper AND rolcanlogin AND rolname = '" + username + "'"; 77 | Statement stmt = conn.createStatement(); 78 | ResultSet rs = stmt.executeQuery(strSQL); 79 | while (rs.next()) 80 | { 81 | authenticated = true; 82 | } 83 | 84 | conn.close(); 85 | 86 | } 87 | catch (SQLException ex) 88 | { 89 | throw new SQLException(ex.getMessage()); 90 | } 91 | return authenticated; 92 | } 93 | 94 | public static String setSessionID(Map header) 95 | { 96 | String sessionID = getSessionID(header); 97 | if (sessionID.equals("0")) 98 | { 99 | Random session = new Random(); 100 | int sessionInt = session.nextInt(1000000); 101 | sessionID = Integer.toString(sessionInt); 102 | } 103 | return sessionID; 104 | 105 | } 106 | 107 | public static boolean keepAlive(String sessionID) throws SQLException 108 | { 109 | boolean alive = false; 110 | boolean activeSession = false; 111 | try 112 | { 113 | Connection conn = UIConnectionFactory.getConnection(); 114 | Statement stmt = conn.createStatement(); 115 | 116 | //make sure session is valid 117 | String strSQL = "SELECT session_id FROM os.sessions WHERE session_id = " + sessionID + " AND expire_date > current_timestamp AT TIME ZONE 'UTC' LIMIT 1"; 118 | ResultSet rs = stmt.executeQuery(strSQL); 119 | while (rs.next()) 120 | { 121 | activeSession = true; 122 | } 123 | conn.close(); 124 | 125 | if (activeSession) 126 | { 127 | insertSession(sessionID); 128 | alive = true; 129 | } 130 | 131 | } 132 | catch (SQLException ex) 133 | { 134 | System.out.println(ex.getMessage()); 135 | throw new SQLException(ex.getMessage()); 136 | } 137 | return alive; 138 | } 139 | 140 | public static String getVersion() 141 | { 142 | String version = "HEAP"; 143 | 144 | try 145 | { 146 | Connection conn = UIConnectionFactory.getConnection(); 147 | Statement stmt = conn.createStatement(); 148 | 149 | String strSQL = "SELECT CASE WHEN position ('HAWQ' IN version()) > 0 THEN 'HAWQ'\n"; 150 | strSQL += " WHEN position ('HAWQ' IN version()) = 0\n"; 151 | strSQL += " AND (split_part(split_part(substr(version(), position('Greenplum Database' in version())+19), ' ', 1), '.', 1))::int >= 4\n"; 152 | strSQL += " AND (split_part(split_part(substr(version(), position('Greenplum Database' in version())+19), ' ', 1), '.', 2))::int >= 3 THEN 'AO'\n"; 153 | strSQL += " ELSE 'HEAP'\n"; 154 | strSQL += "END"; 155 | 156 | ResultSet rs = stmt.executeQuery(strSQL); 157 | while (rs.next()) 158 | { 159 | version = rs.getString(1); 160 | } 161 | 162 | conn.close(); 163 | } 164 | catch (SQLException ex) 165 | { 166 | 167 | } 168 | return version; 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/UIView.java: -------------------------------------------------------------------------------- 1 | public class UIView 2 | { 3 | public static String viewLogin(String loginMessage) 4 | { 5 | 6 | String myScript = getJavaScriptFunctions(); 7 | String onLoad = "login()"; 8 | String msg = OutsourcerView.getHead(myScript, onLoad); 9 | msg += "
\n"; 10 | msg += "\n"; 11 | msg += "\n"; 12 | msg += "

Oracle2GP

\n"; 13 | msg += "
\n"; 14 | msg += "\n"; 15 | msg += "\n"; 16 | msg += "\n"; 33 | if (!(loginMessage == "")) 34 | msg += "\n"; 35 | msg += "
\n"; 17 | msg += "\n"; 18 | msg += "\n"; 19 | msg += "\n"; 20 | msg += "\n"; 21 | msg += "\n"; 22 | msg += "\n"; 23 | msg += "\n"; 24 | msg += "\n"; 25 | msg += "\n"; 26 | msg += "\n"; 27 | msg += "\n"; 28 | msg += "\n"; 29 | msg += "\n"; 30 | msg += "\n"; 31 | msg += "
用户名:
密码:
\n"; 32 | msg += "
" + loginMessage + "
\n"; 36 | msg += "
\n"; 37 | msg += "\n"; 38 | return msg; 39 | } 40 | 41 | private static String getJavaScriptFunctions() 42 | { 43 | String myScript = "function login()\n"; 44 | myScript += "{\n"; 45 | myScript += " document.getElementById(\"username\").focus()\n"; 46 | myScript += "}\n"; 47 | 48 | return myScript; 49 | } 50 | } -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$1$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$1$1.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$1.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$AsyncRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$AsyncRunner.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$DefaultAsyncRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$DefaultAsyncRunner.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$DefaultTempFile.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$DefaultTempFile.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$DefaultTempFileManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$DefaultTempFileManager.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$DefaultTempFileManagerFactory.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$DefaultTempFileManagerFactory.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$HTTPSession.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$HTTPSession.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$Method.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$Method.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$Response$Status.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$Response$Status.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$Response.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$Response.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$TempFile.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$TempFile.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$TempFileManager.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$TempFileManager.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD$TempFileManagerFactory.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD$TempFileManagerFactory.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/NanoHTTPD.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/NanoHTTPD.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/ServerRunner.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/ServerRunner.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/SimpleWebServer$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/SimpleWebServer$1.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/SimpleWebServer$2.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/SimpleWebServer$2.class -------------------------------------------------------------------------------- /src/fi/iki/elonen/SimpleWebServer.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MLikeWater/Oracle2GP/e360aa6db09420213c56e944225f14f8a5f51e6d/src/fi/iki/elonen/SimpleWebServer.class -------------------------------------------------------------------------------- /src/manifest.txt: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Main-Class: ExternalData 3 | Specification-Title: "Outsourcer" 4 | Specification-Version: "1.0" 5 | Created-By: 1.6.0_65-b14-462-11M4609 6 | Build-Date: Wed May 18 11:10:56 EDT 2016 7 | -------------------------------------------------------------------------------- /yml/oracle2gp.yml: -------------------------------------------------------------------------------- 1 | --- 2 | VERSION: 1.0.0.1 3 | TRANSFORMATIONS: 4 | externaldata: 5 | TYPE: input 6 | CONTENT: data 7 | COMMAND: /bin/bash bin/getdata %filename% 8 | 9 | agentstop: 10 | TYPE: input 11 | CONTENT: data 12 | COMMAND: /bin/bash bin/agentstop 13 | 14 | agentstart: 15 | TYPE: input 16 | CONTENT: data 17 | COMMAND: /bin/bash bin/agentstart 18 | 19 | agentstatus: 20 | TYPE: input 21 | CONTENT: data 22 | COMMAND: /bin/bash bin/agentstatus 23 | 24 | osstart: 25 | TYPE: input 26 | CONTENT: data 27 | COMMAND: /bin/bash bin/osstart 28 | 29 | osstop: 30 | TYPE: input 31 | CONTENT: data 32 | COMMAND: /bin/bash bin/osstop 33 | 34 | osstatus: 35 | TYPE: input 36 | CONTENT: data 37 | COMMAND: /bin/bash bin/osstatus 38 | 39 | uistop: 40 | TYPE: input 41 | CONTENT: data 42 | COMMAND: /bin/bash bin/uistop 43 | 44 | uistart: 45 | TYPE: input 46 | CONTENT: data 47 | COMMAND: /bin/bash bin/uistart 48 | 49 | sessions: 50 | TYPE: input 51 | CONTENT: data 52 | COMMAND: /bin/cat log/Oracle2GP-sessions.log 53 | 54 | --------------------------------------------------------------------------------