├── src ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── io.digdag.spi.Plugin │ └── java │ │ └── com │ │ └── takemikami │ │ └── github │ │ └── digdag │ │ └── plugin │ │ └── shresult │ │ ├── ShResultPlugin.java │ │ └── ShResultOperatorFactory.java └── test │ └── java │ └── com │ └── takemikami │ └── github │ └── digdag │ └── plugin │ └── shresult │ └── ShResultOperatorFactoryTest.java ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── .github ├── ISSUE_TEMPLATE.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── ci.yaml ├── .gitignore ├── LICENSE ├── gradlew.bat ├── README.md ├── gradlew └── config └── checkstyle └── checkstyle.xml /src/main/resources/META-INF/services/io.digdag.spi.Plugin: -------------------------------------------------------------------------------- 1 | com.takemikami.github.digdag.plugin.shresult.ShResultPlugin -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takemikami/digdag-plugin-shresult/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | _Describe the problem or feature._ 3 | 4 | ## Proposal 5 | _How does this change address the problem?_ 6 | 7 | ## Problem Information 8 | _Describe environment and reproduction scenario for the problem, when this issue is defect or problem._ 9 | 10 | ## TODOs 11 | - [ ] Use github checklists. When done, check the box. 12 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | _Describe the problem or feature in addition to a link to the issues._ 3 | 4 | ## Approach 5 | _How does this change address the problem?_ 6 | 7 | #### Open Questions and Pre-Merge TODOs 8 | - [ ] Use github checklists. When solved, check the box and explain the answer. 9 | 10 | ## Learning 11 | _Describe the research stage_ 12 | 13 | _Links to blog posts, patterns, libraries or addons used to solve this problem_ 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Java CI 2 | 3 | on: [ push ] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | - name: Set up JDK 1.8 13 | uses: actions/setup-java@v1 14 | with: 15 | java-version: 1.8 16 | 17 | - name: Cache Gradle packages 18 | uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/.gradle/caches 22 | ~/.gradle/wrapper 23 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 24 | restore-keys: | 25 | ${{ runner.os }}-gradle- 26 | 27 | - name: Test 28 | run: ./gradlew build 29 | 30 | - name: Report 31 | run: ./gradlew jacocoTestReport coveralls 32 | env: 33 | COVERALLS_REPO_TOKEN: ${{secrets.COVERALLS_REPO_TOKEN}} 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #### for Java, see https://github.com/github/gitignore/blob/master/Java.gitignore 2 | # Compiled class file 3 | *.class 4 | 5 | # Log file 6 | *.log 7 | 8 | # BlueJ files 9 | *.ctxt 10 | 11 | # Mobile Tools for Java (J2ME) 12 | .mtj.tmp/ 13 | 14 | # Package Files # 15 | *.jar 16 | *.war 17 | *.nar 18 | *.ear 19 | *.zip 20 | *.tar.gz 21 | *.rar 22 | 23 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 24 | hs_err_pid* 25 | 26 | #### for Gradle, see https://github.com/github/gitignore/blob/master/Gradle.gitignore 27 | .gradle 28 | /build/ 29 | 30 | # Ignore Gradle GUI config 31 | gradle-app.setting 32 | 33 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 34 | !gradle-wrapper.jar 35 | 36 | # Cache of project 37 | .gradletasknamecache 38 | 39 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 40 | # gradle/wrapper/gradle-wrapper.properties -------------------------------------------------------------------------------- /src/main/java/com/takemikami/github/digdag/plugin/shresult/ShResultPlugin.java: -------------------------------------------------------------------------------- 1 | package com.takemikami.github.digdag.plugin.shresult; 2 | 3 | import com.google.inject.Inject; 4 | import io.digdag.spi.CommandExecutor; 5 | import io.digdag.spi.OperatorFactory; 6 | import io.digdag.spi.OperatorProvider; 7 | import io.digdag.spi.Plugin; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | public class ShResultPlugin implements Plugin { 12 | 13 | @Override 14 | public Class getServiceProvider(Class type) { 15 | if (type == OperatorProvider.class) { 16 | return ShResultOperatorProvider.class.asSubclass(type); 17 | } else { 18 | return null; 19 | } 20 | } 21 | 22 | public static class ShResultOperatorProvider implements OperatorProvider { 23 | 24 | @Inject 25 | protected CommandExecutor commandExecutor; 26 | 27 | @Override 28 | public List get() { 29 | return Arrays.asList(new ShResultOperatorFactory(commandExecutor)); 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Mikami, Takeshi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/test/java/com/takemikami/github/digdag/plugin/shresult/ShResultOperatorFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.takemikami.github.digdag.plugin.shresult; 2 | 3 | import static junit.framework.TestCase.assertEquals; 4 | 5 | import com.google.common.base.Optional; 6 | import com.google.inject.Inject; 7 | import io.digdag.client.config.ConfigException; 8 | import io.digdag.spi.CommandExecutor; 9 | import io.digdag.spi.PrivilegedVariables; 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | import org.junit.Test; 15 | 16 | public class ShResultOperatorFactoryTest { 17 | 18 | @Inject 19 | protected CommandExecutor commandExecutor; 20 | 21 | @Test 22 | public void testType() throws Exception { 23 | ShResultOperatorFactory factory = new ShResultOperatorFactory(commandExecutor); 24 | assertEquals("sh_result", factory.getType()); 25 | } 26 | 27 | @Test 28 | public void testCreateVariableObjectFromStdoutText() throws Exception { 29 | String stdoutData = "text message"; 30 | String obj = (String) ShResultOperatorFactory 31 | .createVariableObjectFromStdout(stdoutData, "text"); 32 | assertEquals("text message", obj); 33 | } 34 | 35 | @Test 36 | public void testCreateVariableObjectFromStdoutJsonListMap() throws Exception { 37 | String stdoutData = "[{\"id\": \"001\",\"name\": \"hoge\"},{\"id\": \"002\",\"name\": \"fuga\"}]"; 38 | List> obj = (List>) ShResultOperatorFactory 39 | .createVariableObjectFromStdout(stdoutData, "json-list-map"); 40 | assertEquals(2, obj.size()); 41 | assertEquals("001", obj.get(0).get("id")); 42 | assertEquals("hoge", obj.get(0).get("name")); 43 | } 44 | 45 | @Test 46 | public void testCreateVariableObjectFromStdoutNewlineDelimited() throws Exception { 47 | String stdoutData = "hoge\nfuga\n"; 48 | List obj = (List) ShResultOperatorFactory 49 | .createVariableObjectFromStdout(stdoutData, "newline-delimited"); 50 | assertEquals(2, obj.size()); 51 | assertEquals("hoge", obj.get(0)); 52 | } 53 | 54 | @Test 55 | public void testCreateVariableObjectFromStdoutSpaceDelimited() throws Exception { 56 | String stdoutData = "hoge fuga\nfoo bar"; 57 | List obj = (List) ShResultOperatorFactory 58 | .createVariableObjectFromStdout(stdoutData, "space-delimited"); 59 | System.out.println(obj); 60 | assertEquals(4, obj.size()); 61 | assertEquals("hoge", obj.get(0)); 62 | assertEquals("bar", obj.get(3)); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # digdag-plugin-shresult 2 | 3 | [![JitPack](https://jitpack.io/v/takemikami/digdag-plugin-shresult.svg)](https://jitpack.io/#takemikami/digdag-plugin-shresult) 4 | [![Build Status](https://travis-ci.org/takemikami/digdag-plugin-shresult.svg?branch=master)](https://travis-ci.org/takemikami/digdag-plugin-shresult) 5 | [![Coverage Status](https://coveralls.io/repos/github/takemikami/digdag-plugin-shresult/badge.svg?branch=master)](https://coveralls.io/github/takemikami/digdag-plugin-shresult?branch=master) 6 | 7 | digdag-plugin-shresult is plugin storing output of shell to digdag store. 8 | 9 | ## Getting Started 10 | 11 | setup digdag, see https://www.digdag.io/ 12 | 13 | create digdag workflow, following code is sample workflow. 14 | 15 | sample.dig ... for each by resultset of BigQuery. 16 | 17 | ``` 18 | _export: 19 | plugin: 20 | repositories: 21 | - https://jitpack.io 22 | dependencies: 23 | - com.github.takemikami:digdag-plugin-shresult:0.0.3 24 | 25 | +step1: 26 | sh_result>: | 27 | bq query --format=json 'select name from dataset1.table1 limit 10' 28 | destination_variable: resultset 29 | stdout_format: json-list-map 30 | 31 | +step2: 32 | for_each>: 33 | rv: ${resultset} 34 | _parallel: 35 | true 36 | _do: 37 | echo>: ${rv.name} 38 | ``` 39 | 40 | run sample workflow. 41 | 42 | ``` 43 | digdag run sample.dig 44 | ``` 45 | 46 | ## Usage 47 | 48 | ### sh_result>: Shell scripts and store output 49 | 50 | sh_result> operator runs a shell scripts and set output to digdag store. 51 | 52 | #### Options: 53 | 54 | - sh_result>: COMMAND [ARGS...] 55 | 56 | Name of the command to run. 57 | 58 | - destination_variable: Name 59 | 60 | Specifies a digdag-variable to store the standard output of script in. 61 | 62 | - stdout_format: Name 63 | 64 | Type of standard output format. 65 | 66 | - text 67 | - newline-delimited 68 | - space-delimited 69 | - json-list-map 70 | 71 | #### Stdout Formats 72 | 73 | - text ... set each stdout to single value. 74 | 75 | example 76 | ``` 77 | text message 78 | ``` 79 | 80 | - newline-delimited ... set newline-delimited stdout to list. 81 | 82 | example 83 | ``` 84 | item1 85 | item2 86 | item3 87 | ``` 88 | 89 | - space-delimited ... set space-delimited stdout to list. 90 | 91 | example 92 | ``` 93 | item1 item2 item3 94 | ``` 95 | 96 | - json-list-map ... set json-based stdout to list of map. 97 | 98 | example 99 | ``` 100 | [ 101 | { 102 | "id": "001", 103 | "name": "item1" 104 | }, 105 | { 106 | "id": "002", 107 | "name": "item2" 108 | } 109 | ] 110 | ``` 111 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /src/main/java/com/takemikami/github/digdag/plugin/shresult/ShResultOperatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.takemikami.github.digdag.plugin.shresult; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.JsonNode; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import com.fasterxml.jackson.databind.node.ObjectNode; 7 | import com.google.common.base.Throwables; 8 | import com.google.common.collect.ImmutableList; 9 | import com.google.common.collect.Maps; 10 | import com.google.inject.Inject; 11 | import io.digdag.client.config.Config; 12 | import io.digdag.client.config.ConfigElement; 13 | import io.digdag.client.config.ConfigFactory; 14 | import io.digdag.spi.CommandContext; 15 | import io.digdag.spi.CommandExecutor; 16 | import io.digdag.spi.CommandRequest; 17 | import io.digdag.spi.CommandStatus; 18 | import io.digdag.spi.Operator; 19 | import io.digdag.spi.OperatorContext; 20 | import io.digdag.spi.OperatorFactory; 21 | import io.digdag.spi.TaskExecutionException; 22 | import io.digdag.spi.TaskResult; 23 | import io.digdag.util.BaseOperator; 24 | import io.digdag.util.CommandOperators; 25 | import io.digdag.util.UserSecretTemplate; 26 | import java.io.ByteArrayOutputStream; 27 | import java.io.IOException; 28 | import java.io.PrintStream; 29 | import java.io.Writer; 30 | import java.nio.charset.StandardCharsets; 31 | import java.nio.file.Files; 32 | import java.nio.file.Path; 33 | import java.time.Duration; 34 | import java.util.ArrayList; 35 | import java.util.HashMap; 36 | import java.util.LinkedList; 37 | import java.util.List; 38 | import java.util.Map; 39 | import org.slf4j.Logger; 40 | import org.slf4j.LoggerFactory; 41 | 42 | // see https://github.com/treasure-data/digdag/blob/master/digdag-standards/src/main/java/io/digdag/standards/operator/ShOperatorFactory.java 43 | public class ShResultOperatorFactory implements OperatorFactory { 44 | 45 | private static Logger logger = LoggerFactory.getLogger(ShResultOperatorFactory.class); 46 | 47 | private final CommandExecutor exec; 48 | 49 | @Inject 50 | public ShResultOperatorFactory(CommandExecutor exec) { 51 | this.exec = exec; 52 | } 53 | 54 | @Override 55 | public String getType() { 56 | return "sh_result"; 57 | } 58 | 59 | @Override 60 | public Operator newOperator(OperatorContext operatorContext) { 61 | return new ShResultOperator(operatorContext); 62 | } 63 | 64 | class ShResultOperator extends BaseOperator { 65 | 66 | // TODO extract as config params. 67 | final int scriptPollInterval = (int) Duration.ofSeconds(10).getSeconds(); 68 | 69 | public ShResultOperator(OperatorContext context) { 70 | super(context); 71 | } 72 | 73 | @Override 74 | public TaskResult runTask() { 75 | final Config state = request.getConfig(); 76 | Config params = state.mergeDefault(request.getConfig().getNestedOrGetEmpty("sh")); 77 | 78 | // save System.out 79 | PrintStream console = System.out; 80 | 81 | try { 82 | // prepare capture stdout 83 | final ByteArrayOutputStream baos = new ByteArrayOutputStream(); 84 | final String utf8 = StandardCharsets.UTF_8.name(); 85 | PrintStream ps = new PrintStream(baos); 86 | System.setOut(ps); 87 | 88 | // run script 89 | runCode(state); 90 | 91 | // capture stdout 92 | String stdoutData = baos.toString(utf8); 93 | 94 | String varName = params.get("destination_variable", String.class); 95 | String stdoutFormat = params.get("stdout_format", String.class); 96 | 97 | ConfigFactory cf = request.getConfig().getFactory(); 98 | Config storeParams = cf.create(); 99 | storeParams.set(varName, createVariableObjectFromStdout(stdoutData, stdoutFormat)); 100 | 101 | return TaskResult.defaultBuilder(request) 102 | .storeParams(storeParams) 103 | .build(); 104 | } catch (IOException | InterruptedException e) { 105 | throw Throwables.propagate(e); 106 | } finally { 107 | // restore System.out 108 | System.setOut(console); 109 | } 110 | } 111 | 112 | private void runCode(final Config state) 113 | throws IOException, InterruptedException { 114 | final Config params = request.getConfig() 115 | .mergeDefault(request.getConfig().getNestedOrGetEmpty("sh")); 116 | final Path projectPath = workspace.getProjectPath(); 117 | final CommandContext commandContext = buildCommandContext(projectPath); 118 | 119 | final CommandStatus status; 120 | if (!state.has("commandStatus")) { 121 | // Run the code since command state doesn't exist 122 | status = runCommand(params, commandContext); 123 | } else { 124 | // Check the status of the running command 125 | final ObjectNode previousStatusJson = state.get("commandStatus", ObjectNode.class); 126 | status = exec.poll(commandContext, previousStatusJson); 127 | } 128 | 129 | if (status.isFinished()) { 130 | final int statusCode = status.getStatusCode(); 131 | if (statusCode != 0) { 132 | // Remove the polling state after fetching the result so that the result fetch can be retried 133 | // without resubmitting the code. 134 | state.remove("commandStatus"); 135 | throw new RuntimeException("Command failed with code " + statusCode); 136 | } 137 | return; 138 | } else { 139 | state.set("commandStatus", status); 140 | throw TaskExecutionException.ofNextPolling(scriptPollInterval, ConfigElement.copyOf(state)); 141 | } 142 | } 143 | 144 | private CommandStatus runCommand(final Config params, final CommandContext commandContext) 145 | throws IOException, InterruptedException { 146 | final Path tempDir = workspace 147 | .createTempDir(String.format("digdag-sh-%d-", request.getTaskId())); 148 | final Path workingDirectory = workspace.getPath(); // absolute 149 | final Path runnerPath = tempDir.resolve("runner.sh"); // absolute 150 | 151 | final List shell; 152 | if (params.has("shell")) { 153 | shell = params.getListOrEmpty("shell", String.class); 154 | } else { 155 | shell = ImmutableList.of("/bin/sh"); 156 | } 157 | 158 | final ImmutableList.Builder cmdline = ImmutableList.builder(); 159 | if (params.has("shell")) { 160 | cmdline.addAll(shell); 161 | } else { 162 | cmdline.addAll(shell); 163 | } 164 | cmdline.add(workingDirectory.relativize(runnerPath).toString()); // relative 165 | 166 | final String shScript = UserSecretTemplate.of(params.get("_command", String.class)) 167 | .format(context.getSecrets()); 168 | 169 | final Map environments = Maps.newHashMap(); 170 | params.getKeys() 171 | .forEach(key -> { 172 | if (CommandOperators.isValidEnvKey(key)) { 173 | JsonNode value = params.get(key, JsonNode.class); 174 | String string; 175 | if (value.isTextual()) { 176 | string = value.textValue(); 177 | } else { 178 | string = value.toString(); 179 | } 180 | environments.put(key, string); 181 | } else { 182 | logger.trace("Ignoring invalid env var key: {}", key); 183 | } 184 | }); 185 | 186 | // Set up process environment according to env config. This can also refer to secrets. 187 | CommandOperators.collectEnvironmentVariables(environments, context.getPrivilegedVariables()); 188 | 189 | // Write script content to runnerPath 190 | try (Writer writer = Files.newBufferedWriter(runnerPath)) { 191 | writer.write(shScript); 192 | } 193 | 194 | final CommandRequest commandRequest = buildCommandRequest(commandContext, workingDirectory, 195 | tempDir, environments, cmdline.build()); 196 | return exec.run(commandContext, commandRequest); 197 | 198 | // TaskExecutionException could not be thrown here to poll the task by non-blocking for process-base 199 | // command executor. Because they will be bounded by the _instance_ where the command was executed 200 | // first. 201 | } 202 | 203 | private CommandContext buildCommandContext(final Path projectPath) { 204 | return CommandContext.builder() 205 | .localProjectPath(projectPath) 206 | .taskRequest(this.request) 207 | .build(); 208 | } 209 | 210 | private CommandRequest buildCommandRequest(final CommandContext commandContext, 211 | final Path workingDirectory, 212 | final Path tempDir, 213 | final Map environments, 214 | final List cmdline) { 215 | final Path projectPath = commandContext.getLocalProjectPath(); 216 | final Path relativeWorkingDirectory = projectPath.relativize(workingDirectory); // relative 217 | final Path ioDirectory = projectPath.relativize(tempDir); // relative 218 | return CommandRequest.builder() 219 | .workingDirectory(relativeWorkingDirectory) 220 | .environments(environments) 221 | .commandLine(cmdline) 222 | .ioDirectory(ioDirectory) 223 | .build(); 224 | } 225 | } 226 | 227 | public static Object createVariableObjectFromStdout( 228 | String stdoutData, 229 | String stdoutFormat 230 | ) { 231 | // stdout is text 232 | if ("text".equals(stdoutFormat)) { 233 | return stdoutData; 234 | } 235 | 236 | // case of '*-delimited' 237 | String delimiter = null; 238 | if ("newline-delimited".equals(stdoutFormat)) { 239 | delimiter = "\n"; 240 | } else if ("space-delimited".equals(stdoutFormat)) { 241 | delimiter = "\n| "; 242 | } 243 | if (delimiter != null) { 244 | List listObj = new LinkedList<>(); 245 | for (String s : stdoutData.split(delimiter)) { 246 | if (s.trim().length() > 0) { 247 | listObj.add(s.trim()); 248 | } 249 | } 250 | return listObj; 251 | } 252 | 253 | if ("json-list-map".equals(stdoutFormat)) { 254 | // stdout is json format 255 | List> jsonObj; 256 | try { 257 | ObjectMapper mapper = new ObjectMapper(); 258 | jsonObj = mapper 259 | .readValue(stdoutData, new TypeReference>>() { 260 | }); 261 | } catch (IOException e) { 262 | throw Throwables.propagate(e); 263 | } 264 | return jsonObj; 265 | 266 | } 267 | return null; 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 55 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 72 | 73 | 74 | 76 | 77 | 78 | 84 | 85 | 86 | 87 | 90 | 91 | 92 | 93 | 94 | 98 | 99 | 100 | 101 | 102 | 104 | 105 | 106 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 125 | 127 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 175 | 176 | 177 | 179 | 181 | 182 | 183 | 184 | 186 | 187 | 188 | 189 | 191 | 192 | 193 | 194 | 196 | 197 | 198 | 199 | 201 | 202 | 203 | 204 | 206 | 207 | 208 | 209 | 211 | 212 | 213 | 214 | 216 | 217 | 218 | 219 | 221 | 222 | 223 | 224 | 226 | 227 | 228 | 229 | 231 | 232 | 233 | 234 | 236 | 237 | 238 | 239 | 241 | 243 | 245 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 276 | 277 | 278 | 281 | 282 | 283 | 284 | 290 | 291 | 292 | 293 | 296 | 297 | 298 | 299 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 314 | 315 | 316 | 317 | 318 | 319 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 336 | 343 | 344 | 345 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 361 | 362 | 363 | 364 | --------------------------------------------------------------------------------