├── .github └── workflows │ └── deploy.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── action.yml ├── app.py ├── entrypoint.sh ├── requirements.txt └── test ├── test1.csv ├── test1.txt └── test2.csv /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: scp files 2 | on: 3 | push: 4 | branches: 5 | - latest 6 | 7 | env: 8 | TARGET_DIR: /home/github/test 9 | 10 | jobs: 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: checkout 17 | uses: actions/checkout@v1 18 | 19 | - name: ssh scp ssh pipelines 20 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 21 | env: 22 | WELCOME: "ssh scp ssh pipelines" 23 | LASTSSH: "Doing something after copying" 24 | with: 25 | host: ${{ secrets.DC_HOST }} 26 | user: ${{ secrets.DC_USER }} 27 | pass: ${{ secrets.DC_PASS }} 28 | port: ${{ secrets.DC_PORT }} 29 | connect_timeout: 10s 30 | first_ssh: |- 31 | rm -rf /home/github/test 32 | ls -la 33 | echo $WELCOME 34 | mkdir -p /home/github/test/test1 && 35 | mkdir -p /home/github/test/test2 && 36 | scp: |- 37 | './test/*' => /home/github/test/ 38 | ./test/test1* => $TARGET_DIR/test1/ 39 | ./test/test*.csv => "/home/github/test/test2/" 40 | last_ssh: |- 41 | echo $LASTSSH && 42 | (mkdir test1/test || true) 43 | ls -la 44 | 45 | - name: scp ssh pipelines 46 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 47 | env: 48 | LASTSSH: "Doing something after copying" 49 | with: 50 | host: ${{ secrets.DC_HOST }} 51 | user: ${{ secrets.DC_USER }} 52 | pass: ${{ secrets.DC_PASS }} 53 | scp: |- 54 | ./test/test1* => /home/github/test/test1/ 55 | "." => "$TARGET_DIR/test3/" 56 | last_ssh: |- 57 | echo $LASTSSH 58 | ls -la 59 | 60 | - name: scp pipelines 61 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 62 | env: 63 | WELCOME: "scp pipelines" 64 | TO_DIR: /home/github/test/test4 65 | with: 66 | host: ${{ secrets.DC_HOST }} 67 | user: ${{ secrets.DC_USER }} 68 | pass: ${{ secrets.DC_PASS }} 69 | scp: |- 70 | '.' => $TO_DIR -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | 3 | test.* 4 | .env -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.3-slim-buster 2 | 3 | LABEL "maintainer"="Scott Ng " 4 | LABEL "repository"="https://github.com/cross-the-world/ssh-scp-ssh-pipelines" 5 | LABEL "version"="v1.1.0" 6 | 7 | LABEL "com.github.actions.name"="ssh-scp-ssh-pipelines" 8 | LABEL "com.github.actions.description"="Pipeline: ssh -> scp -> ssh" 9 | LABEL "com.github.actions.icon"="terminal" 10 | LABEL "com.github.actions.color"="gray-dark" 11 | 12 | RUN apt-get update -y && \ 13 | apt-get install -y ca-certificates openssh-client openssl sshpass 14 | 15 | COPY requirements.txt /requirements.txt 16 | RUN pip3 install -r /requirements.txt 17 | 18 | RUN mkdir -p /opt/tools 19 | 20 | COPY entrypoint.sh /opt/tools/entrypoint.sh 21 | RUN chmod +x /opt/tools/entrypoint.sh 22 | 23 | COPY app.py /opt/tools/app.py 24 | RUN chmod +x /opt/tools/app.py 25 | 26 | ENTRYPOINT ["/opt/tools/entrypoint.sh"] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 cross-the-world 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SSH SCP SSH Pipelines 2 | 3 | [Github actions](https://help.github.com/en/actions/creating-actions/creating-a-docker-container-action) 4 | 5 | [SSH action](https://github.com/cross-the-world/ssh-pipeline) 6 | 7 | [SCP action](https://github.com/cross-the-world/scp-pipeline) 8 | 9 | This action allows doing in order 10 | 1. ssh if defined 11 | 2. scp if defined 12 | 3. ssh if defined 13 | 14 | ## Inputs 15 | see the [action.yml](./action.yml) file for more detail imformation. 16 | 17 | ### `host` 18 | 19 | **Required** ssh remote host. 20 | 21 | ### `port` 22 | 23 | **NOT Required** ssh remote port. Default 22 24 | 25 | ### `user` 26 | 27 | **Required** ssh remote user. 28 | 29 | ### `pass` 30 | 31 | **NOT Required** ssh remote pass. 32 | 33 | ### `key` 34 | 35 | **NOT Required** ssh remote key as string. 36 | 37 | ### `connect_timeout` 38 | 39 | **NOT Required** connection timeout to remote host. Default 30s 40 | 41 | ### `first_ssh` 42 | 43 | **NOT Required** execute pre-commands before scp. 44 | 45 | ### `scp` 46 | 47 | **NOT Required** scp from local to remote. 48 | 49 | **Syntax** 50 | local_path => remote_path 51 | e.g. 52 | /opt/test/* => /home/github/test 53 | 54 | ### `last_ssh` 55 | 56 | **NOT Required** execute pre-commands after scp. 57 | 58 | 59 | ## Usages 60 | see the [deploy.yml](./.github/workflows/deploy.yml) file for more detail imformation. 61 | 62 | #### ssh scp ssh pipelines 63 | ```yaml 64 | - name: ssh scp ssh pipelines 65 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 66 | env: 67 | WELCOME: "ssh scp ssh pipelines" 68 | LASTSSH: "Doing something after copying" 69 | with: 70 | host: ${{ secrets.DC_HOST }} 71 | user: ${{ secrets.DC_USER }} 72 | pass: ${{ secrets.DC_PASS }} 73 | port: ${{ secrets.DC_PORT }} 74 | connect_timeout: 10s 75 | first_ssh: | 76 | rm -rf /home/github/test 77 | ls -la \necho $WELCOME 78 | mkdir -p /home/github/test/test1 && 79 | mkdir -p /home/github/test/test2 && 80 | scp: | 81 | './test/*' => /home/github/test/ 82 | ./test/test1* => /home/github/test/test1/ 83 | ./test/test*.csv => "/home/github/test/test2/" 84 | last_ssh: | 85 | echo $LASTSSH && 86 | (mkdir test1/test || true) 87 | || ls -la 88 | ``` 89 | 90 | #### scp ssh pipelines 91 | ```yaml 92 | - name: scp ssh pipelines 93 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 94 | env: 95 | LASTSSH: "Doing something after copying" 96 | with: 97 | host: ${{ secrets.DC_HOST }} 98 | user: ${{ secrets.DC_USER }} 99 | pass: ${{ secrets.DC_PASS }} 100 | scp: | 101 | ./test/test1* => /home/github/test/test1/ 102 | ./test/test*.csv => "/home/github/test/test2/" 103 | last_ssh: | 104 | echo $LASTSSH 105 | ls -la 106 | ``` 107 | 108 | #### scp pipelines 109 | ```yaml 110 | - name: scp pipelines 111 | uses: cross-the-world/ssh-scp-ssh-pipelines@latest 112 | with: 113 | host: ${{ secrets.DC_HOST }} 114 | user: ${{ secrets.DC_USER }} 115 | pass: ${{ secrets.DC_PASS }} 116 | scp: | 117 | './test/*' => /home/github/test/ 118 | ``` 119 | 120 | 121 | -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- 1 | name: 'ssh-scp-ssh-pipelines' 2 | description: 'Pipelines: ssh -> scp -> ssh' 3 | author: 'Scott Ng' 4 | inputs: 5 | host: 6 | description: 'ssh remote host' 7 | required: true 8 | port: 9 | description: 'ssh remote port' 10 | default: 22 11 | user: 12 | description: 'ssh remote user' 13 | required: true 14 | key: 15 | description: 'content of ssh private key. ex raw content of ~/.ssh/id_rsa' 16 | required: false 17 | pass: 18 | description: 'ssh remote password' 19 | required: false 20 | connect_timeout: 21 | description: 'connection timeout to remote host' 22 | default: "30s" 23 | required: false 24 | first_ssh: 25 | description: 'execute pre-commands before scp' 26 | required: false 27 | scp: 28 | description: 'scp from local to remote' 29 | required: false 30 | last_ssh: 31 | description: 'execute post-commands after scp' 32 | required: false 33 | runs: 34 | using: 'docker' 35 | image: 'Dockerfile' 36 | branding: 37 | icon: 'terminal' 38 | color: 'gray-dark' -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from os import environ, path 2 | from glob import glob 3 | 4 | import paramiko 5 | import scp 6 | import sys 7 | import math 8 | import re 9 | import tempfile 10 | import os 11 | 12 | 13 | envs = environ 14 | INPUT_HOST = envs.get("INPUT_HOST") 15 | INPUT_PORT = int(envs.get("INPUT_PORT", "22")) 16 | INPUT_USER = envs.get("INPUT_USER") 17 | INPUT_PASS = envs.get("INPUT_PASS") 18 | INPUT_KEY = envs.get("INPUT_KEY") 19 | INPUT_CONNECT_TIMEOUT = envs.get("INPUT_CONNECT_TIMEOUT", "30s") 20 | INPUT_SCP = envs.get("INPUT_SCP") 21 | INPUT_FIRST_SSH = envs.get("INPUT_FIRST_SSH") 22 | INPUT_LAST_SSH = envs.get("INPUT_LAST_SSH") 23 | 24 | 25 | seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "M": 86400*30} 26 | pattern_seconds_per_unit = re.compile(r'^(' + "|".join(['\\d+'+k for k in seconds_per_unit.keys()]) + ')$') 27 | 28 | 29 | def convert_to_seconds(s): 30 | if s is None: 31 | return 30 32 | if isinstance(s, str): 33 | return int(s[:-1]) * seconds_per_unit[s[-1]] if pattern_seconds_per_unit.search(s) else 30 34 | if (isinstance(s, int) or isinstance(s, float)) and not math.isnan(s): 35 | return round(s) 36 | return 30 37 | 38 | 39 | strips = [" ", "\"", " ", "'", " "] 40 | 41 | 42 | def strip_and_parse_envs(p): 43 | if not p: 44 | return None 45 | for c in strips: 46 | p = p.strip(c) 47 | return path.expandvars(p) if p != "." else f"{path.realpath(p)}/*" 48 | 49 | 50 | def connect(callback=None): 51 | tmp = tempfile.NamedTemporaryFile(delete=False) 52 | try: 53 | ssh = paramiko.SSHClient() 54 | p_key = None 55 | if INPUT_KEY: 56 | tmp.write(INPUT_KEY.encode()) 57 | tmp.close() 58 | p_key = paramiko.RSAKey.from_private_key_file(filename=tmp.name) 59 | ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 60 | ssh.connect(INPUT_HOST, port=INPUT_PORT, username=INPUT_USER, 61 | pkey=p_key, password=INPUT_PASS, 62 | timeout=convert_to_seconds(INPUT_CONNECT_TIMEOUT)) 63 | except Exception as err: 64 | print(f"Connect error\n{err}") 65 | sys.exit(1) 66 | 67 | else: 68 | if callback: 69 | callback(ssh) 70 | 71 | finally: 72 | os.unlink(tmp.name) 73 | tmp.close() 74 | 75 | 76 | # Define progress callback that prints the current percentage completed for the file 77 | def progress(filename, size, sent): 78 | sys.stdout.write(f"{filename}... {float(sent)/float(size)*100:.2f}%\n") 79 | 80 | 81 | def ssh_process(ssh, input_ssh): 82 | commands = [c.strip() for c in input_ssh.splitlines() if c is not None] 83 | command_str = "" 84 | l = len(commands) 85 | for i in range(len(commands)): 86 | c = path.expandvars(commands[i]) 87 | if c == "": 88 | continue 89 | if c.endswith('&&') or c.endswith('||') or c.endswith(';'): 90 | c = c[0:-2] if i == (l-1) else c 91 | else: 92 | c = f"{c} &&" if i < (l-1) else c 93 | command_str = f"{command_str} {c}" 94 | command_str = command_str.strip() 95 | print(command_str) 96 | 97 | stdin, stdout, stderr = ssh.exec_command(command_str) 98 | 99 | ssh_exit_status = stdout.channel.recv_exit_status() 100 | 101 | out = "".join(stdout.readlines()) 102 | out = out.strip() if out is not None else None 103 | if out: 104 | print(f"Success: \n{out}") 105 | 106 | err = "".join(stderr.readlines()) 107 | err = err.strip() if err is not None else None 108 | if err: 109 | print(f"Error: \n{err}") 110 | 111 | if ssh_exit_status != 0: 112 | print(f"ssh exit status: {ssh_exit_status}") 113 | sys.exit(1) 114 | 115 | pass 116 | 117 | 118 | def scp_process(ssh, input_scp): 119 | copy_list = [] 120 | for c in input_scp.splitlines(): 121 | if not c: 122 | continue 123 | l2r = c.split("=>") 124 | if len(l2r) == 2: 125 | local = strip_and_parse_envs(l2r[0]) 126 | remote = strip_and_parse_envs(l2r[1]) 127 | if local and remote: 128 | copy_list.append({"l": local, "r": remote}) 129 | continue 130 | print(f"SCP ignored {c.strip()}") 131 | print(copy_list) 132 | 133 | if len(copy_list) <= 0: 134 | print("SCP no copy list found") 135 | return 136 | 137 | with scp.SCPClient(ssh.get_transport(), progress=progress, sanitize=lambda x: x) as conn: 138 | for l2r in copy_list: 139 | remote = l2r.get('r') 140 | try: 141 | ssh.exec_command(f"mkdir -p {remote}") 142 | except Exception as err: 143 | print(f"Remote mkdir error. Can't create {remote}\n{err}") 144 | sys.exit(1) 145 | 146 | for f in [f for f in glob(l2r.get('l'))]: 147 | try: 148 | conn.put(f, remote_path=remote, recursive=True) 149 | print(f"{f} -> {remote}") 150 | except Exception as err: 151 | print(f"Scp error. Can't copy {f} on {remote}\n{err}") 152 | sys.exit(1) 153 | pass 154 | 155 | 156 | def processes(): 157 | if INPUT_KEY is None and INPUT_PASS is None: 158 | print("SSH-SCP-SSH invalid (Key/Passwd)") 159 | return 160 | 161 | if not INPUT_FIRST_SSH: 162 | print("SSH-SCP-SSH no first_ssh input found") 163 | else: 164 | print("+++++++++++++++++++Pipeline: RUNNING FIRST SSH+++++++++++++++++++") 165 | connect(lambda c: ssh_process(c, INPUT_FIRST_SSH)) 166 | 167 | if not INPUT_SCP: 168 | print("SSH-SCP-SSH no scp input found") 169 | else: 170 | print("+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++") 171 | connect(lambda c: scp_process(c, INPUT_SCP)) 172 | 173 | if not INPUT_LAST_SSH: 174 | print("SSH-SCP-SSH no last_ssh input found") 175 | else: 176 | print("+++++++++++++++++++Pipeline: RUNNING LAST SSH+++++++++++++++++++") 177 | connect(lambda c: ssh_process(c, INPUT_LAST_SSH)) 178 | 179 | pass 180 | 181 | 182 | if __name__ == '__main__': 183 | processes() 184 | 185 | 186 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++" 4 | 5 | python3 /opt/tools/app.py 6 | RET=$? 7 | 8 | echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++" 9 | 10 | exit $RET 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | paramiko 2 | scp -------------------------------------------------------------------------------- /test/test1.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cross-the-world/ssh-scp-ssh-pipelines/c9b539abea242b337fb703472295d86931942bec/test/test1.csv -------------------------------------------------------------------------------- /test/test1.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cross-the-world/ssh-scp-ssh-pipelines/c9b539abea242b337fb703472295d86931942bec/test/test1.txt -------------------------------------------------------------------------------- /test/test2.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cross-the-world/ssh-scp-ssh-pipelines/c9b539abea242b337fb703472295d86931942bec/test/test2.csv --------------------------------------------------------------------------------