├── .gitignore ├── LICENSE ├── README.md ├── build.sh ├── cmd └── scpm │ └── main.go └── scpm.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Packages # 2 | ############ 3 | # it's better to unpack these files and commit the raw source 4 | # git has its own built in compression methods 5 | *.7z 6 | *.dmg 7 | *.gz 8 | *.iso 9 | *.jar 10 | *.rar 11 | *.tar 12 | *.zip 13 | *.pem 14 | *.deb 15 | *.rpm 16 | 17 | # Logs and databases # 18 | ###################### 19 | *.log 20 | *.sql 21 | *.sqlite 22 | 23 | # OS generated files # 24 | ###################### 25 | .DS_Store 26 | .DS_Store? 27 | ._* 28 | .Spotlight-V100 29 | .Trashes 30 | Icon? 31 | ehthumbs.db 32 | Thumbs.db 33 | *~ 34 | .idea/* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Gronpipmaster 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scpm 2 | 3 | Copy files over ssh protocol to multiple servers, scp multiplexer. 4 | 5 | ## Demo 6 | ``` 7 | scpm --in="~/Video/dog-team09012015.mp4" --path="example1.com:/tmp/a" --path="example2.com:/tmp/b" 8 | Start copy /home/gron/Video/dog-team09012015.mp4 9 | example1.com:22 /tmp/a 5.59 MB / 109.99 MB [=>-----------------------------] 5.09 % 1.12 MB/s 1m33s 10 | example2.com:22 /tmp/b 5.69 MB / 109.99 MB [=>-----------------------------] 5.17 % 1.14 MB/s 1m31s 11 | ``` 12 | 13 | ## Install 14 | ```bash 15 | go get github.com/gophergala/scpm/cmd/scpm 16 | ``` 17 | 18 | ## Usage 19 | ``` 20 | scpm --in="/path/to" \ 21 | --path="user@server0.com:/path/to" \ 22 | --path="user@server1.com:/path/to" \ 23 | --path="user@server2.com:/path/to" \ 24 | --path="user@server3.com:/path/to" \ 25 | --path="user@server4.com:/path/to" 26 | 27 | ``` 28 | 29 | ## Features 30 | 1. Copy once file to once server 31 | 2. Copy once file to multiple servers 32 | 3. Recursive copy folder to multiple server 33 | 4. Multi progress bar 34 | 4. Support custom identity key, system identity, or plain password 35 | 36 | ## TODO 37 | 1. Flag config.json -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | readonly FULL_PATH=`pwd` 3 | readonly PROJECT_NAME=scpm 4 | readonly PROJECT_PATH=github.com/gophergala/scpm/ 5 | readonly BUILD_PATH=$FULL_PATH/tmp 6 | VERSION="0.0" 7 | ARCH="amd64" 8 | 9 | make_package_files() 10 | { 11 | mkdir -p $BUILD_PATH/usr/bin 12 | chmod 755 -R $BUILD_PATH 13 | echo "Download dependens for package project. Please wait ..." 14 | go get -v -t github.com/gophergala/scpm/cmd/scpm 15 | cd $GOPATH/src/$PROJECT_PATH 16 | VERSION=$(git describe --abbrev=5 --tags) 17 | go build -o $BUILD_PATH/usr/bin/$PROJECT_NAME -ldflags "-X main.version $VERSION" github.com/gophergala/scpm/cmd/scpm || exit 1 18 | echo "Done." 19 | } 20 | 21 | make_control_file() 22 | { 23 | echo "Make Debian control file ..." 24 | mkdir -p $BUILD_PATH/DEBIAN 25 | #calculate installed size 26 | full_size=`du -s $BUILD_PATH/usr | awk '{print $1}'` 27 | 28 | filename=$BUILD_PATH/DEBIAN/control 29 | echo "Package: $PROJECT_NAME" > $filename 30 | echo "Version: $VERSION" >> $filename 31 | echo "Architecture: $ARCH" >> $filename 32 | echo "Section: devel" >> $filename 33 | echo "Priority: extra" >> $filename 34 | echo "Installed-Size: $full_size" >> $filename 35 | echo "Maintainer: gronpipmaster " >> $filename 36 | echo "Description: System info, avg free space and memory." >> $filename 37 | 38 | echo "Done." 39 | } 40 | 41 | make_package() 42 | { 43 | local DEB_NAME=${PROJECT_NAME}_${VERSION}_${ARCH} 44 | echo "Make $DEB_NAME.deb" 45 | fakeroot dpkg-deb -b $BUILD_PATH ${FULL_PATH}/${DEB_NAME}.deb || exit 1 46 | echo "Make $DEB_NAME.rpm" 47 | fakeroot alien --to-rpm --scripts ${FULL_PATH}/${DEB_NAME}.deb || exit 1 48 | } 49 | 50 | clean() 51 | { 52 | rm -r $BUILD_PATH 53 | } 54 | 55 | make_package_files 56 | make_control_file 57 | make_package 58 | clean 59 | 60 | echo "All Done." 61 | exit 0 -------------------------------------------------------------------------------- /cmd/scpm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/codegangsta/cli" 6 | "github.com/gophergala/scpm" 7 | // "log" 8 | "os" 9 | "runtime" 10 | ) 11 | 12 | var ( 13 | version string 14 | globalFlags = []cli.Flag{ 15 | cli.IntFlag{ 16 | Name: "port, p", 17 | Value: 22, 18 | }, 19 | cli.StringFlag{ 20 | Name: "identity, i", 21 | Usage: "ssh identity to use for connecting to the host", 22 | }, 23 | cli.StringFlag{ 24 | Name: "in", 25 | Value: "", 26 | Usage: "/path/to/file or /path/to/folder", 27 | }, 28 | cli.StringSliceFlag{ 29 | Name: "path", 30 | Value: &cli.StringSlice{}, 31 | Usage: "user@example.com:/path/to", 32 | }, 33 | } 34 | ) 35 | 36 | func main() { 37 | runtime.GOMAXPROCS(runtime.NumCPU()) 38 | app := cli.NewApp() 39 | app.Name = "scpm" 40 | // app.EnableBashCompletion = true 41 | app.Author = "gronpipmaster" 42 | app.Email = "gronpipmaster@gmail.com" 43 | app.Version = version 44 | app.Usage = "Copy files over ssh protocol to multiple servers." 45 | app.Flags = globalFlags 46 | app.Action = action 47 | app.Run(os.Args) 48 | } 49 | 50 | func action(ctx *cli.Context) { 51 | hosts := []*scpm.Host{} 52 | for _, host := range ctx.GlobalStringSlice("path") { 53 | h, err := scpm.NewHost(host, ctx.GlobalString("identity"), ctx.GlobalInt("port")) 54 | if err != nil { 55 | fatalln(err) 56 | } 57 | hosts = append(hosts, h) 58 | } 59 | if len(ctx.GlobalString("in")) == 0 { 60 | fatalln("Field --in required.") 61 | } 62 | scp, err := scpm.New( 63 | hosts, 64 | ctx.GlobalString("in"), 65 | ) 66 | if err != nil { 67 | fatalln(err) 68 | } 69 | //Create chanels for wait quit signal 70 | quit := make(chan bool) 71 | //Init and run 72 | go scp.Run(quit) 73 | for { 74 | select { 75 | case <-quit: 76 | os.Exit(0) 77 | } 78 | } 79 | } 80 | 81 | func fatalln(i ...interface{}) { 82 | fmt.Println(i...) 83 | os.Exit(1) 84 | } 85 | -------------------------------------------------------------------------------- /scpm.go: -------------------------------------------------------------------------------- 1 | package scpm 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "github.com/gronpipmaster/pb" 8 | "golang.org/x/crypto/ssh" 9 | "io" 10 | "io/ioutil" 11 | "log" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | var homeFolder = os.Getenv("HOME") 20 | 21 | type Host struct { 22 | User string 23 | Addr string 24 | Output string 25 | Client *ssh.Client 26 | sess *ssh.Session 27 | Identity *ssh.ClientConfig 28 | } 29 | 30 | func NewHost(host string, key string, port int) (h *Host, err error) { 31 | h = new(Host) 32 | if strings.Index(host, "@") == -1 { 33 | h.User = os.Getenv("LOGNAME") 34 | } else { 35 | arrHost := strings.Split(host, "@") 36 | h.User = arrHost[0] 37 | host = strings.Replace(host, h.User+"@", "", -1) 38 | } 39 | if strings.Index(host, ":") == -1 { 40 | err = errors.New("host incorrect") 41 | return 42 | } 43 | arrHost := strings.Split(host, ":") 44 | h.Addr = arrHost[0] + ":" + fmt.Sprint(port) 45 | h.Output = arrHost[1] 46 | keys := []string{ 47 | key, 48 | homeFolder + "/.ssh/id_rsa", 49 | homeFolder + "/.ssh/id_dsa", 50 | homeFolder + "/.ssh/id_ecdsa", 51 | } 52 | h.Identity = &ssh.ClientConfig{User: h.User} 53 | for _, k := range keys { 54 | //create ssh.Config from private keys 55 | f, err := os.Open(k) 56 | if err != nil { 57 | continue 58 | } 59 | data, err := ioutil.ReadAll(f) 60 | if err != nil { 61 | return h, err 62 | } 63 | signer, err := ssh.ParsePrivateKey(data) 64 | if err != nil { 65 | return h, err 66 | } 67 | h.Identity.Auth = append(h.Identity.Auth, ssh.PublicKeys(signer)) 68 | f.Close() 69 | } 70 | return 71 | } 72 | 73 | func (h Host) String() string { 74 | str := fmt.Sprintf("%s ", h.Addr+" "+h.Output) 75 | //TODO fixed str size if > 50 76 | return str 77 | } 78 | 79 | func (h *Host) Auth() error { 80 | var err error 81 | h.Client, err = ssh.Dial("tcp", h.Addr, h.Identity) 82 | if err != nil { 83 | h.Identity.Auth = append(h.Identity.Auth, ssh.Password(dialogPassword(h.Addr))) 84 | h.Client, err = ssh.Dial("tcp", h.Addr, h.Identity) 85 | if err != nil { 86 | return err 87 | } 88 | } 89 | return nil 90 | } 91 | 92 | func (h *Host) Copy(tree *Tree, wg *sync.WaitGroup, bar *pb.ProgressBar) { 93 | defer func() { 94 | bar.Finish() 95 | wg.Done() 96 | }() 97 | for _, file := range tree.Files { 98 | in := file.Dir + string(os.PathSeparator) + file.Info.Name() 99 | out := strings.Replace(in, tree.BaseDir, h.Output, -1) 100 | if err := h.cp(in, out, bar); err != nil && err != io.EOF { 101 | log.Println(err) 102 | } 103 | } 104 | } 105 | 106 | func (h *Host) exec(cmd string) error { 107 | var err error 108 | h.sess, err = h.Client.NewSession() 109 | if err != nil { 110 | return err 111 | } 112 | defer h.sess.Close() 113 | return h.sess.Run(cmd) 114 | } 115 | 116 | const ( 117 | cmdCat string = "cat > %s" 118 | cmdStat string = "stat %s" 119 | cmdMkDir string = "mkdir -p %s" 120 | ) 121 | 122 | //remote cp 123 | func (h *Host) cp(in, out string, bar *pb.ProgressBar) error { 124 | var err error 125 | //create remote dir 126 | dir := filepath.Dir(out) 127 | //check folder exists 128 | if err := h.exec(fmt.Sprintf(cmdStat, dir)); err != nil { 129 | if err := h.mkdir(dir); err != nil { 130 | return err 131 | } 132 | } 133 | //open fd file 134 | f, err := os.Open(in) 135 | if err != nil { 136 | return err 137 | } 138 | defer f.Close() 139 | //open ssh session 140 | h.sess, err = h.Client.NewSession() 141 | if err != nil { 142 | return err 143 | } 144 | defer h.sess.Close() 145 | dest, err := h.sess.StdinPipe() 146 | if err != nil { 147 | return err 148 | } 149 | if err = h.sess.Start(fmt.Sprintf(cmdCat, out)); err != nil { 150 | return err 151 | } 152 | writer := io.MultiWriter(dest, bar) 153 | _, err = io.Copy(writer, f) 154 | return err 155 | } 156 | 157 | //remote mkdir 158 | func (h *Host) mkdir(dir string) error { 159 | return h.exec(fmt.Sprintf(cmdMkDir, dir)) 160 | } 161 | 162 | type Scp struct { 163 | hosts []*Host 164 | tree *Tree 165 | wg *sync.WaitGroup 166 | done chan bool 167 | } 168 | 169 | func New(hosts []*Host, path string) (scp *Scp, err error) { 170 | if strings.Index(path, "~") != -1 { 171 | path = strings.Replace(path, "~", homeFolder, -1) 172 | } 173 | absPath, err := filepath.Abs(path) 174 | if err != nil { 175 | return 176 | } 177 | fmt.Println("Start copy", absPath) 178 | if len(hosts) == 0 { 179 | err = errors.New("hosts is nil") 180 | return 181 | } 182 | scp = new(Scp) 183 | scp.hosts = hosts 184 | scp.tree, err = NewTree(absPath) 185 | if err != nil { 186 | return 187 | } 188 | scp.wg = new(sync.WaitGroup) 189 | scp.done = make(chan bool) 190 | return 191 | } 192 | 193 | func (s *Scp) Run(quit chan bool) { 194 | pool := &pb.Pool{} 195 | for _, host := range s.hosts { 196 | if err := host.Auth(); err != nil { 197 | fmt.Println("Auth err:", err) 198 | continue 199 | } 200 | s.wg.Add(1) 201 | bar := pb.New(int(s.tree.Size)).SetUnits(pb.U_BYTES).SetRefreshRate(time.Millisecond * 10) 202 | bar.ShowSpeed = true 203 | bar.Prefix(host.String()) 204 | pool.Add(bar) 205 | go host.Copy(s.tree, s.wg, bar) 206 | } 207 | pool.Start() 208 | go func() { 209 | s.wg.Wait() 210 | time.Sleep(10 * time.Millisecond) 211 | s.done <- true 212 | }() 213 | for { 214 | select { 215 | case <-s.done: 216 | quit <- true 217 | } 218 | } 219 | } 220 | 221 | type Tree struct { 222 | BaseDir string 223 | Size int64 224 | Files []File 225 | } 226 | 227 | type File struct { 228 | Info os.FileInfo 229 | Dir string 230 | } 231 | 232 | func NewTree(path string) (t *Tree, err error) { 233 | t = new(Tree) 234 | t.BaseDir = filepath.Dir(path) 235 | info, err := os.Stat(path) 236 | if err != nil { 237 | return 238 | } 239 | if info.IsDir() { 240 | err = filepath.Walk(path, t.Scan) 241 | return 242 | } 243 | t.Files = append(t.Files, File{Info: info, Dir: t.BaseDir}) 244 | t.Size = info.Size() 245 | return 246 | } 247 | 248 | func (t *Tree) Scan(path string, fileInfo os.FileInfo, errInp error) (err error) { 249 | if errInp != nil { 250 | log.Println(errInp) 251 | return nil 252 | } 253 | if fileInfo.IsDir() { 254 | return nil 255 | } 256 | t.Files = append(t.Files, File{Info: fileInfo, Dir: filepath.Dir(path)}) 257 | t.Size += fileInfo.Size() 258 | return 259 | } 260 | 261 | func dialogPassword(addr string) (pass string) { 262 | reader := bufio.NewReader(os.Stdin) 263 | fmt.Println(addr) 264 | fmt.Print("Enter Password: ") 265 | pass, _ = reader.ReadString('\n') 266 | return strings.TrimSpace(pass) 267 | } 268 | --------------------------------------------------------------------------------