├── .gitignore ├── README.md ├── file_checker.go ├── file_sftp_handler.go ├── file_watcher.go ├── main.go ├── sftp_util.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .idea/* 3 | Mancy 4 | Mancy.exe 5 | mancy_config.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | #### Mancy is a file watcher and supported to auto upload the changes to remote server via ssh/sftp. 4 | 5 | I start this project for learning golang and solve some problem in our develop environment. 6 | 7 | ## Usage 8 | 1. go get github.com/rootrl/Mancy 9 | 2. go build 10 | 3. run "Mancy" or "Mancy.exe"(windows) 11 | 12 | ## Todo 13 | * The variable "sftpClient" is used as a global variable, that's not cool, Maybe better to define a sftp struct and provides set of methods -------------------------------------------------------------------------------- /file_checker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // Check if the file is contains the ignore file 8 | // Return true if contains 9 | func fileChecker(filename string) bool { 10 | 11 | for _, ignoreFile := range config.IgnoreFiles { 12 | if strings.Contains(filename, string(ignoreFile)) { 13 | return true 14 | } 15 | } 16 | 17 | return false 18 | 19 | } -------------------------------------------------------------------------------- /file_sftp_handler.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | "log" 6 | "sync" 7 | 8 | "github.com/pkg/sftp" 9 | ) 10 | 11 | var ( 12 | wg sync.WaitGroup 13 | err error 14 | sftpClient *sftp.Client 15 | ) 16 | 17 | func fileSftpHandler() { 18 | // Connect 19 | sftpClient, err = connect(config.SshUserName, config.SshPassword, config.SshHost, config.SshPort) 20 | 21 | if err != nil { 22 | log.Fatal("SSH connect error: ", err) 23 | } 24 | 25 | defer sftpClient.Close() 26 | 27 | // Check if the remote dir exist 28 | if _, err := sftpClient.Stat(remoteDir); err != nil { 29 | panic("Remote dir dose not exist: " + remoteDir) 30 | } 31 | 32 | wg.Add(5) 33 | 34 | // Create Event handler 35 | go func() { 36 | for { 37 | select { 38 | 39 | case <-time.After(fileHandleTimeOut): 40 | //timeout for 3 seconds 41 | 42 | case createFileName := <-fileCreateEvent: 43 | log.Print("createFile:" + createFileName) 44 | 45 | if isDir(createFileName) == 1 { 46 | uploadDirectory(createFileName, remoteDir) 47 | } else { 48 | uploadFile(createFileName, remoteDir) 49 | } 50 | } 51 | } 52 | 53 | defer wg.Done() 54 | }() 55 | 56 | // Write event handler 57 | go func() { 58 | for { 59 | select { 60 | 61 | case <-time.After(fileHandleTimeOut): 62 | //timeout for 3 seconds 63 | 64 | case writeFileName := <-fileWriteEvent: 65 | log.Print("write file: " + writeFileName) 66 | 67 | isDir := isDir(writeFileName) 68 | 69 | if isDir == 1 { 70 | uploadDirectory(writeFileName, remoteDir) 71 | } else if isDir == 0 { 72 | uploadFile(writeFileName, remoteDir) 73 | } 74 | } 75 | } 76 | 77 | defer wg.Done() 78 | }() 79 | 80 | // Remove event handler 81 | go func() { 82 | for { 83 | select { 84 | 85 | case <-time.After(fileHandleTimeOut): 86 | //timeout for 3 seconds 87 | 88 | case removeFileName := <-fileRemoveEvent: 89 | log.Print("remove file: " + removeFileName) 90 | 91 | remove(removeFileName, remoteDir) 92 | } 93 | } 94 | 95 | defer wg.Done() 96 | }() 97 | 98 | // Rename event handler 99 | go func() { 100 | for { 101 | select { 102 | 103 | case <-time.After(fileHandleTimeOut): 104 | //timeout for 3 seconds 105 | 106 | case renameFileName := <-fileRenameEvent: 107 | log.Print("rename file: " + renameFileName) 108 | 109 | // Rename same as remove 110 | remove(renameFileName, remoteDir) 111 | } 112 | } 113 | 114 | defer wg.Done() 115 | }() 116 | 117 | // Chmod event handler 118 | go func() { 119 | for { 120 | select { 121 | 122 | case <-time.After(fileHandleTimeOut): 123 | //timeout for 3 seconds 124 | 125 | case chmodFileName := <-fileChmodEvent: 126 | log.Print("chmod file" + chmodFileName) 127 | } 128 | } 129 | 130 | defer wg.Done() 131 | }() 132 | 133 | wg.Wait() 134 | } 135 | -------------------------------------------------------------------------------- /file_watcher.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | "github.com/fsnotify/fsnotify" 8 | ) 9 | 10 | type Watch struct { 11 | watch *fsnotify.Watcher; 12 | } 13 | 14 | // handler jobs done 15 | var eventDone = make(chan bool) 16 | 17 | // Watch a directory 18 | func (w *Watch) watchDir(dir string) { 19 | 20 | // Walk all directory 21 | filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 22 | 23 | // Just watch directory(all child can be watched) 24 | if info.IsDir() { 25 | path, err := filepath.Abs(path) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | err = w.watch.Add(path) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | } 34 | 35 | return nil 36 | }) 37 | 38 | log.Print("Watching: ", dir) 39 | 40 | // Handle the watch events 41 | go eventsHandler(w) 42 | 43 | // Await 44 | <-eventDone 45 | } 46 | 47 | // Handle the watch events 48 | func eventsHandler(w *Watch) { 49 | for { 50 | select { 51 | case ev := <-w.watch.Events: 52 | { 53 | // Create event 54 | if ev.Op&fsnotify.Create == fsnotify.Create { 55 | fi, err := os.Stat(ev.Name) 56 | 57 | if !fileChecker(ev.Name) { 58 | fileCreateEvent <- ev.Name 59 | } 60 | 61 | if err == nil && fi.IsDir() { 62 | w.watch.Add(ev.Name) 63 | } 64 | } 65 | 66 | // write event 67 | if ev.Op&fsnotify.Write == fsnotify.Write { 68 | if !fileChecker(ev.Name) { 69 | fileWriteEvent <- ev.Name 70 | } 71 | } 72 | 73 | // delete event 74 | if ev.Op&fsnotify.Remove == fsnotify.Remove { 75 | 76 | fi, err := os.Stat(ev.Name) 77 | 78 | if err == nil && fi.IsDir() { 79 | w.watch.Remove(ev.Name) 80 | } 81 | 82 | if !fileChecker(ev.Name) { 83 | fileRemoveEvent <- ev.Name 84 | } 85 | } 86 | 87 | // Rename 88 | if ev.Op&fsnotify.Rename == fsnotify.Rename { 89 | w.watch.Remove(ev.Name) 90 | 91 | if !fileChecker(ev.Name) { 92 | fileRenameEvent <- ev.Name 93 | } 94 | } 95 | // Chmod 96 | if ev.Op&fsnotify.Chmod == fsnotify.Chmod { 97 | if !fileChecker(ev.Name) { 98 | fileChmodEvent <- ev.Name 99 | } 100 | } 101 | } 102 | case err := <-w.watch.Errors: 103 | { 104 | log.Fatal(err) 105 | eventDone <- true 106 | return 107 | } 108 | } 109 | } 110 | 111 | eventDone <- true 112 | } 113 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/fsnotify/fsnotify" 5 | "log" 6 | "time" 7 | "path/filepath" 8 | ) 9 | 10 | // Package const 11 | // Config file path name 12 | const ( 13 | ConfigFilePathName = "./mancy_config.json" 14 | ) 15 | 16 | // Package variables 17 | var ( 18 | // Local dir name witch will be watched 19 | localDir string 20 | 21 | // Local dir name witch will be watched 22 | remoteDir string 23 | 24 | // Config data struct 25 | config Config 26 | 27 | // Global chan variables 28 | // file_watcher will write the chan and file_handle will read the chan 29 | // create file 30 | fileCreateEvent = make(chan string) 31 | 32 | // write 33 | fileWriteEvent = make(chan string) 34 | 35 | // remove 36 | fileRemoveEvent = make(chan string) 37 | 38 | // rename 39 | fileRenameEvent = make(chan string) 40 | 41 | // chmod 42 | fileChmodEvent = make(chan string) 43 | 44 | // watchMainJob chan 45 | watcherHandlerDone = make(chan bool) 46 | 47 | // fileHandleMainJob chan 48 | fileHandlerDone = make(chan bool) 49 | 50 | // timeout for watcher event 51 | fileHandleTimeOut = time.Second * 4 52 | ) 53 | 54 | // Config file struct 55 | type Config struct{ 56 | // filePath 57 | LocalDir string `localDir` 58 | RemoteDir string `remoteDir` 59 | 60 | // ssh 61 | SshHost string `sshHost` 62 | SshPort int `sshPort` 63 | SshUserName string `sshUserName` 64 | SshPassword string `sshPassword` 65 | 66 | // ignoreFiles 67 | IgnoreFiles []string `ignoreFiles` 68 | } 69 | 70 | // Init 71 | func init() { 72 | // Reset log format 73 | log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) 74 | 75 | // Check and generate config file 76 | checkConfigFile() 77 | 78 | // Parse config file 79 | parseConfigFile() 80 | 81 | // Set localDir as ABS path 82 | localDir, err = filepath.Abs(config.LocalDir) 83 | 84 | // Set remote dir 85 | remoteDir = config.RemoteDir 86 | 87 | if err != nil { 88 | log.Fatal("Init localFilePath error: ") 89 | panic(err) 90 | } 91 | } 92 | 93 | func main() { 94 | 95 | watch, _ := fsnotify.NewWatcher() 96 | 97 | w := Watch{ 98 | watch: watch, 99 | } 100 | 101 | // Watch the local directory 102 | go func() { 103 | w.watchDir(localDir) 104 | watcherHandlerDone <- true 105 | }() 106 | 107 | // handle the file events 108 | go func() { 109 | // Handle file with sftp (autoUpload changes) 110 | // And you can change the handler whatever you need like rsync 111 | fileSftpHandler() 112 | 113 | fileHandlerDone <- true 114 | }() 115 | 116 | // Waiting job done 117 | <-watcherHandlerDone 118 | <-fileHandlerDone 119 | } 120 | -------------------------------------------------------------------------------- /sftp_util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "log" 6 | "path" 7 | "io/ioutil" 8 | "fmt" 9 | "time" 10 | "net" 11 | 12 | "github.com/pkg/sftp" 13 | "golang.org/x/crypto/ssh" 14 | ) 15 | 16 | // Connect 17 | func connect(user, password, host string, port int) (*sftp.Client, error) { 18 | 19 | var ( 20 | auth []ssh.AuthMethod 21 | addr string 22 | clientConfig *ssh.ClientConfig 23 | sshClient *ssh.Client 24 | sftpClient *sftp.Client 25 | 26 | err error 27 | ) 28 | 29 | auth = make([]ssh.AuthMethod, 0) 30 | auth = append(auth, ssh.Password(password)) 31 | 32 | clientConfig = &ssh.ClientConfig{ 33 | User: user, 34 | Auth: auth, 35 | Timeout: 30 * time.Second, 36 | HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { 37 | return nil 38 | }, 39 | } 40 | 41 | addr = fmt.Sprintf("%s:%d", host, port) 42 | 43 | if sshClient, err = ssh.Dial("tcp", addr, clientConfig); err != nil { 44 | return nil, err 45 | } 46 | 47 | if sftpClient, err = sftp.NewClient(sshClient); err != nil { 48 | return nil, err 49 | } 50 | 51 | return sftpClient, nil 52 | 53 | } 54 | 55 | // UploadFile 56 | func uploadFile(localFilePath, remoteDir string) { 57 | 58 | srcFile, err := os.Open(localFilePath) 59 | 60 | if err != nil { 61 | log.Fatal(err) 62 | } 63 | 64 | defer srcFile.Close() 65 | 66 | var remoteFileName = getChangedFileName(localFilePath) 67 | 68 | dstFile, err := sftpClient.Create(path.Join(remoteDir, remoteFileName)) 69 | 70 | if err != nil { 71 | log.Fatal("dst error: ", err, path.Join(remoteDir, remoteFileName)) 72 | } 73 | 74 | defer dstFile.Close() 75 | 76 | buf, err := ioutil.ReadAll(srcFile) 77 | 78 | if err != nil { 79 | panic(err) 80 | } 81 | 82 | dstFile.Write(buf) 83 | 84 | log.Print("uploaded file: ", localFilePath) 85 | } 86 | 87 | // UploadDirectory 88 | func uploadDirectory(localPath, remotePath string) { 89 | localFiles, err := ioutil.ReadDir(localPath) 90 | 91 | if err != nil { 92 | panic(err) 93 | } 94 | 95 | changedFileName := getChangedFileName(localPath) 96 | 97 | sftpClient.Mkdir(path.Join(remotePath, changedFileName)) 98 | 99 | log.Print("make remote dir: ", path.Join(remotePath, getChangedFileName(localPath))) 100 | 101 | for _, backupDir := range localFiles { 102 | localFilePath := path.Join(localPath, backupDir.Name()) 103 | remoteFilePath := path.Join(remotePath, changedFileName, backupDir.Name()) 104 | 105 | if backupDir.IsDir() { 106 | sftpClient.Mkdir(remoteFilePath) 107 | uploadDirectory(localFilePath, remotePath) 108 | } else { 109 | uploadFile(path.Join(localPath, backupDir.Name()), remotePath) 110 | } 111 | } 112 | } 113 | 114 | // Remove whatever a file or directory 115 | func remove(filename, remoteDir string) { 116 | remoteFileName := path.Join(remoteDir, getChangedFileName(filename)) 117 | fileInfo, err := sftpClient.Stat(remoteFileName) 118 | 119 | if err != nil { 120 | log.Println(err) 121 | return 122 | } 123 | 124 | if fileInfo.IsDir() { 125 | removeDirectory(remoteFileName, remoteDir) 126 | } else { 127 | removeFile(remoteFileName) 128 | } 129 | } 130 | 131 | // Remove file 132 | func removeFile(filename string) { 133 | 134 | err := sftpClient.Remove(filename) 135 | 136 | if err != nil { 137 | log.Fatal("Can' remove file :", filename) 138 | } else { 139 | log.Print("Removed file: ", filename) 140 | } 141 | } 142 | 143 | // remove Directory 144 | func removeDirectory(dir , remoteDir string) { 145 | fileInfos, err := sftpClient.ReadDir(dir) 146 | 147 | if err != nil { 148 | log.Fatal("Readdir err: ", err) 149 | } 150 | 151 | for _, fileInfo := range fileInfos { 152 | if fileInfo.IsDir() { 153 | fmt.Println("remove directory: ", path.Join(dir, fileInfo.Name())) 154 | removeDirectory(path.Join(dir, fileInfo.Name()), remoteDir) 155 | } else { 156 | fmt.Println(path.Join(remoteDir, getChangedFileName(dir))) 157 | removeFile(path.Join(remoteDir, getChangedFileName(dir), fileInfo.Name())) 158 | } 159 | } 160 | 161 | sftpClient.RemoveDirectory(dir) 162 | } -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "path/filepath" 7 | "log" 8 | "io/ioutil" 9 | "encoding/json" 10 | "fmt" 11 | ) 12 | 13 | // Check the filePath if is a dir 14 | func isDir(fileName string) int { 15 | fileInfo, err := os.Stat(fileName) 16 | 17 | if err != nil { 18 | log.Println("file dose not exist: " + fileName) 19 | return -1 20 | } 21 | 22 | if fileInfo.IsDir() { 23 | return 1 24 | } else { 25 | return 0 26 | } 27 | } 28 | 29 | // Get the change part of filePath 30 | func getChangedFileName(fileName string) string { 31 | fileName = strings.Replace(fileName, localDir, "", -1) 32 | return filepath.ToSlash(fileName) 33 | } 34 | 35 | // Check and generate config file 36 | func checkConfigFile() { 37 | 38 | // Check 39 | if _, err := os.Stat(ConfigFilePathName); !os.IsNotExist(err) { 40 | return 41 | } 42 | 43 | // Generate if config not exist 44 | 45 | // config template 46 | var tpl string = `{ 47 | "localDir": "./", 48 | "remoteDir": "Remote dir (absolute path like: /root/a)", 49 | 50 | "sshHost": "Your host", 51 | "sshPort": 22, 52 | "sshUserName": "Your user name", 53 | "sshPassword": "Your password", 54 | 55 | "ignoreFiles": [ 56 | ".git", 57 | ".idea", 58 | ".swp", 59 | ".swx", 60 | "___jb_old___", 61 | "___jb_tmp___" 62 | ] 63 | }` 64 | 65 | if err := ioutil.WriteFile(ConfigFilePathName, []byte(tpl), 0755); err != nil { 66 | panic(err) 67 | } 68 | 69 | fmt.Println("\n", `Your are first time run this script, The config file 'config,json' not found, And already generated for you.`, 70 | `Please reset the config item as your own infomation suck as ssh host & account, then run script again.`, "\n") 71 | os.Exit(0) 72 | } 73 | 74 | // Parse config file 75 | func parseConfigFile() { 76 | 77 | configJson, err := ioutil.ReadFile(ConfigFilePathName) 78 | 79 | if err != nil { 80 | panic(err) 81 | } 82 | 83 | err = json.Unmarshal(configJson, &config) 84 | 85 | if err != nil { 86 | panic(err) 87 | } 88 | } 89 | --------------------------------------------------------------------------------