├── data ├── wav │ └── xxx.wav ├── fixtures │ ├── global │ │ ├── file.txt │ │ ├── dir │ │ │ ├── dirfile.txt │ │ │ └── subdir │ │ │ │ └── subdirfile.log │ │ ├── WalkPathByPattern │ │ │ ├── file.txt │ │ │ ├── test.part1.rar │ │ │ ├── test.part2.rar │ │ │ ├── test.part3.rar │ │ │ ├── dir │ │ │ │ ├── dirfile.txt │ │ │ │ └── subdir │ │ │ │ │ └── subdirfile.log │ │ │ ├── textfile.txt │ │ │ └── documents (2010) │ │ │ │ └── document (2010).txt │ │ ├── AreFilesEqual │ │ │ ├── not-equal.txt │ │ │ ├── equal1.txt │ │ │ └── equal2.txt │ │ ├── textfile.txt │ │ ├── CopyResumed │ │ │ ├── test2-src.txt │ │ │ ├── test1-src.txt │ │ │ ├── test4-src.txt │ │ │ ├── test3-dst-partial.txt │ │ │ ├── test4-dst-exists.txt │ │ │ ├── test3-src.txt │ │ │ └── test2-dst-larger.txt │ │ ├── documents (2010) │ │ │ └── document (2010).txt │ │ └── ReadAllLines │ │ │ ├── 10-with-empty.txt │ │ │ └── 10-lines.txt │ └── file │ │ ├── WalkPathByPattern │ │ ├── file.txt │ │ ├── dir │ │ │ ├── dirfile.txt │ │ │ └── subdir │ │ │ │ └── subdirfile.log │ │ ├── test.part1.rar │ │ ├── test.part2.rar │ │ ├── test.part3.rar │ │ ├── textfile.txt │ │ └── documents (2010) │ │ │ └── document (2010).txt │ │ ├── AreFilesEqual │ │ ├── not-equal.txt │ │ ├── equal1.txt │ │ └── equal2.txt │ │ ├── CopyResumed │ │ ├── test2-src.txt │ │ ├── test1-src.txt │ │ ├── test4-src.txt │ │ ├── test3-dst-partial.txt │ │ ├── test4-dst-exists.txt │ │ ├── test3-src.txt │ │ └── test2-dst-larger.txt │ │ └── ReadAllLines │ │ ├── 10-with-empty.txt │ │ └── 10-lines.txt └── doc │ └── DEVELOPER-NOTES.md ├── _config.yml ├── .idea ├── misc.xml ├── encodings.xml ├── vcs.xml ├── preferred-vcs.xml └── modules.xml ├── designpattern └── observer │ ├── interfaces.go │ ├── observable.go │ └── observable_test.go ├── matcher ├── interfaces.go ├── regex.go ├── regex_test.go ├── composite.go ├── composite_test.go ├── filetype_test.go ├── filesize.go ├── fileage.go ├── filetype.go ├── fileage_test.go └── filesize_test.go ├── action ├── interfaces.go ├── find.go ├── abstract_transfer.go ├── move.go ├── copy.go ├── delete.go ├── serve.go ├── receive.go └── abstract.go ├── filesystem ├── path.go ├── filesystem.go ├── sftpfs_context.go ├── path_windows.go ├── osfs.go ├── sftpfs_file.go ├── osfs_windows.go └── sftpfs.go ├── unrelease ├── sftpd ├── file_tree_test.go ├── sftp_fs_context.go ├── file_tree.go ├── vfshandler.go ├── path_mapper.go ├── sftpd.go └── path_mapper_test.go ├── apputils ├── net.go └── net_test.go ├── .gitignore ├── testhelpers └── testhelpers.go ├── file ├── tree_test.go ├── path_test.go ├── locator_cache.go ├── tree.go ├── walk_observer.go ├── walk_observer_test.go ├── path.go ├── locator_cache_test.go ├── locator.go ├── compare │ ├── stitch.go │ └── stitch_test.go └── locator_test.go ├── graft.iml ├── release ├── pattern ├── destination_test.go ├── destination.go ├── base.go ├── source.go ├── functions.go ├── functions_test.go ├── source_test.go └── base_test.go ├── bitflag ├── bitflag_parser.go └── bitflag_parser_test.go ├── transfer ├── messageprinter_observer.go ├── messageprinter_observer_test.go ├── progress_handler_test.go ├── progress_handler.go ├── strategy.go └── strategy_test.go ├── .goreleaser.yml ├── LICENSE ├── sftpfs ├── file.go └── sftp.go ├── graft.go └── README.md /data/wav/xxx.wav: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/file.txt: -------------------------------------------------------------------------------- 1 | file -------------------------------------------------------------------------------- /data/fixtures/global/dir/dirfile.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/file.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/file.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/dir/subdir/subdirfile.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/dir/dirfile.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/test.part1.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/test.part2.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/test.part3.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/test.part1.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/test.part2.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/test.part3.rar: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/file/AreFilesEqual/not-equal.txt: -------------------------------------------------------------------------------- 1 | not equal -------------------------------------------------------------------------------- /data/fixtures/global/AreFilesEqual/not-equal.txt: -------------------------------------------------------------------------------- 1 | not equal -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/dir/dirfile.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/textfile.txt: -------------------------------------------------------------------------------- 1 | das ist ein test textfile -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/dir/subdir/subdirfile.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/dir/subdir/subdirfile.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-architect 2 | show_downloads: true 3 | -------------------------------------------------------------------------------- /data/fixtures/file/AreFilesEqual/equal1.txt: -------------------------------------------------------------------------------- 1 | abcd1234 2 | abcd1234 3 | -------------------------------------------------------------------------------- /data/fixtures/file/AreFilesEqual/equal2.txt: -------------------------------------------------------------------------------- 1 | abcd1234 2 | abcd1234 3 | -------------------------------------------------------------------------------- /data/fixtures/global/AreFilesEqual/equal1.txt: -------------------------------------------------------------------------------- 1 | abcd1234 2 | abcd1234 3 | -------------------------------------------------------------------------------- /data/fixtures/global/AreFilesEqual/equal2.txt: -------------------------------------------------------------------------------- 1 | abcd1234 2 | abcd1234 3 | -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/textfile.txt: -------------------------------------------------------------------------------- 1 | das ist ein test textfile -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/textfile.txt: -------------------------------------------------------------------------------- 1 | das ist ein test textfile -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test2-src.txt: -------------------------------------------------------------------------------- 1 | this is a small src with larger dst -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test2-src.txt: -------------------------------------------------------------------------------- 1 | this is a small src with larger dst -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test1-src.txt: -------------------------------------------------------------------------------- 1 | this is a file without existing destination -------------------------------------------------------------------------------- /data/fixtures/global/documents (2010)/document (2010).txt: -------------------------------------------------------------------------------- 1 | das ist ein test für document -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test4-src.txt: -------------------------------------------------------------------------------- 1 | this is a file where src and dst are fully equal -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test1-src.txt: -------------------------------------------------------------------------------- 1 | this is a file without existing destination -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test4-src.txt: -------------------------------------------------------------------------------- 1 | this is a file where src and dst are fully equal -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test3-dst-partial.txt: -------------------------------------------------------------------------------- 1 | this is the full content of a file with a partial -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test4-dst-exists.txt: -------------------------------------------------------------------------------- 1 | this is a file where src and dst are fully equal -------------------------------------------------------------------------------- /data/fixtures/file/WalkPathByPattern/documents (2010)/document (2010).txt: -------------------------------------------------------------------------------- 1 | das ist ein test für document -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test3-dst-partial.txt: -------------------------------------------------------------------------------- 1 | this is the full content of a file with a partial -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test4-dst-exists.txt: -------------------------------------------------------------------------------- 1 | this is a file where src and dst are fully equal -------------------------------------------------------------------------------- /data/fixtures/global/WalkPathByPattern/documents (2010)/document (2010).txt: -------------------------------------------------------------------------------- 1 | das ist ein test für document -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test3-src.txt: -------------------------------------------------------------------------------- 1 | this is the full content of a file with a partial existing destination -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test3-src.txt: -------------------------------------------------------------------------------- 1 | this is the full content of a file with a partial existing destination -------------------------------------------------------------------------------- /data/fixtures/file/CopyResumed/test2-dst-larger.txt: -------------------------------------------------------------------------------- 1 | this is a dst that is larger than its source and therefore cannot be copied -------------------------------------------------------------------------------- /data/fixtures/global/CopyResumed/test2-dst-larger.txt: -------------------------------------------------------------------------------- 1 | this is a dst that is larger than its source and therefore cannot be copied -------------------------------------------------------------------------------- /designpattern/observer/interfaces.go: -------------------------------------------------------------------------------- 1 | package designpattern 2 | 3 | type ObserverInterface interface { 4 | Notify(args...interface{}) 5 | } -------------------------------------------------------------------------------- /data/fixtures/file/ReadAllLines/10-with-empty.txt: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | line 4 5 | line 5 6 | 7 | line 7 8 | 9 | 10 | line 10 -------------------------------------------------------------------------------- /data/fixtures/global/ReadAllLines/10-with-empty.txt: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | line 4 5 | line 5 6 | 7 | line 7 8 | 9 | 10 | line 10 -------------------------------------------------------------------------------- /data/fixtures/file/ReadAllLines/10-lines.txt: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | line 4 5 | line 5 6 | line 6 7 | line 7 8 | line 8 9 | line 9 10 | line 10 -------------------------------------------------------------------------------- /data/fixtures/global/ReadAllLines/10-lines.txt: -------------------------------------------------------------------------------- 1 | line 1 2 | line 2 3 | line 3 4 | line 4 5 | line 5 6 | line 6 7 | line 7 8 | line 8 9 | line 9 10 | line 10 -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/preferred-vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ApexVCS 5 | 6 | -------------------------------------------------------------------------------- /matcher/interfaces.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | type CompositeInterface interface{ 4 | Add(child CompositeInterface) 5 | } 6 | 7 | type MatcherInterface interface { 8 | Matches(pattern interface{}) bool 9 | } 10 | -------------------------------------------------------------------------------- /action/interfaces.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import "github.com/urfave/cli" 4 | 5 | type ConnectionInterface interface { 6 | Connect(url string) 7 | Disconnect() error 8 | } 9 | 10 | 11 | type CliActionInterface interface { 12 | Execute(c *cli.Context) error 13 | } -------------------------------------------------------------------------------- /filesystem/path.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package filesystem 4 | 5 | import ( 6 | "path/filepath" 7 | 8 | "github.com/spf13/afero" 9 | ) 10 | 11 | func Walk(fs afero.Fs, root string, walkFn filepath.WalkFunc) error { 12 | return afero.Walk(fs, root, walkFn) 13 | } 14 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /unrelease: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | RELEASE_TAG="$1" 4 | 5 | if [ "$RELEASE_TAG" == "" ]; then 6 | echo "release version is missing!" 7 | echo "Usage:" 8 | echo " ./unrelease v0.2.0" 9 | exit 1 10 | fi 11 | 12 | 13 | git tag -d "$RELEASE_TAG" 14 | git push origin ":refs/tags/$RELEASE_TAG" -------------------------------------------------------------------------------- /filesystem/filesystem.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "github.com/spf13/afero" 7 | ) 8 | 9 | func CleanPath(fs afero.Fs, path string) string { 10 | if fs.Name() == NameSftpfs { 11 | return filepath.ToSlash(filepath.Clean(path)) 12 | } 13 | 14 | return filepath.Clean(path) 15 | } 16 | -------------------------------------------------------------------------------- /sftpd/file_tree_test.go: -------------------------------------------------------------------------------- 1 | package sftpd_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/sftpd" 7 | ) 8 | 9 | func TestFileTreeIntegrate(t *testing.T) { 10 | expect := assert.New(t) 11 | fileTree := sftpd.NewFileTree("data/fixtures") 12 | expect.NotNil(fileTree) 13 | 14 | } -------------------------------------------------------------------------------- /matcher/regex.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | type RegexMatcher struct { 8 | MatcherInterface 9 | regex *regexp.Regexp 10 | } 11 | 12 | func NewRegexMatcher(regex *regexp.Regexp) *RegexMatcher { 13 | return &RegexMatcher{ 14 | regex: regex, 15 | } 16 | } 17 | 18 | func (f *RegexMatcher) Matches(subject interface{}) bool { 19 | return f.regex.MatchString(subject.(string)) 20 | } -------------------------------------------------------------------------------- /apputils/net.go: -------------------------------------------------------------------------------- 1 | package apputils 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | func GetOutboundIpAsString(fallbackValue string , dialCallback func (network, address string) (net.Conn, error)) (string, error) { 8 | conn, err := dialCallback("udp", "8.8.8.8:80") 9 | if err != nil { 10 | return fallbackValue, err 11 | } 12 | defer conn.Close() 13 | localAddr := conn.LocalAddr().(*net.UDPAddr) 14 | return localAddr.IP.String(), nil 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | /.idea 27 | /.ghtoken 28 | /dist 29 | /_old 30 | /data/tmp 31 | !/data/tmp/.gitkeep 32 | -------------------------------------------------------------------------------- /matcher/regex_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "regexp" 7 | "github.com/sandreas/graft/matcher" 8 | ) 9 | 10 | func TestRegexMatcher(t *testing.T) { 11 | expect := assert.New(t) 12 | 13 | compiledRegex, _ := regexp.Compile("^a(.*)$") 14 | 15 | subject := matcher.NewRegexMatcher(compiledRegex) 16 | 17 | expect.True(subject.Matches("abcd")) 18 | expect.False(subject.Matches("other value")) 19 | } -------------------------------------------------------------------------------- /testhelpers/testhelpers.go: -------------------------------------------------------------------------------- 1 | package testhelpers 2 | 3 | import ( 4 | "github.com/spf13/afero" 5 | "strings" 6 | ) 7 | 8 | func MockFileSystem(initialFiles map[string]string) afero.Fs { 9 | mockFs := afero.NewMemMapFs() 10 | 11 | for key, value := range initialFiles { 12 | if strings.HasSuffix(key, "/") || strings.HasSuffix(key, "\\") { 13 | mockFs.MkdirAll(key, 0644) 14 | } else { 15 | afero.WriteFile(mockFs,key, []byte(value), 0755) 16 | } 17 | } 18 | return mockFs 19 | } -------------------------------------------------------------------------------- /file/tree_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/sandreas/graft/file" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFileTreeIntegrateAndList(t *testing.T) { 11 | expect := assert.New(t) 12 | 13 | fileTree := file.NewTreeNode("fixtures") 14 | expect.NoError(fileTree.Integrate("fixtures/subdir/file.txt")) 15 | expect.NoError(fileTree.Integrate("fixtures/other-subdir/file.txt")) 16 | list := fileTree.List("fixtures") 17 | expect.Len(list, 2) 18 | } 19 | -------------------------------------------------------------------------------- /graft.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /matcher/composite.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | type CompositeMatcher struct { 4 | CompositeInterface 5 | MatcherInterface 6 | children []MatcherInterface 7 | } 8 | 9 | func NewCompositeMatcher() *CompositeMatcher { 10 | return &CompositeMatcher{} 11 | } 12 | 13 | func (f *CompositeMatcher) Add(child MatcherInterface) { 14 | f.children = append(f.children, child) 15 | } 16 | 17 | func (f *CompositeMatcher) Matches(subject interface{}) bool { 18 | for _,val := range f.children { 19 | if ! val.Matches(subject) { 20 | return false 21 | } 22 | } 23 | return true 24 | } -------------------------------------------------------------------------------- /designpattern/observer/observable.go: -------------------------------------------------------------------------------- 1 | package designpattern 2 | 3 | 4 | type ObservableInterface interface { 5 | RegisterObserver(observerInterface ObserverInterface) 6 | notifyObservers(args...interface{}) 7 | } 8 | 9 | 10 | type Observable struct { 11 | observers []ObserverInterface 12 | ObservableInterface 13 | } 14 | 15 | func (p *Observable) RegisterObserver(observer ObserverInterface) { 16 | p.observers = append(p.observers, observer) 17 | } 18 | 19 | func (p *Observable) NotifyObservers(args...interface{}) { 20 | for _, o := range p.observers { 21 | o.Notify(args...) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /apputils/net_test.go: -------------------------------------------------------------------------------- 1 | package apputils_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "net" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/sandreas/graft/apputils" 9 | ) 10 | 11 | func mockErrorCallback(protocol, address string) (net.Conn, error) { 12 | return nil, errors.New("Mock error: " + protocol + ", " + address) 13 | } 14 | 15 | func TestGetOutboundIpAsStringFallback(t *testing.T) { 16 | expect := assert.New(t) 17 | 18 | fallback := "local" 19 | got, err := apputils.GetOutboundIpAsString(fallback, mockErrorCallback) 20 | expect.Equal(fallback, got) 21 | expect.Error(err) 22 | } 23 | 24 | -------------------------------------------------------------------------------- /matcher/composite_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/matcher" 7 | ) 8 | 9 | type TestMatcher struct { 10 | matcher.MatcherInterface 11 | } 12 | func (f *TestMatcher) Matches(subject interface{}) bool { 13 | return subject == "test" 14 | } 15 | 16 | 17 | func TestCompositeMatcher(t *testing.T) { 18 | expect := assert.New(t) 19 | testMatcher := &TestMatcher{} 20 | 21 | subject := matcher.NewCompositeMatcher() 22 | subject.Add(testMatcher) 23 | 24 | expect.True(subject.Matches("test")) 25 | expect.False(subject.Matches("other value")) 26 | } 27 | -------------------------------------------------------------------------------- /release: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | RELEASE_TAG="$1" 4 | RELEASE_COMMENT="$2" 5 | 6 | if [ "$RELEASE_TAG" == "" ]; then 7 | echo "release version is missing!" 8 | echo "Usage:" 9 | echo " ./release v0.2.0" 10 | exit 1 11 | fi 12 | 13 | if [ "$RELEASE_COMMENT" == "" ]; then 14 | RELEASE_COMMENT="release $RELEASE_TAG" 15 | 16 | fi 17 | 18 | TOKEN_FILE=".ghtoken" 19 | if ! [ -f "$TOKEN_FILE" ]; then 20 | echo "Please store your github token to $TOKEN_FILE to perform a release" 21 | exit 2 22 | fi 23 | export GITHUB_TOKEN=$(cat $TOKEN_FILE) 24 | 25 | git tag -a "$RELEASE_TAG" -m "$RELEASE_COMMENT" 26 | git push origin "$RELEASE_TAG" 27 | goreleaser -------------------------------------------------------------------------------- /pattern/destination_test.go: -------------------------------------------------------------------------------- 1 | package pattern_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/sandreas/graft/pattern" 8 | "github.com/sandreas/graft/testhelpers" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestNewDestinationPattern(t *testing.T) { 13 | expect := assert.New(t) 14 | 15 | mockFs := testhelpers.MockFileSystem(map[string]string{ 16 | "data/tmp/": "", 17 | }) 18 | sep := string(os.PathSeparator) 19 | destinationPattern := pattern.NewDestinationPattern(mockFs, "data/tmp/new$1_file") 20 | expect.Equal("data"+sep+"tmp"+sep, destinationPattern.Path) 21 | expect.Equal("new${1}_file", destinationPattern.Pattern) 22 | } 23 | -------------------------------------------------------------------------------- /bitflag/bitflag_parser.go: -------------------------------------------------------------------------------- 1 | package bitflag 2 | 3 | 4 | type Flag byte 5 | 6 | 7 | type Parser struct { 8 | activeFlags Flag 9 | } 10 | 11 | 12 | func NewParser( params ...Flag) *Parser { 13 | parser := &Parser{} 14 | parser.parse(params) 15 | return parser 16 | } 17 | func (parser *Parser) parse(params []Flag) { 18 | size := len(params) 19 | 20 | parser.activeFlags = 0x00 21 | for i := 0; i < size; i++ { 22 | parser.activeFlags |= params[i] 23 | } 24 | } 25 | 26 | func (parser *Parser) SetFlag(flagToSet Flag) { 27 | parser.activeFlags |= flagToSet 28 | } 29 | 30 | func (parser *Parser) HasFlag(flagToCheck Flag) bool { 31 | return parser.activeFlags & flagToCheck != 0 32 | } 33 | -------------------------------------------------------------------------------- /pattern/destination.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "regexp" 5 | "github.com/spf13/afero" 6 | ) 7 | 8 | type DestinationPattern struct { 9 | BasePattern 10 | } 11 | 12 | func NewDestinationPattern(fs afero.Fs, patternString string) *DestinationPattern { 13 | destinationPattern := &DestinationPattern{} 14 | destinationPattern.Fs = fs 15 | destinationPattern.parse(patternString) 16 | destinationPattern.fixRegex() 17 | return destinationPattern 18 | } 19 | 20 | // replace $1_ with ${1}_ to prevent problems during rename 21 | func (p *DestinationPattern) fixRegex() { 22 | dollarUnderscore, _ := regexp.Compile("\\$([1-9][0-9]*)_") 23 | p.Pattern = dollarUnderscore.ReplaceAllString(p.Pattern, "${$1}_") 24 | } 25 | -------------------------------------------------------------------------------- /matcher/filetype_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/matcher" 7 | "github.com/sandreas/graft/testhelpers" 8 | ) 9 | 10 | func TestTypeMatcherForFileWithStat(t *testing.T) { 11 | expect := assert.New(t) 12 | fileToCheck := "file.txt" 13 | dirToCheck := "dir/" 14 | 15 | mockFs := testhelpers.MockFileSystem(map[string]string{ 16 | fileToCheck: "file", 17 | dirToCheck: "", 18 | }) 19 | 20 | m := matcher.NewFileTypeMatcher(matcher.TypeDirectory) 21 | m.Fs = mockFs 22 | expect.False(m.Matches(fileToCheck)) 23 | expect.True(m.Matches(dirToCheck)) 24 | 25 | m = matcher.NewFileTypeMatcher(matcher.TypeFile) 26 | m.Fs = mockFs 27 | expect.True(m.Matches(fileToCheck)) 28 | expect.False(m.Matches(dirToCheck)) 29 | } 30 | -------------------------------------------------------------------------------- /designpattern/observer/observable_test.go: -------------------------------------------------------------------------------- 1 | package designpattern_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/designpattern/observer" 7 | ) 8 | 9 | type FakeObserver struct { 10 | designpattern.ObservableInterface 11 | NotifyCalls int 12 | Argument int 13 | } 14 | 15 | func (fo *FakeObserver) Notify(a...interface{}) { 16 | fo.NotifyCalls++ 17 | fo.Argument = a[0].(int) 18 | } 19 | 20 | func TestNewDestinationPattern(t *testing.T) { 21 | expect := assert.New(t) 22 | 23 | observable := &designpattern.Observable{} 24 | observer := &FakeObserver{} 25 | observable.RegisterObserver(observer) 26 | 27 | expect.Equal(0, observer.NotifyCalls) 28 | expect.Equal(0, observer.Argument) 29 | observable.NotifyObservers(1234) 30 | expect.Equal(1, observer.NotifyCalls) 31 | expect.Equal(1234, observer.Argument) 32 | } -------------------------------------------------------------------------------- /transfer/messageprinter_observer.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "github.com/sandreas/graft/designpattern/observer" 5 | "strings" 6 | ) 7 | 8 | type MessagePrinterObserver struct { 9 | designpattern.ObserverInterface 10 | outputCallback func(format string, a ...interface{}) (int, error) 11 | } 12 | 13 | func NewMessagePrinterObserver(handle func(format string, a ...interface{}) (int, error)) *MessagePrinterObserver { 14 | return &MessagePrinterObserver{ 15 | outputCallback: handle, 16 | } 17 | } 18 | 19 | func (ph *MessagePrinterObserver) Notify(a...interface{}) { 20 | var str string 21 | var params[]interface{} 22 | a_len := len(a) 23 | if a_len > 0 { 24 | str = a[0].(string) 25 | } 26 | if a_len > 1 { 27 | params = a[1:] 28 | } else { 29 | str = strings.Replace(a[0].(string), "%", "%%", -1) 30 | } 31 | ph.outputCallback(str, params...) 32 | } 33 | -------------------------------------------------------------------------------- /matcher/filesize.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "github.com/sandreas/graft/filesystem" 5 | "github.com/spf13/afero" 6 | ) 7 | 8 | type FileSizeMatcher struct { 9 | MatcherInterface 10 | Fs afero.Fs 11 | minSize int64 12 | maxSize int64 13 | } 14 | 15 | func NewFileSizeMatcher(minSize, maxSize int64) *FileSizeMatcher { 16 | return &FileSizeMatcher{ 17 | Fs: filesystem.NewOsFs(), 18 | minSize: minSize, 19 | maxSize: maxSize, 20 | } 21 | } 22 | 23 | func (fsMatcher *FileSizeMatcher) Matches(subject interface{}) bool { 24 | fi, err := fsMatcher.Fs.Stat(subject.(string)) 25 | 26 | if err != nil || fi.IsDir() { 27 | return false 28 | } 29 | 30 | if fsMatcher.minSize < 0 { 31 | return fi.Size() <= fsMatcher.maxSize 32 | } 33 | 34 | if fsMatcher.maxSize < 0 { 35 | return fi.Size() >= fsMatcher.minSize 36 | } 37 | 38 | return fi.Size() >= fsMatcher.minSize && fi.Size() <= fsMatcher.maxSize 39 | } 40 | -------------------------------------------------------------------------------- /matcher/fileage.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/sandreas/graft/filesystem" 7 | "github.com/spf13/afero" 8 | ) 9 | 10 | type FileAgeMatcher struct { 11 | MatcherInterface 12 | Fs afero.Fs 13 | minAge time.Time 14 | maxAge time.Time 15 | } 16 | 17 | func NewFileAgeMatcher(minAge, maxAge time.Time) *FileAgeMatcher { 18 | return &FileAgeMatcher{ 19 | Fs: filesystem.NewOsFs(), 20 | minAge: minAge, 21 | maxAge: maxAge, 22 | } 23 | } 24 | 25 | func (faMatcher *FileAgeMatcher) Matches(subject interface{}) bool { 26 | fi, err := faMatcher.Fs.Stat(subject.(string)) 27 | if err != nil { 28 | return false 29 | } 30 | 31 | if faMatcher.maxAge.IsZero() { 32 | return faMatcher.minAge.Before(fi.ModTime()) 33 | } 34 | 35 | if faMatcher.minAge.IsZero() { 36 | return faMatcher.maxAge.After(fi.ModTime()) 37 | } 38 | 39 | return faMatcher.maxAge.After(fi.ModTime()) && faMatcher.minAge.Before(fi.ModTime()) 40 | 41 | } 42 | -------------------------------------------------------------------------------- /sftpd/sftp_fs_context.go: -------------------------------------------------------------------------------- 1 | package sftpd 2 | 3 | import ( 4 | "golang.org/x/crypto/ssh" 5 | "github.com/pkg/sftp" 6 | ) 7 | 8 | type SftpFsContext struct { 9 | sshc *ssh.Client 10 | sshcfg *ssh.ClientConfig 11 | Sftpc *sftp.Client 12 | } 13 | func (ctx *SftpFsContext) Disconnect() error { 14 | ctx.Sftpc.Close() 15 | ctx.sshc.Close() 16 | return nil 17 | } 18 | 19 | func NewSftpFsContext(user, password, host string) (*SftpFsContext, error) { 20 | sshcfg := &ssh.ClientConfig{ 21 | User: user, 22 | Auth: []ssh.AuthMethod{ 23 | ssh.Password(password), 24 | }, 25 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 26 | //HostKeyCallback: ssh.FixedHostKey(hostKey), 27 | } 28 | 29 | sshc, err := ssh.Dial("tcp", host, sshcfg) 30 | if err != nil { 31 | return nil,err 32 | } 33 | 34 | sftpc, err := sftp.NewClient(sshc) 35 | if err != nil { 36 | return nil,err 37 | } 38 | 39 | ctx := &SftpFsContext{ 40 | sshc: sshc, 41 | sshcfg: sshcfg, 42 | Sftpc: sftpc, 43 | } 44 | 45 | return ctx,nil 46 | } 47 | -------------------------------------------------------------------------------- /action/find.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/urfave/cli" 8 | ) 9 | 10 | type FindAction struct { 11 | AbstractAction 12 | } 13 | 14 | func (action *FindAction) Execute(c *cli.Context) error { 15 | action.PrepareExecution(c, 1) 16 | log.Printf("find") 17 | if err := action.locateSourceFiles(); err != nil { 18 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 19 | } 20 | action.ShowFoundFiles() 21 | return nil 22 | } 23 | func (action *FindAction) ShowFoundFiles() { 24 | 25 | if len(action.locator.SourceFiles) == 0 { 26 | action.suppressablePrintf("\nNo matching files found!\n") 27 | return 28 | } 29 | 30 | showMatches := action.CliContext.Bool("show-matches") && !action.CliParameters.Quiet 31 | for _, path := range action.locator.SourceFiles { 32 | fmt.Println(path) // quiet does not influence the output of the file listing, since this is the only sense of this action 33 | if showMatches { 34 | action.ShowMatchesForPath(path) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /matcher/filetype.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "github.com/sandreas/graft/filesystem" 5 | "github.com/spf13/afero" 6 | ) 7 | 8 | const ( 9 | TypeFile = "f" 10 | TypeDirectory = "d" 11 | ) 12 | 13 | type FileTypeMatcher struct { 14 | MatcherInterface 15 | Fs afero.Fs 16 | matchingTypes []string 17 | } 18 | 19 | func NewFileTypeMatcher(matchingTypes ...string) *FileTypeMatcher { 20 | return &FileTypeMatcher{ 21 | Fs: filesystem.NewOsFs(), 22 | matchingTypes: matchingTypes, 23 | } 24 | } 25 | 26 | func (faMatcher *FileTypeMatcher) Matches(subject interface{}) bool { 27 | fi, err := faMatcher.Fs.Stat(subject.(string)) 28 | if err != nil { 29 | return false 30 | } 31 | 32 | if fi.IsDir() { 33 | return contains(faMatcher.matchingTypes, TypeDirectory) 34 | } 35 | 36 | return contains(faMatcher.matchingTypes, TypeFile) 37 | } 38 | 39 | func contains(s []string, e string) bool { 40 | for _, a := range s { 41 | if a == e { 42 | return true 43 | } 44 | } 45 | return false 46 | } 47 | -------------------------------------------------------------------------------- /filesystem/sftpfs_context.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "golang.org/x/crypto/ssh" 5 | "github.com/pkg/sftp" 6 | ) 7 | 8 | type SftpFsContext struct { 9 | sshc *ssh.Client 10 | sshcfg *ssh.ClientConfig 11 | SftpClient *sftp.Client 12 | } 13 | 14 | func (ctx *SftpFsContext) Close() error { 15 | ctx.SftpClient.Close() 16 | ctx.sshc.Close() 17 | return nil 18 | } 19 | 20 | func NewSftpFsContext(user, password, host string) (*SftpFsContext, error) { 21 | 22 | sshcfg := &ssh.ClientConfig{ 23 | User: user, 24 | Auth: []ssh.AuthMethod{ 25 | ssh.Password(password), 26 | }, 27 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 28 | //HostKeyCallback: ssh.FixedHostKey(hostKey), 29 | } 30 | 31 | sshc, err := ssh.Dial("tcp", host, sshcfg) 32 | if err != nil { 33 | return nil,err 34 | } 35 | 36 | sftpc, err := sftp.NewClient(sshc) 37 | if err != nil { 38 | return nil,err 39 | } 40 | 41 | ctx := &SftpFsContext{ 42 | sshc: sshc, 43 | sshcfg: sshcfg, 44 | SftpClient: sftpc, 45 | } 46 | 47 | return ctx,nil 48 | } -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: graft 2 | release: 3 | github: 4 | owner: sandreas 5 | name: graft 6 | brew: 7 | install: bin.install "graft" 8 | builds: 9 | - goos: 10 | - linux 11 | - darwin 12 | - windows 13 | goarch: 14 | - amd64 15 | - arm 16 | - arm64 17 | goarm: 18 | - 6 19 | - 7 20 | ignore: 21 | - goos: darwin 22 | goarch: 386 23 | - goos: linux 24 | goarch: arm 25 | goarm: 7 26 | main: . 27 | ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} 28 | binary: graft 29 | archive: 30 | format: tar.gz 31 | name_template: '{{ .Binary }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ 32 | .Arm }}{{ end }}' 33 | files: 34 | - LICENCE* 35 | - README* 36 | - CHANGELOG* 37 | format_overrides: 38 | - goos: windows 39 | format: zip 40 | replacements: 41 | amd64: 64bit 42 | darwin: macOS 43 | snapshot: 44 | name_template: SNAPSHOT-{{ .Commit }} 45 | checksum: 46 | name_template: '{{ .ProjectName }}_{{ .Version }}_sha256_checksums.txt' 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 sandreas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /action/abstract_transfer.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "github.com/sandreas/graft/filesystem" 5 | "github.com/sandreas/graft/pattern" 6 | "github.com/urfave/cli" 7 | ) 8 | 9 | type AbstractTransferAction struct { 10 | AbstractAction 11 | destinationPattern *pattern.DestinationPattern 12 | } 13 | 14 | func (action *AbstractTransferAction) prepareTransferAction(c *cli.Context, positionalArgumentsCount int, positionalDefaultsIfUnset ...string) error { 15 | if err := action.PrepareExecution(c, positionalArgumentsCount, positionalDefaultsIfUnset...); err != nil { 16 | return err 17 | } 18 | if err := action.locateSourceFiles(); err != nil { 19 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 20 | } 21 | 22 | if err := action.prepareDestination(); err != nil { 23 | return cli.NewExitError(err.Error(), ErrorPrepareDestination) 24 | } 25 | return nil 26 | } 27 | 28 | func (action *AbstractTransferAction) prepareDestination() error { 29 | destinationFs := filesystem.NewOsFs() 30 | action.destinationPattern = pattern.NewDestinationPattern(destinationFs, action.PositionalArguments.Get(1)) 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /transfer/messageprinter_observer_test.go: -------------------------------------------------------------------------------- 1 | package transfer_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/transfer" 7 | ) 8 | 9 | var lastFakePrintfString string 10 | var lastFakePrintfParams []interface{} 11 | 12 | func FakePrintf(format string, a ...interface{}) (int, error) { 13 | lastFakePrintfString = format 14 | lastFakePrintfParams = []interface{}{} 15 | for value := range a { 16 | lastFakePrintfParams = append(lastFakePrintfParams, value) 17 | } 18 | 19 | return 0, nil 20 | 21 | } 22 | 23 | func TestParse(t *testing.T) { 24 | expect := assert.New(t) 25 | handler := transfer.NewMessagePrinterObserver(FakePrintf) 26 | 27 | handler.Notify("test-message") 28 | expect.Equal("test-message", lastFakePrintfString) 29 | expect.Len(lastFakePrintfParams, 0) 30 | 31 | handler.Notify("test-message with %s", "string") 32 | expect.Equal("test-message with %s", lastFakePrintfString) 33 | expect.Len(lastFakePrintfParams, 1) 34 | 35 | handler.Notify("test-message with a 100% percent sign and no parameters") 36 | expect.Equal("test-message with a 100%% percent sign and no parameters", lastFakePrintfString) 37 | expect.Len(lastFakePrintfParams, 0) 38 | } -------------------------------------------------------------------------------- /action/move.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "log" 5 | "github.com/urfave/cli" 6 | "github.com/sandreas/graft/transfer" 7 | "time" 8 | ) 9 | 10 | type MoveAction struct { 11 | AbstractTransferAction 12 | } 13 | 14 | func (action *MoveAction) Execute(c *cli.Context) error { 15 | log.Printf("move") 16 | 17 | if err := action.prepareTransferAction(c, 2); err != nil { 18 | return err 19 | } 20 | 21 | if err := action.MoveFiles(); err != nil { 22 | return cli.NewExitError(err.Error(), ErrorMoveFiles) 23 | } 24 | 25 | return nil 26 | } 27 | 28 | func (action *MoveAction) MoveFiles() error { 29 | messagePrinter := transfer.NewMessagePrinterObserver(action.suppressablePrintf) 30 | transferStrategy, err := transfer.NewTransferStrategy(transfer.Move, action.sourcePattern, action.destinationPattern) 31 | if err != nil { 32 | return err 33 | } 34 | transferStrategy.ProgressHandler = transfer.NewCopyProgressHandler(int64(32*1024), 1*time.Second) 35 | transferStrategy.RegisterObserver(messagePrinter) 36 | transferStrategy.DryRun = action.CliContext.Bool("dry-run") 37 | transferStrategy.KeepTimes = action.CliContext.Bool("times") 38 | return transferStrategy.Perform(action.locator.SourceFiles) 39 | 40 | } 41 | -------------------------------------------------------------------------------- /action/copy.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "log" 5 | "github.com/urfave/cli" 6 | "github.com/sandreas/graft/transfer" 7 | "time" 8 | ) 9 | 10 | type CopyAction struct { 11 | AbstractTransferAction 12 | } 13 | 14 | func (action *CopyAction) Execute(c *cli.Context) error { 15 | log.Printf("copy") 16 | 17 | if err := action.prepareTransferAction(c, 2); err != nil { 18 | return err 19 | } 20 | 21 | if err := action.CopyFiles(); err != nil { 22 | return cli.NewExitError(err.Error(), ErrorCopyFiles) 23 | } 24 | 25 | return nil 26 | } 27 | 28 | func (action *CopyAction) CopyFiles() error { 29 | messagePrinter := transfer.NewMessagePrinterObserver(action.suppressablePrintf) 30 | transferStrategy, err := transfer.NewTransferStrategy(transfer.CopyResumed, action.sourcePattern, action.destinationPattern) 31 | if err != nil { 32 | return err 33 | } 34 | transferStrategy.ProgressHandler = transfer.NewCopyProgressHandler(int64(32*1024), 1*time.Second) 35 | transferStrategy.RegisterObserver(messagePrinter) 36 | transferStrategy.DryRun = action.CliContext.Bool("dry-run") 37 | transferStrategy.KeepTimes = action.CliContext.Bool("times") 38 | return transferStrategy.Perform(action.locator.SourceFiles) 39 | 40 | } 41 | -------------------------------------------------------------------------------- /file/path_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | ) 7 | 8 | func TestNewPath(t *testing.T) { 9 | expect := assert.New(t) 10 | 11 | p := NewPath("../data/test.txt") 12 | expect.Equal(".."+separator+"data"+separator+"test.txt", p.String()) 13 | expect.True(p.IsFile()) 14 | expect.False(p.IsDir()) 15 | expect.False(p.IsAbs()) 16 | 17 | 18 | p = NewPath("/tmp/test") 19 | expect.Equal(""+separator+"tmp"+separator+"test", p.String()) 20 | expect.True(p.IsFile()) 21 | expect.False(p.IsDir()) 22 | expect.True(p.IsAbs()) 23 | 24 | 25 | p = NewPath("./data/test.txt") 26 | expect.Equal("data"+separator+"test.txt", p.String()) 27 | expect.True(p.IsFile()) 28 | expect.False(p.IsDir()) 29 | expect.False(p.IsAbs()) 30 | 31 | p = NewPath("./data/fixtures/") 32 | expect.Equal("data"+separator+"fixtures"+separator, p.String()) 33 | expect.False(p.IsFile()) 34 | expect.True(p.IsDir()) 35 | expect.False(p.IsAbs()) 36 | 37 | //p = NewPath(".\\data//fixtures\\") 38 | //expect.Equal("data"+separator+"fixtures"+separator+"", p.String()) 39 | //expect.False(p.IsFile()) 40 | //expect.True(p.IsDir()) 41 | //expect.False(p.IsAbs()) 42 | 43 | //p := NewPath("\\\\tmp\\test\\uncpath.txt") 44 | //expect.Equal(separator+separator+"tmp"+separator+"test"+separator+"uncpath.txt", p.String()) 45 | //expect.False(p.IsFile()) 46 | //expect.True(p.IsDir()) 47 | //expect.True(p.IsAbs()) 48 | } 49 | -------------------------------------------------------------------------------- /sftpd/file_tree.go: -------------------------------------------------------------------------------- 1 | package sftpd 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | ) 7 | 8 | type FileTree struct { 9 | basePath string 10 | key string 11 | children []FileTree 12 | } 13 | 14 | 15 | func normalizePath(path string) string { 16 | path = filepath.ToSlash(path) 17 | path = strings.TrimPrefix(path, "./") 18 | if path == "." { 19 | return "" 20 | } 21 | return strings.TrimRight(path, "/") 22 | } 23 | 24 | 25 | func NewFileTree(basePath string) *FileTree { 26 | basePath = normalizePath(basePath) 27 | rootPath := "" 28 | if strings.HasPrefix(basePath, "/") { 29 | rootPath = "/" 30 | } 31 | return &FileTree{ 32 | basePath: basePath, 33 | key: rootPath, 34 | } 35 | } 36 | 37 | //func (tree *FileTree) Integrate(path string) error { 38 | // path = strings.TrimPrefix(normalizePath(path), tree.basePath) 39 | // 40 | // if tree.key == "" && strings.HasPrefix(path, "/") { 41 | // return errors.New("Absolute path " + path + " cannot be integrated into relative tree") 42 | // } 43 | // 44 | // parts := strings.Split(path, "/") 45 | // 46 | // len := len(tree.children) 47 | // for i:=0;i ph.Interval*10 { 62 | ph.Interval = 500 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /file/walk_observer_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/file" 7 | ) 8 | 9 | var lastFakePrintfString string 10 | var lastFakePrintfParams []interface{} 11 | 12 | func FakePrintf(format string, a ...interface{}) (int, error) { 13 | lastFakePrintfString = format 14 | lastFakePrintfParams = []interface{}{} 15 | for value := range a { 16 | lastFakePrintfParams = append(lastFakePrintfParams, value) 17 | } 18 | 19 | return 0, nil 20 | 21 | } 22 | 23 | func TestParse(t *testing.T) { 24 | expect := assert.New(t) 25 | handler := file.NewWalkObserver(FakePrintf) 26 | handler.Interval = 2 27 | handler.Notify(file.LocatorIncreaseItems) 28 | 29 | expect.Equal("\rscanning - total: %d, matches: %d", lastFakePrintfString) 30 | expect.Len(lastFakePrintfParams, 2) 31 | 32 | handler.Notify(file.LocatorIncreaseItems) 33 | 34 | expect.Equal("\rscanning - total: %d, matches: %d", lastFakePrintfString) 35 | expect.Len(lastFakePrintfParams, 2) 36 | 37 | handler.Notify(file.LocatorIncreaseItems) 38 | handler.Notify(file.LocatorIncreaseMatches) 39 | 40 | expect.Equal("\rscanning - total: %d, matches: %d", lastFakePrintfString) 41 | expect.Len(lastFakePrintfParams, 2) 42 | 43 | for i := 0; i < 20; i++ { 44 | handler.Notify(file.LocatorIncreaseItems) 45 | } 46 | 47 | expect.Equal(int64(500), handler.Interval) 48 | 49 | handler.Notify(file.LocatorIncreaseErrors) 50 | expect.Equal("\rscanning - total: %d, matches: %d, errors: %d", lastFakePrintfString) 51 | expect.Len(lastFakePrintfParams, 3) 52 | 53 | handler.Notify(file.LocatorFinish) 54 | expect.Equal("\n", lastFakePrintfString) 55 | expect.Len(lastFakePrintfParams, 0) 56 | } -------------------------------------------------------------------------------- /matcher/fileage_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/matcher" 7 | "time" 8 | "github.com/sandreas/graft/testhelpers" 9 | ) 10 | 11 | // time.Time{} 12 | 13 | func TestFileAgeMatcher(t *testing.T) { 14 | expect := assert.New(t) 15 | 16 | time2014 := time.Date(2014, 1, 1, 1, 1, 1, 1, time.Local) 17 | modTime := time.Date(2015, 1, 1, 1, 1, 1, 1, time.Local) 18 | time2016 := time.Date(2016, 1, 1, 1, 1, 1, 1, time.Local) 19 | fileToCheck := "file.txt" 20 | mockFs := testhelpers.MockFileSystem(map[string]string{ 21 | fileToCheck: "", 22 | }) 23 | mockFs.Chtimes(fileToCheck, modTime, modTime) 24 | 25 | 26 | subject := matcher.NewFileAgeMatcher( time.Time{}, time2016) 27 | subject.Fs = mockFs 28 | expect.True(subject.Matches(fileToCheck)) 29 | 30 | subject = matcher.NewFileAgeMatcher( time2014, time.Time{}) 31 | subject.Fs = mockFs 32 | expect.True(subject.Matches(fileToCheck)) 33 | 34 | subject = matcher.NewFileAgeMatcher( time2014, time2016) 35 | subject.Fs = mockFs 36 | expect.True(subject.Matches(fileToCheck)) 37 | 38 | subject = matcher.NewFileAgeMatcher( time.Time{}, time.Time{}) 39 | subject.Fs = mockFs 40 | expect.True(subject.Matches(fileToCheck)) 41 | } 42 | 43 | func TestFileAgeMatcherWithoutStat(t *testing.T) { 44 | expect := assert.New(t) 45 | fileToCheck := "../data/fixtures/global/file.txt" 46 | m := matcher.NewFileAgeMatcher(time.Time{}, time.Now()) 47 | expect.True(m.Matches(fileToCheck)) 48 | } 49 | 50 | func TestFileNotExists(t *testing.T) { 51 | expect := assert.New(t) 52 | fileToCheck := "not-exists.txt" 53 | m := matcher.NewFileAgeMatcher( time.Time{}, time.Now()) 54 | expect.False(m.Matches(fileToCheck)) 55 | } 56 | 57 | -------------------------------------------------------------------------------- /pattern/base.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "path/filepath" 5 | 6 | "os" 7 | 8 | "github.com/sandreas/graft/filesystem" 9 | "github.com/spf13/afero" 10 | "strings" 11 | ) 12 | 13 | type BasePattern struct { 14 | Fs afero.Fs 15 | Path string 16 | Pattern string 17 | isDirectory bool 18 | isLocalFs bool 19 | } 20 | 21 | func NewBasePattern(fs afero.Fs, patternString string) *BasePattern { 22 | _, isOsFs := fs.(*afero.OsFs) 23 | _, isMemMapFs := fs.(*afero.MemMapFs) 24 | 25 | basePattern := &BasePattern{ 26 | Fs: fs, 27 | isLocalFs: isOsFs || isMemMapFs, 28 | } 29 | 30 | basePattern.parse(patternString) 31 | return basePattern 32 | } 33 | 34 | func (p *BasePattern) parse(patternString string) { 35 | 36 | sep := string(os.PathSeparator) 37 | patternString = strings.TrimPrefix(patternString, "./") 38 | pathPart := patternString 39 | 40 | for { 41 | if fi, err := p.Fs.Stat(pathPart); err == nil { 42 | p.Path = filesystem.CleanPath(p.Fs, pathPart) 43 | p.isDirectory = fi == nil || fi.IsDir() 44 | if p.isDirectory && !os.IsPathSeparator(p.Path[len(p.Path)-1]) { 45 | p.Path += sep 46 | } 47 | break 48 | } 49 | 50 | parent := filepath.Dir(pathPart) 51 | if parent == pathPart || parent == "." { 52 | p.Path = "" 53 | p.Pattern = pathPart 54 | break 55 | } 56 | pathPart = parent 57 | } 58 | 59 | if len(patternString) > 0 && pathPart != patternString { 60 | p.Pattern = patternString[len(p.Path):] 61 | } 62 | if p.Path == "" || p.Path == "." { 63 | p.Path = "." + sep 64 | p.isDirectory = true 65 | } 66 | } 67 | 68 | func (p *BasePattern) IsDir() bool { 69 | return !p.IsFile() 70 | } 71 | 72 | func (p *BasePattern) IsFile() bool { 73 | return !p.isDirectory && p.Pattern == "" 74 | } 75 | -------------------------------------------------------------------------------- /filesystem/path_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package filesystem 4 | 5 | import ( 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | 10 | "github.com/spf13/afero" 11 | ) 12 | 13 | func Walk(fs afero.Fs, root string, walkFn filepath.WalkFunc) error { 14 | info, err := lstatIfOs(fs, root) 15 | if err != nil { 16 | return walkFn(root, nil, err) 17 | } 18 | return walk(fs, root, info, walkFn) 19 | } 20 | 21 | func lstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) { 22 | _, ok := fs.(*OsFs) 23 | if ok { 24 | info, err = os.Lstat(makeAbsolute(path)) 25 | } else { 26 | info, err = fs.Stat(path) 27 | } 28 | return info, err 29 | } 30 | 31 | func walk(fs afero.Fs, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { 32 | err := walkFn(path, info, nil) 33 | if err != nil { 34 | if info.IsDir() && err == filepath.SkipDir { 35 | return nil 36 | } 37 | return err 38 | } 39 | 40 | if !info.IsDir() { 41 | return nil 42 | } 43 | 44 | names, err := readDirNames(fs, path) 45 | if err != nil { 46 | return walkFn(path, info, err) 47 | } 48 | 49 | for _, name := range names { 50 | filename := filepath.Join(path, name) 51 | fileInfo, err := lstatIfOs(fs, filename) 52 | if err != nil { 53 | if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { 54 | return err 55 | } 56 | } else { 57 | err = walk(fs, filename, fileInfo, walkFn) 58 | if err != nil { 59 | if !fileInfo.IsDir() || err != filepath.SkipDir { 60 | return err 61 | } 62 | } 63 | } 64 | } 65 | return nil 66 | } 67 | 68 | func readDirNames(fs afero.Fs, dirname string) ([]string, error) { 69 | f, err := fs.Open(dirname) 70 | if err != nil { 71 | return nil, err 72 | } 73 | names, err := f.Readdirnames(-1) 74 | f.Close() 75 | if err != nil { 76 | return nil, err 77 | } 78 | sort.Strings(names) 79 | return names, nil 80 | } 81 | -------------------------------------------------------------------------------- /matcher/filesize_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/matcher" 7 | "github.com/sandreas/graft/testhelpers" 8 | ) 9 | 10 | func TestSizeMatcherForFileWithStat(t *testing.T) { 11 | expect := assert.New(t) 12 | fileToCheck := "file.txt" 13 | 14 | mockFs := testhelpers.MockFileSystem(map[string]string{ 15 | fileToCheck: "file", 16 | }) 17 | 18 | m := matcher.NewFileSizeMatcher(-1, 2) 19 | m.Fs = mockFs 20 | expect.False(m.Matches(fileToCheck)) 21 | 22 | m = matcher.NewFileSizeMatcher(-1, 4) 23 | m.Fs = mockFs 24 | expect.True(m.Matches(fileToCheck)) 25 | 26 | m = matcher.NewFileSizeMatcher(-1, 15) 27 | m.Fs = mockFs 28 | expect.True(m.Matches(fileToCheck)) 29 | 30 | m = matcher.NewFileSizeMatcher(0, -1) 31 | m.Fs = mockFs 32 | expect.True(m.Matches(fileToCheck)) 33 | 34 | m = matcher.NewFileSizeMatcher(5, -1) 35 | m.Fs = mockFs 36 | expect.False(m.Matches(fileToCheck)) 37 | 38 | m = matcher.NewFileSizeMatcher(3, 4) 39 | m.Fs = mockFs 40 | expect.True(m.Matches(fileToCheck)) 41 | 42 | m = matcher.NewFileSizeMatcher(3, 3) 43 | m.Fs = mockFs 44 | expect.False(m.Matches(fileToCheck)) 45 | 46 | m = matcher.NewFileSizeMatcher(5, 4) 47 | m.Fs = mockFs 48 | expect.False(m.Matches(fileToCheck)) 49 | 50 | m = matcher.NewFileSizeMatcher(-1, -1) 51 | m.Fs = mockFs 52 | expect.False(m.Matches(fileToCheck)) 53 | } 54 | 55 | func TestSizeMatcherForFile(t *testing.T) { 56 | expect := assert.New(t) 57 | fileToCheck := "../data/fixtures/global/file.txt" 58 | m := matcher.NewFileSizeMatcher(3, 4) 59 | expect.True(m.Matches(fileToCheck)) 60 | } 61 | 62 | func TestSizeMatcherForDirWithStat(t *testing.T) { 63 | expect := assert.New(t) 64 | dirToCheck := "fixtures/" 65 | 66 | mockFs := testhelpers.MockFileSystem(map[string]string{ 67 | dirToCheck: "", 68 | }) 69 | 70 | m := matcher.NewFileSizeMatcher(0, 5) 71 | m.Fs = mockFs 72 | expect.False(m.Matches(dirToCheck)) 73 | } 74 | -------------------------------------------------------------------------------- /bitflag/bitflag_parser_test.go: -------------------------------------------------------------------------------- 1 | package bitflag_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/bitflag" 7 | ) 8 | const ( 9 | FLAG_ONE bitflag.Flag = 1 << iota 10 | FLAG_TWO 11 | FLAG_THREE 12 | FLAG_FOUR 13 | ) 14 | 15 | 16 | func TestParseSingleOrConcatBitFlagParam(t *testing.T) { 17 | expect := assert.New(t) 18 | 19 | subject := bitflag.NewParser(FLAG_ONE|FLAG_FOUR) 20 | 21 | expect.True(subject.HasFlag(FLAG_ONE)) 22 | expect.False(subject.HasFlag(FLAG_TWO)) 23 | expect.False(subject.HasFlag(FLAG_THREE)) 24 | expect.True(subject.HasFlag(FLAG_FOUR)) 25 | } 26 | 27 | 28 | func TestParseMultipleBitFlagParams(t *testing.T) { 29 | expect := assert.New(t) 30 | 31 | subject := bitflag.NewParser(FLAG_ONE, FLAG_FOUR) 32 | 33 | expect.True(subject.HasFlag(FLAG_ONE)) 34 | expect.False(subject.HasFlag(FLAG_TWO)) 35 | expect.False(subject.HasFlag(FLAG_THREE)) 36 | expect.True(subject.HasFlag(FLAG_FOUR)) 37 | } 38 | 39 | func TestParseNoFlags(t *testing.T) { 40 | expect := assert.New(t) 41 | 42 | subject := bitflag.NewParser() 43 | 44 | expect.False(subject.HasFlag(FLAG_ONE)) 45 | expect.False(subject.HasFlag(FLAG_TWO)) 46 | expect.False(subject.HasFlag(FLAG_THREE)) 47 | expect.False(subject.HasFlag(FLAG_FOUR)) 48 | } 49 | 50 | 51 | func TestSetFlag(t *testing.T) { 52 | expect := assert.New(t) 53 | 54 | subject := bitflag.NewParser(FLAG_ONE, FLAG_FOUR) 55 | 56 | expect.True(subject.HasFlag(FLAG_ONE)) 57 | expect.False(subject.HasFlag(FLAG_TWO)) 58 | expect.False(subject.HasFlag(FLAG_THREE)) 59 | expect.True(subject.HasFlag(FLAG_FOUR)) 60 | 61 | subject.SetFlag(FLAG_TWO) 62 | expect.True(subject.HasFlag(FLAG_ONE)) 63 | expect.True(subject.HasFlag(FLAG_TWO)) 64 | expect.False(subject.HasFlag(FLAG_THREE)) 65 | expect.True(subject.HasFlag(FLAG_FOUR)) 66 | 67 | subject.SetFlag(FLAG_THREE) 68 | expect.True(subject.HasFlag(FLAG_ONE)) 69 | expect.True(subject.HasFlag(FLAG_TWO)) 70 | expect.True(subject.HasFlag(FLAG_THREE)) 71 | expect.True(subject.HasFlag(FLAG_FOUR)) 72 | } 73 | -------------------------------------------------------------------------------- /file/path.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | //:host 11 | //:device 12 | //:version 13 | //:directory 14 | //:name 15 | //:type 16 | 17 | const ( 18 | typeDir = 1 19 | typeFile = 2 20 | separator = string(os.PathSeparator) 21 | ) 22 | 23 | type Path struct { 24 | fmt.Stringer 25 | volume string 26 | path string 27 | name string 28 | kind int 29 | absolute bool 30 | } 31 | 32 | func NewPath(path string) *Path { 33 | vol := filepath.VolumeName(path) 34 | path = strings.TrimPrefix(path, vol) 35 | kind := getPathType(path) 36 | path = filepath.Clean(path) 37 | name := "" 38 | 39 | if kind == typeFile { 40 | dir := filepath.Dir(path) + separator 41 | name = strings.TrimPrefix(path, dir) 42 | path = strings.TrimSuffix(dir, separator) 43 | } 44 | if path == "." { 45 | path = "" 46 | } 47 | 48 | p := &Path{ 49 | volume: filepath.FromSlash(vol), 50 | path: path, 51 | name: name, 52 | kind: kind, 53 | absolute: false, 54 | } 55 | 56 | p.normalize() 57 | return p 58 | } 59 | 60 | func getPathType(path string) int { 61 | if strings.HasSuffix(path, "/") || strings.HasSuffix(path, "\\") { 62 | return typeDir 63 | } 64 | return typeFile 65 | } 66 | 67 | func (path *Path) normalize() { 68 | if path.volume != "" || strings.HasPrefix(path.path, separator) { 69 | path.absolute = true 70 | path.volume = strings.TrimRight(path.volume, "\\/") 71 | } else { 72 | path.path = strings.TrimPrefix(path.path, "."+separator) 73 | } 74 | } 75 | 76 | func (path *Path) build() string { 77 | if path.absolute { 78 | return path.volume + path.path + separator + path.name 79 | } 80 | return path.path + separator + path.name 81 | } 82 | 83 | func (path *Path) String() string { 84 | return path.build() 85 | } 86 | 87 | func (path *Path) IsDir() bool { 88 | return path.kind == typeDir 89 | } 90 | 91 | func (path *Path) IsFile() bool { 92 | return path.kind == typeFile 93 | } 94 | 95 | func (path *Path) IsAbs() bool { 96 | return path.absolute 97 | } 98 | -------------------------------------------------------------------------------- /pattern/source.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "path/filepath" 5 | "regexp" 6 | 7 | "os" 8 | 9 | "github.com/sandreas/graft/bitflag" 10 | "github.com/spf13/afero" 11 | "strings" 12 | ) 13 | 14 | const ( 15 | CASE_SENSITIVE bitflag.Flag = 1 << iota 16 | USE_REAL_REGEX 17 | ) 18 | 19 | type SourcePattern struct { 20 | BasePattern 21 | caseSensitive bool 22 | useRealRegex bool 23 | } 24 | 25 | func NewSourcePattern(fs afero.Fs, patternString string, params ...bitflag.Flag) *SourcePattern { 26 | sourcePattern := &SourcePattern{} 27 | sourcePattern.Fs = fs 28 | sourcePattern.parse(patternString) 29 | 30 | bitFlags := bitflag.NewParser(params...) 31 | sourcePattern.caseSensitive = bitFlags.HasFlag(CASE_SENSITIVE) 32 | sourcePattern.useRealRegex = bitFlags.HasFlag(USE_REAL_REGEX) 33 | 34 | return sourcePattern 35 | } 36 | 37 | func (p *SourcePattern) Compile() (*regexp.Regexp, error) { 38 | // pattern handling 39 | regexPattern := p.Pattern 40 | if !p.useRealRegex { 41 | regexPattern = GlobToRegexString(p.Pattern) 42 | } 43 | if p.IsDir() && p.Pattern == "" { 44 | regexPattern = "(.*)" 45 | } 46 | 47 | // path handling 48 | regexPath := strings.TrimPrefix(p.Path, "." + string(os.PathSeparator)) 49 | 50 | if regexPath != "" { 51 | regexPath = filepath.ToSlash(p.Path) 52 | if regexPath[len(regexPath)-1] != '/' && !p.IsFile() { 53 | regexPath += "/" 54 | } 55 | regexPath = regexp.QuoteMeta(regexPath) 56 | } 57 | 58 | if !p.caseSensitive { 59 | regexPath = "(?i)" + regexPath 60 | } 61 | 62 | // replace double path separator with single slash 63 | r := regexp.MustCompile("[" +regexp.QuoteMeta(string(os.PathSeparator)) + "/]{2,}") 64 | regexPattern = r.ReplaceAllStringFunc(regexPattern, func(m string) string { 65 | return "/" 66 | }) 67 | 68 | suffix := "$" 69 | compiledPattern, err := regexp.Compile(regexPath + regexPattern + suffix) 70 | if err == nil && compiledPattern.NumSubexp() == 0 && p.Pattern != "" { 71 | compiledPattern, err = regexp.Compile(regexPath + "(" + regexPattern + ")" + suffix) 72 | } 73 | 74 | return compiledPattern, err 75 | } 76 | -------------------------------------------------------------------------------- /action/delete.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | "github.com/urfave/cli" 12 | ) 13 | 14 | type DeleteAction struct { 15 | AbstractAction 16 | } 17 | 18 | func (action *DeleteAction) Execute(c *cli.Context) error { 19 | log.Printf("delete") 20 | 21 | if err := action.PrepareExecution(c, 1); err != nil { 22 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 23 | } 24 | if err := action.locateSourceFiles(); err != nil { 25 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 26 | } 27 | 28 | if err := action.DeleteFiles(); err != nil { 29 | return cli.NewExitError(err.Error(), ErrorDeleteFiles) 30 | } 31 | return nil 32 | } 33 | 34 | func (action *DeleteAction) DeleteFiles() error { 35 | var dirsToRemove = []string{} 36 | dryRun := action.CliContext.Bool("dry-run") 37 | fileCount := len(action.locator.SourceFiles) 38 | if !dryRun && fileCount > 0 && !action.CliParameters.Quiet && !action.CliParameters.Force { 39 | reader := bufio.NewReader(os.Stdin) 40 | fmt.Printf("%d files will be deleted. proceed (y/N)?:", fileCount) 41 | text, _ := reader.ReadString('\n') 42 | 43 | if strings.ToLower(strings.TrimSpace(text)) != "y" { 44 | return errors.New("Deletion aborted by user") 45 | } 46 | } 47 | 48 | for _, path := range action.locator.SourceFiles { 49 | action.suppressablePrintf(path + "\n") 50 | // delete 51 | if !action.CliContext.Bool("dry-run") { 52 | stat, err := action.sourcePattern.Fs.Stat(path) 53 | if !os.IsNotExist(err) { 54 | if stat.Mode().IsRegular() { 55 | if err := action.sourcePattern.Fs.Remove(path); err != nil { 56 | log.Printf("File %s could not be deleted: %s", path, err.Error()) 57 | } 58 | } else if stat.Mode().IsDir() { 59 | dirsToRemove = append(dirsToRemove, path) 60 | } 61 | } 62 | } 63 | } 64 | 65 | for _, path := range dirsToRemove { 66 | if err := action.sourcePattern.Fs.Remove(path); err != nil { 67 | log.Printf("Directory %s could not be deleted: %s", path, err.Error()) 68 | } 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /file/locator_cache_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "github.com/spf13/afero" 5 | "testing" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/sandreas/graft/file" 8 | ) 9 | 10 | 11 | func TestLoadNonExisting(t *testing.T) { 12 | expect := assert.New(t) 13 | 14 | cacheFile := "locator-cache.txt" 15 | subject := file.NewLocatorCache(cacheFile) 16 | subject.Fs = prepareFilesystemTest("", "") 17 | err := subject.Load() 18 | expect.Error(err) 19 | expect.Len(subject.Items, 0) 20 | } 21 | 22 | func TestLoadExisting(t *testing.T) { 23 | expect := assert.New(t) 24 | 25 | cacheFile := "locator-cache.txt" 26 | subject := file.NewLocatorCache(cacheFile) 27 | subject.Fs = prepareFilesystemTest(cacheFile, "data/file1.txt\ndata/file2.txt\n") 28 | err := subject.Load() 29 | 30 | expect.Nil(err) 31 | expect.Len(subject.Items, 2) 32 | } 33 | 34 | func TestSaveErrorWhenFileExists(t *testing.T) { 35 | expect := assert.New(t) 36 | 37 | cacheFile := "locator-cache.txt" 38 | cacheContent := "original-content" 39 | subject := file.NewLocatorCache(cacheFile) 40 | subject.Fs = prepareFilesystemTest(cacheFile, cacheContent) 41 | err := subject.Save() 42 | expect.Error(err) 43 | actual, err := afero.ReadFile(subject.Fs, cacheFile) 44 | expect.Nil(err) 45 | expect.Equal("original-content", string(actual)) 46 | } 47 | 48 | func TestSave(t *testing.T) { 49 | expect := assert.New(t) 50 | 51 | cacheFile := "locator-cache.txt" 52 | subject := file.NewLocatorCache(cacheFile) 53 | subject.Fs = prepareFilesystemTest("", "") 54 | 55 | 56 | subject.Items = []string{ 57 | "data/file1.txt", 58 | "data/file2.txt\n", 59 | "data/file3.txt\r\n", 60 | "data/file3.txt", 61 | } 62 | err := subject.Save() 63 | expect.Nil(err) 64 | 65 | actual, err := afero.ReadFile(subject.Fs, cacheFile) 66 | expect.Nil(err) 67 | expect.Equal("data/file1.txt\ndata/file2.txt\ndata/file3.txt\ndata/file3.txt\n", string(actual)) 68 | } 69 | 70 | func prepareFilesystemTest(cacheFile, content string) afero.Fs { 71 | appFS := afero.NewMemMapFs() 72 | if cacheFile != "" { 73 | afero.WriteFile(appFS, cacheFile, []byte(content), 0644) 74 | } 75 | return appFS 76 | } -------------------------------------------------------------------------------- /filesystem/osfs.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package filesystem 4 | 5 | import ( 6 | "os" 7 | "time" 8 | 9 | "github.com/spf13/afero" 10 | ) 11 | 12 | const ( 13 | NameOsfs = "OsFs" 14 | ) 15 | 16 | type OsFs struct { 17 | afero.Fs 18 | } 19 | 20 | func NewOsFs() afero.Fs { 21 | return &OsFs{} 22 | } 23 | 24 | func (OsFs) Name() string { return NameOsfs } 25 | 26 | func (OsFs) Create(name string) (afero.File, error) { 27 | f, e := os.Create(name) 28 | if f == nil { 29 | // while this looks strange, we need to return a bare nil (of type nil) not 30 | // a nil value of type *os.File or nil won't be nil 31 | return nil, e 32 | } 33 | return f, e 34 | } 35 | 36 | func (OsFs) Mkdir(name string, perm os.FileMode) error { 37 | return os.Mkdir(name, perm) 38 | } 39 | 40 | func (OsFs) MkdirAll(path string, perm os.FileMode) error { 41 | return os.MkdirAll(path, perm) 42 | } 43 | 44 | func (OsFs) Open(name string) (afero.File, error) { 45 | f, e := os.Open(name) 46 | if f == nil { 47 | // while this looks strange, we need to return a bare nil (of type nil) not 48 | // a nil value of type *os.File or nil won't be nil 49 | return nil, e 50 | } 51 | return f, e 52 | } 53 | 54 | func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { 55 | f, e := os.OpenFile(name, flag, perm) 56 | if f == nil { 57 | // while this looks strange, we need to return a bare nil (of type nil) not 58 | // a nil value of type *os.File or nil won't be nil 59 | return nil, e 60 | } 61 | return f, e 62 | } 63 | 64 | func (OsFs) Remove(name string) error { 65 | return os.Remove(name) 66 | } 67 | 68 | func (OsFs) RemoveAll(path string) error { 69 | return os.RemoveAll(path) 70 | } 71 | 72 | func (OsFs) Rename(oldname, newname string) error { 73 | return os.Rename(oldname, newname) 74 | } 75 | 76 | func (OsFs) Stat(name string) (os.FileInfo, error) { 77 | return os.Stat(name) 78 | } 79 | 80 | func (OsFs) Chmod(name string, mode os.FileMode) error { 81 | return os.Chmod(name, mode) 82 | } 83 | 84 | func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { 85 | return os.Chtimes(name, atime, mtime) 86 | } 87 | 88 | func (OsFs) Close() { 89 | // osfs does not need to be closed 90 | } 91 | -------------------------------------------------------------------------------- /file/locator.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "path/filepath" 8 | 9 | "github.com/sandreas/graft/designpattern/observer" 10 | "github.com/sandreas/graft/filesystem" 11 | "github.com/sandreas/graft/matcher" 12 | "github.com/sandreas/graft/pattern" 13 | ) 14 | 15 | const ( 16 | LocatorIncreaseItems = 1 17 | LocatorIncreaseMatches = 2 18 | LocatorFinish = 3 19 | LocatorIncreaseErrors = 4 20 | ) 21 | 22 | type Locator struct { 23 | designpattern.Observable 24 | Src pattern.SourcePattern 25 | SourceFiles []string 26 | } 27 | 28 | func NewLocator(pattern *pattern.SourcePattern) *Locator { 29 | return &Locator{ 30 | Src: *pattern, 31 | } 32 | } 33 | 34 | func (t *Locator) Find(matcher *matcher.CompositeMatcher) { 35 | t.SourceFiles = []string{} 36 | if t.Src.IsFile() { 37 | t.SourceFiles = append(t.SourceFiles, t.Src.Path) 38 | 39 | t.NotifyObservers(LocatorIncreaseMatches) 40 | t.NotifyObservers(LocatorFinish) 41 | return 42 | } 43 | 44 | walkPath := filesystem.CleanPath(t.Src.Fs, t.Src.Path) 45 | walkPathSeparator := string(os.PathSeparator) 46 | if t.Src.Fs.Name() == filesystem.NameSftpfs { 47 | walkPathSeparator = "/" 48 | } 49 | ferr := filesystem.Walk(t.Src.Fs, walkPath, func(innerPath string, info os.FileInfo, err error) error { 50 | if innerPath == "." || innerPath == ".." { 51 | return nil 52 | } 53 | 54 | if err != nil { 55 | t.NotifyObservers(LocatorIncreaseErrors) 56 | log.Printf("WalkError: %s, Details: %v", err.Error(), err) 57 | return nil 58 | } 59 | 60 | normalizedInnerPath := filesystem.CleanPath(t.Src.Fs, innerPath) 61 | 62 | // skip direct path matches (data/* should not match data/ itself) 63 | if normalizedInnerPath == walkPath && t.Src.Pattern != "" { 64 | return nil 65 | } 66 | 67 | if info.IsDir() { 68 | normalizedInnerPath += walkPathSeparator 69 | } 70 | 71 | if matcher.Matches(filepath.ToSlash(normalizedInnerPath)) { 72 | t.SourceFiles = append(t.SourceFiles, normalizedInnerPath) 73 | t.NotifyObservers(LocatorIncreaseMatches) 74 | } else { 75 | t.NotifyObservers(LocatorIncreaseItems) 76 | } 77 | 78 | return nil 79 | }) 80 | 81 | if ferr != nil { 82 | log.Printf("Error walking files: %s\n", ferr.Error()) 83 | } 84 | 85 | t.NotifyObservers(LocatorFinish) 86 | } 87 | -------------------------------------------------------------------------------- /filesystem/sftpfs_file.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | 4 | import ( 5 | "github.com/pkg/sftp" 6 | "os" 7 | "io" 8 | ) 9 | 10 | type File struct { 11 | file *sftp.File 12 | client *sftp.Client 13 | name string 14 | } 15 | 16 | func FileOpen(s *sftp.Client, name string) (*File, error) { 17 | fd, err := s.Open(name) 18 | if err != nil { 19 | return &File{}, err 20 | } 21 | return &File{ 22 | file: fd, 23 | client: s, 24 | name: name, 25 | }, nil 26 | } 27 | 28 | func FileCreate(s *sftp.Client, name string) (*File, error) { 29 | fd, err := s.Create(name) 30 | if err != nil { 31 | return &File{}, err 32 | } 33 | return &File{ 34 | file: fd, 35 | client: s, 36 | name: name, 37 | }, nil 38 | } 39 | 40 | func (f *File) Close() error { 41 | return f.file.Close() 42 | } 43 | 44 | func (f *File) Name() string { 45 | return f.file.Name() 46 | } 47 | 48 | func (f *File) Stat() (os.FileInfo, error) { 49 | return f.file.Stat() 50 | } 51 | 52 | func (f *File) Sync() error { 53 | return nil 54 | } 55 | 56 | func (f *File) Truncate(size int64) error { 57 | return f.file.Truncate(size) 58 | } 59 | 60 | func (f *File) Read(b []byte) (n int, err error) { 61 | return f.file.Read(b) 62 | } 63 | 64 | // TODO 65 | func (f *File) ReadAt(b []byte, off int64) (n int, err error) { 66 | f.file.Seek(off, io.SeekStart) 67 | return f.file.Read(b) 68 | } 69 | 70 | // TODO 71 | func (f *File) Readdir(count int) (res []os.FileInfo, err error) { 72 | name, err := NormalizeDir(f.name, f.client) 73 | if err != nil { 74 | return []os.FileInfo{}, err 75 | 76 | } 77 | 78 | dirs, err := f.client.ReadDir(name) 79 | if len(dirs) > count && count > 0 { 80 | return dirs[0:count-1], err 81 | } 82 | return dirs, err 83 | } 84 | 85 | // TODO 86 | func (f *File) Readdirnames(n int) (names []string, err error) { 87 | dirs, err := f.Readdir(n) 88 | dirNames := []string{} 89 | 90 | if err != nil { 91 | return dirNames, err 92 | } 93 | 94 | for _, dir := range dirs { 95 | dirNames = append(dirNames, dir.Name()) 96 | } 97 | return dirNames, nil 98 | } 99 | 100 | func (f *File) Seek(offset int64, whence int) (int64, error) { 101 | return f.file.Seek(offset, whence) 102 | } 103 | 104 | func (f *File) Write(b []byte) (n int, err error) { 105 | return f.file.Write(b) 106 | } 107 | 108 | // TODO 109 | func (f *File) WriteAt(b []byte, off int64) (n int, err error) { 110 | return 0, nil 111 | } 112 | 113 | func (f *File) WriteString(s string) (ret int, err error) { 114 | return f.file.Write([]byte(s)) 115 | } 116 | -------------------------------------------------------------------------------- /transfer/progress_handler_test.go: -------------------------------------------------------------------------------- 1 | package transfer_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "time" 7 | "github.com/sandreas/graft/transfer" 8 | ) 9 | 10 | const ( 11 | defaultChunkSize = int64(1024 * 32) 12 | defaultReportInterval = 300 * time.Millisecond 13 | ) 14 | 15 | func TestEmptyFile(t *testing.T) { 16 | expect := assert.New(t) 17 | 18 | progressHandler := transfer.NewCopyProgressHandler(defaultChunkSize, defaultReportInterval) 19 | newChunkSize, message := progressHandler.Update(0, 0, defaultChunkSize, time.Now()) 20 | expect.Equal(defaultChunkSize, newChunkSize) 21 | expect.Equal("\r[====================>] 100.00%\n", message) 22 | } 23 | 24 | func TestNonEmptyFile(t *testing.T) { 25 | expect := assert.New(t) 26 | 27 | progressHandler := transfer.NewCopyProgressHandler(defaultChunkSize, defaultReportInterval) 28 | 29 | size := int64(1024 * 1024 * 5) 30 | 31 | layout := "2006-01-02T15:04:05.000Z" 32 | nowAsString := "2017-08-02T21:45:00.000Z" 33 | now, _ := time.Parse(layout, nowAsString) 34 | 35 | newChunkSize, message := progressHandler.Update(0, size, defaultChunkSize, now) 36 | expect.Equal(defaultChunkSize, newChunkSize) 37 | expect.Equal("\r[> ] 0.00%", message) 38 | 39 | nowAsString = "2017-08-02T21:45:00.333Z" 40 | now, _ = time.Parse(layout, nowAsString) 41 | transfered := int64(3 * 1024 * 1024) 42 | newChunkSize, message = progressHandler.Update(transfered, size, defaultChunkSize, now) 43 | expect.Equal(defaultChunkSize, newChunkSize) 44 | expect.Equal("\r[============> ] 60.00% 9.01MiB/s", message) 45 | 46 | nowAsString = "2017-08-02T21:45:00.334Z" 47 | now, _ = time.Parse(layout, nowAsString) 48 | transfered = int64(3*1024*1024 + 50) 49 | newChunkSize, message = progressHandler.Update(transfered, size, defaultChunkSize, now) 50 | expect.Equal(defaultChunkSize, newChunkSize) 51 | expect.Equal("", message) 52 | 53 | nowAsString = "2017-08-02T21:45:01.334Z" 54 | now, _ = time.Parse(layout, nowAsString) 55 | transfered = int64(4 * 1024 * 1024) 56 | newChunkSize, message = progressHandler.Update(transfered, size, defaultChunkSize, now) 57 | expect.Equal(defaultChunkSize, newChunkSize) 58 | expect.Equal("\r[================> ] 80.00% 1022.98KiB/s", message) 59 | 60 | nowAsString = "2017-08-02T21:45:01.734Z" 61 | now, _ = time.Parse(layout, nowAsString) 62 | transfered = size 63 | newChunkSize, message = progressHandler.Update(transfered, size, defaultChunkSize, now) 64 | expect.Equal(defaultChunkSize, newChunkSize) 65 | expect.Equal("\r[====================>] 100.00% 2.50MiB/s\n", message) 66 | } 67 | -------------------------------------------------------------------------------- /sftpd/vfshandler.go: -------------------------------------------------------------------------------- 1 | package sftpd 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "os" 7 | "github.com/pkg/sftp" 8 | "sync" 9 | ) 10 | 11 | type vfs struct { 12 | pathMap PathMapper 13 | pathMapLock sync.Mutex 14 | } 15 | 16 | func VfsHandler(mapper *PathMapper) sftp.Handlers { 17 | virtualFileSystem := &vfs{ 18 | pathMap: *mapper, 19 | } 20 | return sftp.Handlers{ 21 | FileGet: virtualFileSystem, 22 | FilePut: virtualFileSystem, 23 | FileCmd: virtualFileSystem, 24 | FileList: virtualFileSystem, 25 | } 26 | } 27 | 28 | func dumpSftpRequest(message string, r *sftp.Request) { 29 | log.Println(message, "Filepath: ", r.Filepath, ", Target: ", r.Target, ", Method: ", r.Method) 30 | } 31 | 32 | func (fs *vfs) Fileread(r *sftp.Request) (io.ReaderAt, error) { 33 | dumpSftpRequest("Fileread: ", r) 34 | fs.pathMapLock.Lock() 35 | defer fs.pathMapLock.Unlock() 36 | 37 | filePath, err := fs.pathMap.PathTo(r.Filepath) 38 | 39 | if err == nil { 40 | f, err := os.Open(filePath) 41 | if err != nil { 42 | defer f.Close() 43 | } 44 | return f, err 45 | } 46 | return nil, err 47 | } 48 | 49 | func (fs *vfs) Filewrite(r *sftp.Request) (io.WriterAt, error) { 50 | dumpSftpRequest("Filewrite (disabled): ", r) 51 | return nil, os.ErrInvalid 52 | } 53 | 54 | func (fs *vfs) Filecmd(r *sftp.Request) error { 55 | dumpSftpRequest("Filecmd (disabled): ", r) 56 | return os.ErrInvalid 57 | } 58 | 59 | type listerAt []os.FileInfo 60 | 61 | // Modeled after strings.Reader's ReadAt() implementation 62 | func (l listerAt) ListAt(ls []os.FileInfo, offset int64) (int, error) { 63 | var n int 64 | if offset >= int64(len(l)) { 65 | return 0, io.EOF 66 | } 67 | n = copy(ls, l[offset:]) 68 | if n < len(ls) { 69 | return n, io.EOF 70 | } 71 | return n, nil 72 | } 73 | 74 | 75 | func (fs *vfs) Filelist(r *sftp.Request) (sftp.ListerAt, error) { 76 | dumpSftpRequest("Fileinfo: ", r) 77 | fs.pathMapLock.Lock() 78 | defer fs.pathMapLock.Unlock() 79 | switch r.Method { 80 | case "List": 81 | listing, ok := fs.pathMap.List(r.Filepath) 82 | if !ok { 83 | return nil, os.ErrInvalid 84 | } 85 | 86 | statList := make([]os.FileInfo, len(listing)) 87 | for i, fileName := range listing { 88 | stat, err := fs.pathMap.Stat(fileName) 89 | if err != nil { 90 | log.Println("Could not stat file", fileName, err) 91 | continue 92 | } 93 | 94 | statList[i] = stat 95 | log.Println("Stat for file "+fileName+": isDir=>", stat.IsDir(), "size=>", stat.Size()) 96 | } 97 | return listerAt(statList), nil 98 | case "Stat": 99 | stat, err := fs.pathMap.Stat(r.Filepath) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return listerAt([]os.FileInfo{stat}), nil 104 | } 105 | return nil, os.ErrInvalid 106 | } 107 | -------------------------------------------------------------------------------- /transfer/progress_handler.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "time" 5 | "github.com/tears-of-noobs/bytefmt" 6 | "math" 7 | "strconv" 8 | "fmt" 9 | 10 | "strings" 11 | ) 12 | 13 | type CopyProgressHandler struct { 14 | //TransferProgressHandlerInterface 15 | bufferSize int64 16 | 17 | timerLastUpdate time.Time 18 | reportInterval time.Duration 19 | bytesLastUpdate int64 20 | 21 | } 22 | 23 | func NewCopyProgressHandler(bufSize int64, repInterval time.Duration) (*CopyProgressHandler) { 24 | return &CopyProgressHandler{ 25 | bufferSize: bufSize, 26 | reportInterval: repInterval, 27 | } 28 | } 29 | 30 | 31 | func (s *CopyProgressHandler) Update(bytesTransferred, size, chunkSize int64, now time.Time) (int64, string) { 32 | s.startTimer(bytesTransferred, 1 * int64(time.Second), now) 33 | shouldReport := true 34 | bytesPerSecond := float64(0) 35 | percent := float64(0) 36 | 37 | shouldReport, bytesPerSecond, percent = s.getReportStatus(bytesTransferred, size, now) 38 | 39 | // fmt.Printf("bytesPerSecond: %+v", bytesPerSecond) 40 | messageSuffix := "" 41 | 42 | if bytesTransferred == 0 { 43 | shouldReport = true 44 | } 45 | 46 | if bytesTransferred == size { 47 | shouldReport = true 48 | percent = 1 49 | messageSuffix = "\n" 50 | } 51 | 52 | if shouldReport { 53 | bandwidthOutput := " " + bytefmt.FormatBytes(bytesPerSecond, 2, true) + "/s" 54 | charCountWhenFullyTransmitted := 20 55 | progressChars := int(math.Floor(percent * float64(charCountWhenFullyTransmitted))) 56 | normalizedInt := percent * 100 57 | percentOutput := strconv.FormatFloat(normalizedInt, 'f', 2, 64) 58 | 59 | if bytesPerSecond == 0 { 60 | bandwidthOutput = "" 61 | } 62 | 63 | progressBar := fmt.Sprintf("[%-" + strconv.Itoa(charCountWhenFullyTransmitted + 1) + "s] " + percentOutput + "%%" + bandwidthOutput, strings.Repeat("=", progressChars) + ">") 64 | 65 | return chunkSize, "\r" + progressBar + messageSuffix 66 | } 67 | 68 | 69 | return chunkSize, "" 70 | } 71 | 72 | 73 | func (s *CopyProgressHandler) startTimer(bytesTransferred, interval int64, now time.Time) { 74 | if bytesTransferred == 0 { 75 | s.timerLastUpdate = now 76 | s.bytesLastUpdate = bytesTransferred 77 | } 78 | } 79 | 80 | func (s *CopyProgressHandler) getReportStatus(bytesTransferred, size int64, now time.Time) (bool, float64, float64) { 81 | nowNano := now.UnixNano() 82 | lastUpdateNano := s.timerLastUpdate.UnixNano() 83 | timeDiffNano := nowNano - lastUpdateNano 84 | timeDiffSeconds := float64(timeDiffNano) / float64(time.Second) 85 | 86 | if timeDiffNano >= s.reportInterval.Nanoseconds() { 87 | bytesDiff := bytesTransferred - s.bytesLastUpdate 88 | bytesPerSecond := float64(float64(bytesDiff) / float64(timeDiffSeconds)) 89 | percent := float64(bytesTransferred) / float64(size) 90 | 91 | s.bytesLastUpdate = bytesTransferred 92 | s.timerLastUpdate = now 93 | 94 | return true, bytesPerSecond, percent 95 | } 96 | 97 | return false, float64(0), float64(0) 98 | } 99 | -------------------------------------------------------------------------------- /file/compare/stitch.go: -------------------------------------------------------------------------------- 1 | package compare 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/spf13/afero" 7 | "errors" 8 | "bytes" 9 | "math" 10 | ) 11 | 12 | type Stitch struct { 13 | BufferSize int64 14 | SourceFileStat os.FileInfo 15 | DestinationFileStat os.FileInfo 16 | SourceFilePointer afero.File 17 | DestinationFilePointer afero.File 18 | 19 | isTransferCompleted bool 20 | } 21 | 22 | func NewStich(src afero.File, dst afero.File, bufferSize int64) (*Stitch, error) { 23 | stitchStrategy := &Stitch{ 24 | BufferSize: bufferSize, 25 | SourceFilePointer: src, 26 | DestinationFilePointer: dst, 27 | } 28 | 29 | var err error 30 | stitchStrategy.SourceFileStat, err = src.Stat() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | if err := stitchStrategy.initialize(); err != nil { 36 | return nil, err 37 | } 38 | 39 | return stitchStrategy, nil 40 | } 41 | func (s *Stitch) initialize() error { 42 | var err error 43 | s.DestinationFileStat, err = s.DestinationFilePointer.Stat() 44 | if os.IsNotExist(err) { 45 | s.isTransferCompleted = false 46 | return nil 47 | } 48 | 49 | if err != nil { 50 | return err 51 | } 52 | 53 | inSize := s.SourceFileStat.Size() 54 | outSize := s.DestinationFileStat.Size() 55 | 56 | if inSize < outSize { 57 | return errors.New("source is smaller than destination") 58 | } 59 | 60 | if outSize == 0 { 61 | return nil 62 | } 63 | 64 | if s.BufferSize > outSize { 65 | s.BufferSize = outSize 66 | } 67 | 68 | if err := s.ensureFileContentsEqual(0); err != nil { 69 | return err 70 | } 71 | 72 | if outSize <= s.BufferSize { 73 | s.isTransferCompleted = inSize == outSize 74 | return nil 75 | } 76 | 77 | backOffset := outSize - s.BufferSize 78 | shouldCheckMiddle := true 79 | if backOffset <= outSize/2 { 80 | backOffset = s.BufferSize 81 | shouldCheckMiddle = false 82 | } 83 | 84 | if err := s.ensureFileContentsEqual(backOffset); err != nil { 85 | return err 86 | } 87 | if !shouldCheckMiddle { 88 | s.isTransferCompleted = inSize == outSize 89 | return nil 90 | } 91 | 92 | middleOffset := int64(math.Floor(float64(outSize/2) - float64(s.BufferSize/2))) 93 | 94 | if err := s.ensureFileContentsEqual(middleOffset); err != nil { 95 | return err 96 | } 97 | 98 | s.isTransferCompleted = inSize == outSize 99 | return nil 100 | } 101 | 102 | func (s *Stitch) ensureFileContentsEqual(offset int64) error { 103 | fiBuf := make([]byte, s.BufferSize) 104 | _, err := s.SourceFilePointer.ReadAt(fiBuf, offset) 105 | 106 | if err != nil { 107 | return err 108 | } 109 | 110 | foBuf := make([]byte, s.BufferSize) 111 | _, err = s.DestinationFilePointer.ReadAt(foBuf, offset) 112 | 113 | if err != nil { 114 | return err 115 | } 116 | if ! bytes.Equal(fiBuf, foBuf) { 117 | return errors.New("source file does not match destination file") 118 | } 119 | return nil 120 | } 121 | 122 | func (s *Stitch) IsComplete() bool { 123 | return s.isTransferCompleted 124 | } 125 | -------------------------------------------------------------------------------- /filesystem/osfs_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package filesystem 4 | 5 | import ( 6 | 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/spf13/afero" 12 | "strings" 13 | ) 14 | 15 | type OsFs struct { 16 | afero.Fs 17 | } 18 | 19 | func NewOsFs() afero.Fs { 20 | return &OsFs{} 21 | } 22 | 23 | func (OsFs) Name() string { return "OsFs" } 24 | 25 | func (OsFs) Create(name string) (afero.File, error) { 26 | f, e := os.Create(makeAbsolute(name)) 27 | if f == nil { 28 | // while this looks strange, we need to return a bare nil (of type nil) not 29 | // a nil value of type *os.File or nil won't be nil 30 | return nil, e 31 | } 32 | return f, e 33 | } 34 | 35 | func (OsFs) Mkdir(name string, perm os.FileMode) error { 36 | return os.Mkdir(makeAbsolute(name), perm) 37 | } 38 | 39 | func (OsFs) MkdirAll(path string, perm os.FileMode) error { 40 | return os.MkdirAll(makeAbsolute(path), perm) 41 | } 42 | 43 | func (OsFs) Open(name string) (afero.File, error) { 44 | f, e := os.Open(makeAbsolute(name)) 45 | if f == nil { 46 | // while this looks strange, we need to return a bare nil (of type nil) not 47 | // a nil value of type *os.File or nil won't be nil 48 | return nil, e 49 | } 50 | return f, e 51 | } 52 | 53 | func (OsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { 54 | f, e := os.OpenFile(makeAbsolute(name), flag, perm) 55 | if f == nil { 56 | // while this looks strange, we need to return a bare nil (of type nil) not 57 | // a nil value of type *os.File or nil won't be nil 58 | return nil, e 59 | } 60 | return f, e 61 | } 62 | 63 | func (OsFs) Remove(name string) error { 64 | return os.Remove(makeAbsolute(name)) 65 | } 66 | 67 | func (OsFs) RemoveAll(path string) error { 68 | return os.RemoveAll(makeAbsolute(path)) 69 | } 70 | 71 | func (OsFs) Rename(oldname, newname string) error { 72 | return os.Rename(makeAbsolute(oldname), makeAbsolute(newname)) 73 | } 74 | 75 | func (OsFs) Stat(name string) (os.FileInfo, error) { 76 | return os.Stat(makeAbsolute(name)) 77 | } 78 | 79 | func (OsFs) Chmod(name string, mode os.FileMode) error { 80 | return os.Chmod(makeAbsolute(name), mode) 81 | } 82 | 83 | func (OsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { 84 | return os.Chtimes(makeAbsolute(name), atime, mtime) 85 | } 86 | 87 | 88 | // windows cannot handle long relative paths, so relative are converted to absolute paths by default 89 | func makeAbsolute(name string) string { 90 | absolutePath, err := filepath.Abs(name) 91 | if err == nil { 92 | if strings.HasPrefix(absolutePath, `\\?\UNC\`) || strings.HasPrefix(absolutePath, `\\?\`) { 93 | return absolutePath 94 | } 95 | 96 | if strings.HasPrefix(absolutePath, `\\`) { 97 | return strings.Replace(absolutePath, `\\`, `\\?\UNC\`, 1) 98 | } 99 | 100 | // os.MkdirAll fails on absolute paths, that do not exist, e.g. abs := "\\\\?\\D:\\test" 101 | // see https://github.com/golang/go/issues/22230 102 | // return `\\?\` + absolutePath 103 | return absolutePath 104 | } 105 | return name 106 | } 107 | 108 | func (fs *OsFs) Close() { 109 | // osfs does not need to be closed 110 | } 111 | -------------------------------------------------------------------------------- /sftpfs/file.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2015 Jerry Jacobs . 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package sftpfs 15 | 16 | import ( 17 | "github.com/pkg/sftp" 18 | "os" 19 | "io" 20 | ) 21 | 22 | type File struct { 23 | file *sftp.File 24 | client *sftp.Client 25 | name string 26 | } 27 | 28 | func FileOpen(s *sftp.Client, name string) (*File, error) { 29 | fd, err := s.Open(name) 30 | if err != nil { 31 | return &File{}, err 32 | } 33 | return &File{ 34 | file: fd, 35 | client: s, 36 | name: name, 37 | }, nil 38 | } 39 | 40 | func FileCreate(s *sftp.Client, name string) (*File, error) { 41 | fd, err := s.Create(name) 42 | if err != nil { 43 | return &File{}, err 44 | } 45 | return &File{ 46 | file: fd, 47 | client: s, 48 | name: name, 49 | }, nil 50 | } 51 | 52 | func (f *File) Close() error { 53 | return f.file.Close() 54 | } 55 | 56 | func (f *File) Name() string { 57 | return f.file.Name() 58 | } 59 | 60 | func (f *File) Stat() (os.FileInfo, error) { 61 | return f.file.Stat() 62 | } 63 | 64 | func (f *File) Sync() error { 65 | return nil 66 | } 67 | 68 | func (f *File) Truncate(size int64) error { 69 | return f.file.Truncate(size) 70 | } 71 | 72 | func (f *File) Read(b []byte) (n int, err error) { 73 | return f.file.Read(b) 74 | } 75 | 76 | // TODO 77 | func (f *File) ReadAt(b []byte, off int64) (n int, err error) { 78 | f.file.Seek(off, io.SeekStart) 79 | return f.file.Read(b) 80 | } 81 | 82 | // TODO 83 | func (f *File) Readdir(count int) (res []os.FileInfo, err error) { 84 | name, err := NormalizeDir(f.name, f.client) 85 | if err != nil { 86 | return []os.FileInfo{}, err 87 | 88 | } 89 | 90 | dirs, err := f.client.ReadDir(name) 91 | if len(dirs) > count && count > 0 { 92 | return dirs[0:count-1], err 93 | } 94 | return dirs, err 95 | } 96 | 97 | // TODO 98 | func (f *File) Readdirnames(n int) (names []string, err error) { 99 | dirs, err := f.Readdir(n) 100 | dirNames := []string{} 101 | 102 | if err != nil { 103 | return dirNames, err 104 | } 105 | 106 | for _, dir := range dirs { 107 | dirNames = append(dirNames, dir.Name()) 108 | } 109 | return dirNames, nil 110 | } 111 | 112 | func (f *File) Seek(offset int64, whence int) (int64, error) { 113 | return f.file.Seek(offset, whence) 114 | } 115 | 116 | func (f *File) Write(b []byte) (n int, err error) { 117 | return f.file.Write(b) 118 | } 119 | 120 | // TODO 121 | func (f *File) WriteAt(b []byte, off int64) (n int, err error) { 122 | return 0, nil 123 | } 124 | 125 | func (f *File) WriteString(s string) (ret int, err error) { 126 | return f.file.Write([]byte(s)) 127 | } 128 | -------------------------------------------------------------------------------- /sftpd/path_mapper.go: -------------------------------------------------------------------------------- 1 | package sftpd 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | type PathMapper struct { 12 | tree map[string][]string 13 | basePath string 14 | } 15 | 16 | func NewPathMapper(files []string, basePath string) *PathMapper { 17 | pathMapper := &PathMapper{ 18 | basePath: basePath, 19 | } 20 | pathMapper.normalizeBasePath() 21 | pathMapper.buildTree(files) 22 | return pathMapper 23 | } 24 | 25 | func (mapper *PathMapper) normalizeBasePath() { 26 | mapper.basePath = mapper.normalizePath(mapper.basePath) 27 | if mapper.basePath == "" { 28 | mapper.basePath = "." 29 | } 30 | } 31 | 32 | func (mapper *PathMapper) normalizePath(basePath string) string { 33 | basePath = filepath.ToSlash(basePath) 34 | basePath = strings.TrimPrefix(basePath, "./") 35 | if basePath == "." { 36 | return "" 37 | } 38 | return strings.TrimRight(basePath, "/") 39 | } 40 | 41 | func (mapper *PathMapper) List(key string) ([]string, bool) { 42 | normalizedKey := mapper.slashify(key) 43 | value, ok := mapper.tree[normalizedKey] 44 | return value, ok 45 | } 46 | 47 | func (mapper *PathMapper) PathTo(reference string) (string, error) { 48 | normalizedKey := mapper.slashify(reference) 49 | _, ok := mapper.tree[normalizedKey] 50 | if ! ok { 51 | return "", errors.New("PathTo " + reference + " not found") 52 | } 53 | 54 | path := filepath.FromSlash(mapper.normalizePath(mapper.basePath + normalizedKey)) 55 | if path == "" { 56 | path = "." 57 | } 58 | return path, nil 59 | } 60 | 61 | func (mapper *PathMapper) Stat(reference string) (os.FileInfo, error) { 62 | path, err := mapper.PathTo(reference) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return os.Stat(path) 68 | } 69 | 70 | func (mapper *PathMapper) slashify(path string) string { 71 | toSlash := filepath.ToSlash(path) 72 | trimmed := strings.TrimLeft(toSlash, "/") 73 | return "/" + trimmed 74 | } 75 | 76 | func (mapper *PathMapper) buildTree(matchingPaths []string) { 77 | mapper.tree = make(map[string][]string) 78 | 79 | sort.Strings(matchingPaths) 80 | 81 | for _, path := range matchingPaths { 82 | normalizedPath := mapper.normalizePath(path) 83 | key := mapper.slashify(strings.TrimPrefix(normalizedPath, mapper.basePath)) 84 | for { 85 | 86 | mapper.tree[key] = []string{} 87 | idx := strings.LastIndex(key, "/") 88 | key = key[0:idx] 89 | if key == "" { 90 | break 91 | } 92 | } 93 | mapper.tree["/"] = []string{} 94 | } 95 | 96 | for key := range mapper.tree { 97 | if key == "/" { 98 | continue 99 | } 100 | idx := strings.LastIndex(key, "/") 101 | dir := key[0:idx] 102 | if dir == "" { 103 | dir = "/" 104 | } 105 | mapper.tree[dir] = append(mapper.tree[dir], key) 106 | } 107 | 108 | for key := range mapper.tree { 109 | sort.Strings(mapper.tree[key]) 110 | } 111 | } 112 | 113 | func (mapper *PathMapper) normalizePathMapItem(path string) (string, string) { 114 | parentPath := mapper.normalizePath(filepath.Dir(path)) 115 | parentWithoutBaseDir := strings.TrimPrefix(parentPath, mapper.basePath) 116 | key := mapper.slashify(parentWithoutBaseDir) 117 | return key, parentPath 118 | } 119 | -------------------------------------------------------------------------------- /data/doc/DEVELOPER-NOTES.md: -------------------------------------------------------------------------------- 1 | # Things to do 2 | 3 | - switch mdns library to https://github.com/grandcat/zeroconf 4 | 5 | - global 6 | - implement --show-matches for copy, move and receive 7 | - Improve logging 8 | - Add --exec to find command 9 | - update documentation 10 | - Improve performance 11 | - Show a note if regex does not compile 12 | - graft archive command 13 | - find a device id to ensure that "move" is safe to do / otherwise use copy and delete afterwards 14 | - replace pathmapper with file_tree 15 | - close source fs 16 | - add matcher for mimetype (image/*, image/jpeg) 17 | - max-depth parameter (?) 18 | - hide progress?! 19 | - improve performance of huge amounts of small files 20 | - shouldStop return parameter for filesystem.Walk 21 | - limit-results parameter 22 | 23 | - copy 24 | - support copy strategy: ResumeSkipDifferent=default, ResumeReplaceDifferent (ReplaceAll, ReplaceExisting, SkipExisting) 25 | - compare-strategy: quick, hash, full 26 | 27 | - serve 28 | - supportmultiple mdns entries - switch mdns library to https://github.com/grandcat/zeroconf 29 | - Improve handling of huge amounts of files 30 | 31 | 32 | Possible improvements 33 | - update Matchers to use existing FileInfo for faster matching / use matcher.setFileInfo in FileMatcherInterface 34 | - calculate and show transfer speed 35 | - --verbose (ls -lah like output) 36 | - --files-only / --directories-only 37 | - javascript plugins? https://github.com/robertkrimen/otto 38 | - --hide-progress (for working like find) 39 | - improve progress-bar output (progress speed is not accurate enough) 40 | - sftp-server: 41 | filesystem watcher for sftp server (https://godoc.org/github.com/fsnotify/fsnotify) 42 | accept connections from specific ip: conn, e := listener.Accept() clientAddr := conn.RemoteAddr() if clientAddr 43 | - sftp client 44 | - mdns / bonjour client https://github.com/hashicorp/mdns 45 | - Input / Colors: https://github.com/dixonwille/wlog 46 | 47 | 48 | # Technology links 49 | 50 | ## big list of different libraries 51 | https://github.com/avelino/awesome-go#command-line 52 | 53 | ## command line parser 54 | cli-app: https://github.com/urfave/cli 55 | further info: https://nathanleclaire.com/blog/2014/08/31/why-codegangstas-cli-package-is-the-bomb-and-you-should-use-it/ 56 | nice: https://github.com/alecthomas/kingpin 57 | https://github.com/alexflint/go-arg 58 | 59 | ## File copy 60 | 61 | Bytewise copy: 62 | http://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file-using-golang 63 | http://stackoverflow.com/questions/20602131/io-writeseeker-and-io-readseeker-from-byte-or-file 64 | http://stackoverflow.com/questions/38631982/golang-file-seek-and-file-writeat-not-working-as-expected 65 | 66 | ## file times 67 | http://stackoverflow.com/questions/20875336/how-can-i-get-a-files-ctime-atime-mtime-and-change-them-using-golang 68 | 69 | ## Globbing 70 | https://www.reddit.com/r/golang/comments/41ulfq/glob_for_go_works_much_faster_than_regexp_on/ 71 | 72 | # Tutorials 73 | 74 | ## Organize project structure 75 | https://talks.golang.org/2014/organizeio.slide#22 76 | 77 | ## Unit Testing 78 | https://medium.com/@matryer/5-simple-tips-and-tricks-for-writing-unit-tests-in-golang-619653f90742#.mco6oq8iu 79 | 80 | ## Regex 81 | https://github.com/StefanSchroeder/Golang-Regex-Tutorial/blob/master/01-chapter2.markdown 82 | 83 | ## State of go 84 | https://talks.golang.org/2017/state-of-go.slide#21 -------------------------------------------------------------------------------- /pattern/functions.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "bytes" 5 | "regexp" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | func GlobToRegexString(glob string) string { 12 | var buffer bytes.Buffer 13 | r := strings.NewReader(glob) 14 | 15 | escape := false 16 | for { 17 | r, _, err := r.ReadRune() 18 | if err != nil { 19 | break 20 | } 21 | 22 | if escape { 23 | buffer.WriteRune(r) 24 | escape = false 25 | continue 26 | } 27 | 28 | if r == '\\' { 29 | buffer.WriteRune(r) 30 | escape = true 31 | continue 32 | } 33 | 34 | if r == '*' { 35 | buffer.WriteString(".*") 36 | continue 37 | } 38 | 39 | if r == '.' { 40 | buffer.WriteString("\\.") 41 | continue 42 | } 43 | 44 | buffer.WriteRune(r) 45 | } 46 | 47 | return buffer.String() 48 | } 49 | 50 | func StrToAge(t string, reference time.Time) (time.Time, error) { 51 | modifyPattern, err := regexp.Compile("^([+-]?[0-9]+)[\\s]*([a-zA-Z]+)$") 52 | if err != nil { 53 | return reference, err 54 | } 55 | 56 | if modifyPattern.MatchString(t) { 57 | submatches := modifyPattern.FindStringSubmatch(t) 58 | modifier, err := strconv.Atoi(submatches[1]) 59 | if err != nil { 60 | return reference, err 61 | } 62 | 63 | // age must be negated 64 | modifier *= -1 65 | 66 | unit := strings.ToLower(submatches[2]) 67 | 68 | if strings.HasPrefix(unit, "d") { 69 | return reference.AddDate(0, 0, modifier), nil 70 | } 71 | if strings.HasPrefix(unit, "w") { 72 | return reference.AddDate(0, 0, modifier*7), nil 73 | } 74 | if strings.HasPrefix(unit, "mon") { 75 | return reference.AddDate(0, modifier, 0), nil 76 | } 77 | if strings.HasPrefix(unit, "y") { 78 | return reference.AddDate(modifier, 0, 0), nil 79 | } 80 | 81 | if strings.HasPrefix(unit, "ns") { 82 | unit = "ns" 83 | } else if strings.HasPrefix(unit, "us") || strings.HasPrefix(unit, "µs") { 84 | unit = "us" 85 | } else if strings.HasPrefix(unit, "ms") { 86 | unit = "ms" 87 | } else if strings.HasPrefix(unit, "s") { 88 | unit = "s" 89 | } else if strings.HasPrefix(unit, "m") { 90 | unit = "m" 91 | } else if strings.HasPrefix(unit, "h") { 92 | unit = "h" 93 | } 94 | 95 | d, err := time.ParseDuration(strconv.Itoa(modifier) + unit) 96 | if err != nil { 97 | return reference, err 98 | } 99 | 100 | return reference.Add(d), nil 101 | } 102 | 103 | fixedPattern, err := regexp.Compile("^[0-9]{4}-[0-9]{2}-[0-9]{2}$") 104 | if fixedPattern.MatchString(t) { 105 | layout := "2006-01-02" 106 | return time.Parse(layout, t) 107 | } 108 | 109 | layout := "2006-01-02T15:04:05.000Z" 110 | return time.Parse(layout, t) 111 | } 112 | 113 | func StrToSize(sizeString string) (int64, error) { 114 | 115 | lower := strings.ToLower(sizeString) 116 | 117 | factor := int64(1) 118 | if strings.HasSuffix(lower, "k") { 119 | factor = 1024 120 | lower = strings.TrimSuffix(lower, "k") 121 | } else if strings.HasSuffix(lower, "m") { 122 | factor = 1024 * 1024 123 | lower = strings.TrimSuffix(lower, "m") 124 | } else if strings.HasSuffix(lower, "g") { 125 | factor = 1024 * 1024 * 1024 126 | lower = strings.TrimSuffix(lower, "g") 127 | } else if strings.HasSuffix(lower, "t") { 128 | factor = 1024 * 1024 * 1024 * 1024 129 | lower = strings.TrimSuffix(lower, "t") 130 | } 131 | 132 | ret, err := strconv.Atoi(lower) 133 | 134 | return int64(ret) * factor, err 135 | } 136 | 137 | func BuildMatchList(sourcePattern *regexp.Regexp, subject string) []string { 138 | list := make([]string, 0) 139 | sourcePattern.ReplaceAllStringFunc(subject, func(m string) string { 140 | parts := sourcePattern.FindStringSubmatch(m) 141 | i := 1 142 | for range parts[1:] { 143 | list = append(list, parts[i]) 144 | i++ 145 | 146 | } 147 | return m 148 | }) 149 | return list 150 | } 151 | -------------------------------------------------------------------------------- /pattern/functions_test.go: -------------------------------------------------------------------------------- 1 | package pattern_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "time" 7 | "regexp" 8 | "github.com/sandreas/graft/pattern" 9 | ) 10 | 11 | func TestGlobToRegex(t *testing.T) { 12 | expect := assert.New(t) 13 | expect.Equal(".*\\.jpg", pattern.GlobToRegexString("*.jpg")) 14 | expect.Equal("star-file-\\*\\.jpg", pattern.GlobToRegexString("star-file-\\*.jpg")) 15 | expect.Equal("test\\.(jpg|png)", pattern.GlobToRegexString("test.(jpg|png)")) 16 | expect.Equal("test\\.{1,}", pattern.GlobToRegexString("test.{1,}")) 17 | expect.Equal("fixtures\\(\\..*)", pattern.GlobToRegexString("fixtures\\(.*)")) 18 | } 19 | 20 | func TestStrToSize(t *testing.T) { 21 | expect := assert.New(t) 22 | 23 | size, err := pattern.StrToSize("1080") 24 | expect.Equal(size, int64(1080)) 25 | expect.NoError(err) 26 | 27 | size, err = pattern.StrToSize("1K") 28 | expect.Equal(size, int64(1024)) 29 | expect.NoError(err) 30 | 31 | size, err = pattern.StrToSize("1M") 32 | expect.Equal(size, int64(1024*1024)) 33 | expect.NoError(err) 34 | 35 | size, err = pattern.StrToSize("1G") 36 | expect.Equal(size, int64(1024*1024*1024)) 37 | expect.NoError(err) 38 | 39 | size, err = pattern.StrToSize("1T") 40 | expect.Equal(size, int64(1024*1024*1024*1024)) 41 | expect.NoError(err) 42 | 43 | size, err = pattern.StrToSize("INVALID") 44 | expect.Equal(size, int64(0)) 45 | expect.Error(err) 46 | } 47 | 48 | func TestStrToAge(t *testing.T) { 49 | expect := assert.New(t) 50 | 51 | layout := "2006-01-02T15:04:05.000Z" 52 | str := "2014-11-12T11:45:26.371Z" 53 | referenceTime, _ := time.Parse(layout, str) 54 | 55 | actualTime, _ := pattern.StrToAge(str, time.Now()) 56 | expectedTime := referenceTime 57 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 58 | 59 | tstr := "2014-11-12" 60 | actualTime, _ = pattern.StrToAge(tstr, time.Now()) 61 | expectedTime, _ = time.Parse("2006-01-02", tstr) 62 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 63 | 64 | actualTime, _ = pattern.StrToAge("10 s", referenceTime) 65 | expectedTime = referenceTime.Add(time.Second * -time.Duration(10)) 66 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 67 | 68 | actualTime, _ = pattern.StrToAge("10m", referenceTime) 69 | expectedTime = referenceTime.Add(time.Minute * -time.Duration(10)) 70 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 71 | 72 | actualTime, _ = pattern.StrToAge("10h", referenceTime) 73 | expectedTime = referenceTime.Add(time.Hour * -time.Duration(10)) 74 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 75 | 76 | actualTime, _ = pattern.StrToAge("1 day", referenceTime) 77 | expectedTime = referenceTime.AddDate(0, 0, -1) 78 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 79 | 80 | actualTime, _ = pattern.StrToAge("-5 days", referenceTime) 81 | expectedTime = referenceTime.AddDate(0, 0, 5) 82 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 83 | 84 | actualTime, _ = pattern.StrToAge("1 week", referenceTime) 85 | expectedTime = referenceTime.AddDate(0, 0, -7) 86 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 87 | 88 | actualTime, _ = pattern.StrToAge("3 months", referenceTime) 89 | expectedTime = referenceTime.AddDate(0, -3, 0) 90 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 91 | 92 | actualTime, _ = pattern.StrToAge("2 years", referenceTime) 93 | expectedTime = referenceTime.AddDate(-2, 0, 0) 94 | expect.Equal(expectedTime.UnixNano(), actualTime.UnixNano()) 95 | } 96 | 97 | func TestBuildMatchList(t *testing.T) { 98 | expect := assert.New(t) 99 | compiled, _ := regexp.Compile("data/fixtures/global/(.*)(\\.txt)$") 100 | 101 | list := pattern.BuildMatchList(compiled, "data/fixtures/global/documents (2010)/document (2010).txt") 102 | 103 | 104 | expect.Equal(2, len(list)) 105 | expect.Equal("documents (2010)/document (2010)", list[0]) 106 | expect.Equal(".txt", list[1]) 107 | } -------------------------------------------------------------------------------- /filesystem/sftpfs.go: -------------------------------------------------------------------------------- 1 | package filesystem 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | "fmt" 8 | 9 | "github.com/pkg/sftp" 10 | "github.com/spf13/afero" 11 | ) 12 | 13 | const ( 14 | NameSftpfs = "sftpfs" 15 | ) 16 | 17 | // Fs is a afero.Fs implementation that uses functions provided by the sftp package. 18 | // 19 | // For details in any method, check the documentation of the sftp package 20 | // (github.com/pkg/sftp). 21 | type SftpFs struct { 22 | // *io.Closer 23 | afero.Fs 24 | client *sftp.Client 25 | Context *SftpFsContext 26 | } 27 | 28 | func NewSftpFs(host string, port int, username, password string) (afero.Fs, error) { 29 | hostAndPort := fmt.Sprintf("%s:%d", host, port) 30 | ctx, err := NewSftpFsContext(username, password, hostAndPort) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | fs := &SftpFs{ 36 | client: ctx.SftpClient, 37 | Context: ctx, 38 | } 39 | _, err = fs.client.Getwd() 40 | if err != nil { 41 | return nil, err 42 | } 43 | return fs, nil 44 | 45 | } 46 | 47 | func NormalizeDir(name string, client *sftp.Client) (string, error) { 48 | if name == "." { 49 | return client.Getwd() 50 | } 51 | return name, nil 52 | } 53 | 54 | func (s SftpFs) Name() string { return NameSftpfs } 55 | 56 | func (s SftpFs) Create(name string) (afero.File, error) { 57 | return FileCreate(s.client, name) 58 | } 59 | 60 | func (s SftpFs) Mkdir(name string, perm os.FileMode) error { 61 | err := s.client.Mkdir(name) 62 | if err != nil { 63 | return err 64 | } 65 | return s.client.Chmod(name, perm) 66 | } 67 | 68 | func (s SftpFs) MkdirAll(path string, perm os.FileMode) error { 69 | // Fast path: if we can tell whether path is a directory or file, stop with success or error. 70 | dir, err := s.Stat(path) 71 | if err == nil { 72 | if dir.IsDir() { 73 | return nil 74 | } 75 | return err 76 | } 77 | 78 | // Slow path: make sure parent exists and then call Mkdir for path. 79 | i := len(path) 80 | for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. 81 | i-- 82 | } 83 | 84 | j := i 85 | for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. 86 | j-- 87 | } 88 | 89 | if j > 1 { 90 | // Create parent 91 | err = s.MkdirAll(path[0:j-1], perm) 92 | if err != nil { 93 | return err 94 | } 95 | } 96 | 97 | // Parent now exists; invoke Mkdir and use its result. 98 | err = s.Mkdir(path, perm) 99 | if err != nil { 100 | // Handle arguments like "foo/." by 101 | // double-checking that directory doesn't exist. 102 | dir, err1 := s.Lstat(path) 103 | if err1 == nil && dir.IsDir() { 104 | return nil 105 | } 106 | return err 107 | } 108 | return nil 109 | } 110 | 111 | func (s SftpFs) Open(name string) (afero.File, error) { 112 | return FileOpen(s.client, name) 113 | } 114 | 115 | // changed!!! 116 | func (s SftpFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { 117 | // return s.client.OpenFile(name, flag) 118 | return FileOpen(s.client, name) 119 | } 120 | 121 | func (s SftpFs) Remove(name string) error { 122 | return s.client.Remove(name) 123 | } 124 | 125 | func (s SftpFs) RemoveAll(path string) error { 126 | // TODO have a look at os.RemoveAll 127 | // https://github.com/golang/go/blob/master/src/os/path.go#L66 128 | return nil 129 | } 130 | 131 | func (s SftpFs) Rename(oldname, newname string) error { 132 | return s.client.Rename(oldname, newname) 133 | } 134 | 135 | func (s SftpFs) Stat(name string) (os.FileInfo, error) { 136 | name, err := NormalizeDir(name, s.client) 137 | if err != nil { 138 | return nil, err 139 | } 140 | stat, err := s.client.Stat(name) 141 | return stat, err 142 | } 143 | 144 | func (s SftpFs) Lstat(p string) (os.FileInfo, error) { 145 | return s.client.Lstat(p) 146 | } 147 | 148 | func (s SftpFs) Chmod(name string, mode os.FileMode) error { 149 | return s.client.Chmod(name, mode) 150 | } 151 | 152 | func (s SftpFs) Chtimes(name string, atime time.Time, mtime time.Time) error { 153 | return s.client.Chtimes(name, atime, mtime) 154 | } 155 | 156 | func (s SftpFs) Close() { 157 | s.Context.Close() 158 | } 159 | -------------------------------------------------------------------------------- /sftpfs/sftp.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2015 Jerry Jacobs . 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package sftpfs 15 | 16 | import ( 17 | "os" 18 | "time" 19 | "github.com/pkg/sftp" 20 | "github.com/spf13/afero" 21 | "log" 22 | ) 23 | 24 | // Fs is a afero.Fs implementation that uses functions provided by the sftp package. 25 | // 26 | // For details in any method, check the documentation of the sftp package 27 | // (github.com/pkg/sftp). 28 | type Fs struct { 29 | client *sftp.Client 30 | } 31 | 32 | func NormalizeDir(name string, client *sftp.Client) (string, error) { 33 | if name == "." { 34 | return client.Getwd() 35 | } 36 | return name, nil 37 | } 38 | 39 | func New(client *sftp.Client) afero.Fs { 40 | 41 | fs := &Fs{client: client} 42 | stat, err := fs.client.Getwd() 43 | log.Printf("stat: %#v, err: %#v", stat, err) 44 | return fs 45 | 46 | } 47 | 48 | func (s Fs) Name() string { return "sftpfs" } 49 | 50 | func (s Fs) Create(name string) (afero.File, error) { 51 | return FileCreate(s.client, name) 52 | } 53 | 54 | func (s Fs) Mkdir(name string, perm os.FileMode) error { 55 | err := s.client.Mkdir(name) 56 | if err != nil { 57 | return err 58 | } 59 | return s.client.Chmod(name, perm) 60 | } 61 | 62 | func (s Fs) MkdirAll(path string, perm os.FileMode) error { 63 | // Fast path: if we can tell whether path is a directory or file, stop with success or error. 64 | dir, err := s.Stat(path) 65 | if err == nil { 66 | if dir.IsDir() { 67 | return nil 68 | } 69 | return err 70 | } 71 | 72 | // Slow path: make sure parent exists and then call Mkdir for path. 73 | i := len(path) 74 | for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. 75 | i-- 76 | } 77 | 78 | j := i 79 | for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. 80 | j-- 81 | } 82 | 83 | if j > 1 { 84 | // Create parent 85 | err = s.MkdirAll(path[0:j-1], perm) 86 | if err != nil { 87 | return err 88 | } 89 | } 90 | 91 | // Parent now exists; invoke Mkdir and use its result. 92 | err = s.Mkdir(path, perm) 93 | if err != nil { 94 | // Handle arguments like "foo/." by 95 | // double-checking that directory doesn't exist. 96 | dir, err1 := s.Lstat(path) 97 | if err1 == nil && dir.IsDir() { 98 | return nil 99 | } 100 | return err 101 | } 102 | return nil 103 | } 104 | 105 | func (s Fs) Open(name string) (afero.File, error) { 106 | return FileOpen(s.client, name) 107 | } 108 | 109 | func (s Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { 110 | return nil, nil 111 | } 112 | 113 | func (s Fs) Remove(name string) error { 114 | return s.client.Remove(name) 115 | } 116 | 117 | func (s Fs) RemoveAll(path string) error { 118 | // TODO have a look at os.RemoveAll 119 | // https://github.com/golang/go/blob/master/src/os/path.go#L66 120 | return nil 121 | } 122 | 123 | func (s Fs) Rename(oldname, newname string) error { 124 | return s.client.Rename(oldname, newname) 125 | } 126 | 127 | func (s Fs) Stat(name string) (os.FileInfo, error) { 128 | name, err := NormalizeDir(name, s.client) 129 | if err != nil { 130 | return nil, err 131 | } 132 | stat, err := s.client.Stat(name) 133 | return stat, err 134 | } 135 | 136 | func (s Fs) Lstat(p string) (os.FileInfo, error) { 137 | return s.client.Lstat(p) 138 | } 139 | 140 | func (s Fs) Chmod(name string, mode os.FileMode) error { 141 | return s.client.Chmod(name, mode) 142 | } 143 | 144 | func (s Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { 145 | return s.client.Chtimes(name, atime, mtime) 146 | } 147 | 148 | -------------------------------------------------------------------------------- /action/serve.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "os" 7 | 8 | "os/signal" 9 | "syscall" 10 | 11 | "strconv" 12 | 13 | "crypto/rand" 14 | "fmt" 15 | "path/filepath" 16 | 17 | "github.com/grandcat/zeroconf" 18 | "github.com/sandreas/graft/apputils" 19 | "github.com/sandreas/graft/sftpd" 20 | "github.com/urfave/cli" 21 | ) 22 | 23 | type ServeAction struct { 24 | AbstractAction 25 | } 26 | 27 | func (action *ServeAction) Execute(c *cli.Context) error { 28 | action.PrepareExecution(c, 1, "*") 29 | log.Printf("serve") 30 | if err := action.locateSourceFiles(); err != nil { 31 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 32 | } 33 | if err := action.ServeFoundFiles(); err != nil { 34 | return cli.NewExitError(err.Error(), ErrorStartingServer) 35 | } 36 | return nil 37 | } 38 | 39 | func pseudoUuid() (uuid string) { 40 | 41 | b := make([]byte, 16) 42 | _, err := rand.Read(b) 43 | if err != nil { 44 | fmt.Println("Error: ", err) 45 | return 46 | } 47 | 48 | uuid = fmt.Sprintf("%X-%X-%X-%X-%X", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]) 49 | 50 | return 51 | } 52 | 53 | func (action *ServeAction) ServeFoundFiles() error { 54 | var err error 55 | var homeDir string 56 | 57 | if len(action.locator.SourceFiles) == 0 && !action.CliParameters.Force { 58 | action.suppressablePrintf("\nNo matching files found, server does not need to be started - use force to start server anyway\n") 59 | return nil 60 | } 61 | 62 | if homeDir, err = action.createHomeDirectoryIfNotExists(); err != nil { 63 | return err 64 | } 65 | 66 | if _, err = action.sourceFs.Stat(action.sourcePattern.Path); err != nil { 67 | return err 68 | } 69 | 70 | basePath := filepath.Dir(action.sourcePattern.Path) 71 | pathMapper := sftpd.NewPathMapper(action.locator.SourceFiles, basePath) 72 | listenAddress := "0.0.0.0" 73 | outboundIp := "localhost" 74 | if action.CliContext.String("host") != "" { 75 | listenAddress = action.CliContext.String("host") 76 | outboundIp = listenAddress 77 | } else { 78 | outboundIp, err = apputils.GetOutboundIpAsString("localhost", net.Dial) 79 | if err != nil { 80 | log.Printf("Error on GetOutboundIpAsString: %v", err) 81 | } 82 | } 83 | 84 | username := action.CliContext.String("username") 85 | password := action.CliContext.String("password") 86 | port := action.CliContext.Int("port") 87 | 88 | if password == "" { 89 | password, err = action.promptPassword("\nWhich password shall be requested for user " + username + " on login?") 90 | if err != nil { 91 | return err 92 | } 93 | } 94 | 95 | if !action.CliContext.Bool("no-zeroconf") { 96 | action.suppressablePrintf("Publishing service via mdns: active\n") 97 | 98 | uuid := pseudoUuid() 99 | port := action.CliParameters.Port 100 | 101 | name := "graft-sftp-server-" + uuid + "_" + outboundIp + ":" + strconv.Itoa(port) 102 | service := "_graft._tcp" 103 | domain := "local." 104 | 105 | server, err := zeroconf.Register(name, service, domain, port, []string{"txtv=0.2", "domain=" + domain, "ip=" + outboundIp}, nil) 106 | if err != nil { 107 | panic(err) 108 | } 109 | defer server.Shutdown() 110 | log.Println("Published service:") 111 | log.Println("- Name:", name) 112 | log.Println("- Type:", service) 113 | log.Println("- Domain:", domain) 114 | log.Println("- Port:", port) 115 | 116 | } 117 | 118 | go func() { 119 | action.suppressablePrintf("Running sftp server, login as %s@%s:%d\nPress CTRL+C to stop\n", username, outboundIp, port) 120 | // sftpListener, err := sftpd.NewSimpleSftpServer(homeDir, listenAddress, port, username, password, pathMapper) 121 | _, err := sftpd.NewSimpleSftpServer(homeDir, listenAddress, port, username, password, pathMapper) 122 | if err != nil { 123 | log.Printf("Error starting sftp server: " + err.Error()) 124 | } 125 | }() 126 | 127 | // Clean exit. 128 | sig := make(chan os.Signal, 1) 129 | signal.Notify(sig, os.Interrupt, syscall.SIGTERM) 130 | // Timeout timer 131 | 132 | select { 133 | case <-sig: 134 | // Exit by user 135 | } 136 | 137 | // TODO Ctrl+C handling 138 | //handler := make(chan os.Signal, 1) 139 | //signal.Notify(handler, os.Interrupt) 140 | //for sig := range handler { 141 | // if sig == os.Interrupt { 142 | // //bonjourListener.Shutdown() 143 | // //sftpListener.Close() 144 | // wg.Done() 145 | // time.Sleep(1e9) 146 | // break 147 | // } 148 | //} 149 | 150 | return nil 151 | } 152 | -------------------------------------------------------------------------------- /graft.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/sandreas/graft/action" 7 | "github.com/urfave/cli" 8 | ) 9 | 10 | func main() { 11 | globalFlags := []cli.Flag{ 12 | cli.BoolFlag{Name: "quiet, q", Usage: "do not show any output"}, // does quiet make sense in find? 13 | cli.BoolFlag{Name: "force, f", Usage: "force the requested action - even if it might be not a good idea"}, // does force make sense in find? 14 | cli.BoolFlag{Name: "debug", Usage: "debug mode with logging to Stdout and into $HOME/.graft/application.log"}, 15 | cli.BoolFlag{Name: "regex", Usage: "use a real regex instead of glob patterns (e.g. src/.*\\.jpg)"}, 16 | cli.BoolFlag{Name: "case-sensitive", Usage: "be case sensitive when matching files and folders"}, 17 | cli.StringFlag{Name: "max-age", Usage: "maximum age (e.g. 2d / 8w / 2016-12-24 / etc. )"}, 18 | cli.StringFlag{Name: "min-age", Usage: "minimum age (e.g. 2d / 8w / 2016-12-24 / etc. )"}, 19 | cli.StringFlag{Name: "max-size", Usage: "maximum size in bytes or format string (e.g. 2G / 8M / 1000K etc. )"}, 20 | cli.StringFlag{Name: "min-size", Usage: "minimum size in bytes or format string (e.g. 2G / 8M / 1000K etc. )"}, 21 | cli.StringFlag{Name: "type", Usage: "only match items with given type (--type=f for files, --type=d for directories)"}, 22 | cli.StringFlag{Name: "export-to", Usage: "export found matches to a text file - one line per item (can also be used as save cache for large scans)"}, 23 | cli.StringFlag{Name: "files-from", Usage: "import found matches from file - one line per item (can also be used as load cache for large scans)"}, 24 | } 25 | 26 | networkFlags := []cli.Flag{ 27 | cli.StringFlag{Name: "host", Usage: "Specify the hostname for the server (client mode only)"}, 28 | cli.IntFlag{Name: "port", Usage: "Specifiy server port (used for server- and client mode)", Value: 2022}, 29 | cli.StringFlag{Name: "username", Usage: "Specify server username (used in server- and client mode)", Value: "graft"}, 30 | cli.StringFlag{Name: "password", Usage: "Specify server password (used for server- and client mode)"}, 31 | } 32 | 33 | findFlags := []cli.Flag{ 34 | cli.BoolFlag{Name: "show-matches", Usage: "do not show matches for search pattern ($1=filename)"}, 35 | cli.BoolFlag{Name: "client", Usage: "client mode - act as sftp client and search files remotely instead of local search"}, 36 | } 37 | 38 | serveFlags := []cli.Flag{ 39 | cli.BoolFlag{Name: "no-zeroconf", Usage: "do not use mdns/zeroconf to publish multicast sftp server (graft receive will not work without parameters)"}, 40 | } 41 | 42 | dryRunFlags := []cli.Flag{ 43 | cli.BoolFlag{Name: "dry-run", Usage: "simulation mode - shows output but files remain unaffected"}, 44 | } 45 | 46 | transferFlags := []cli.Flag{ 47 | cli.BoolFlag{Name: "times", Usage: "transfer source modify times to destination"}, 48 | } 49 | 50 | app := cli.NewApp() 51 | app.Name = "graft" 52 | app.Version = "0.2" 53 | app.Usage = "find, copy and serve files" 54 | 55 | app.Commands = []cli.Command{ 56 | { 57 | Name: "find", Aliases: []string{"f"}, Action: action.NewActionFactory("find").Execute, 58 | Usage: "find files", 59 | Flags:mergeFlags(globalFlags, networkFlags, findFlags), 60 | }, 61 | { 62 | Name: "serve", Aliases: []string{"s"}, Action: action.NewActionFactory("serve").Execute, 63 | Usage: "serve files via sftp server", 64 | Flags: mergeFlags(globalFlags, networkFlags, serveFlags), 65 | }, 66 | { 67 | Name: "copy", Aliases: []string{"c", "cp"}, Action: action.NewActionFactory("copy").Execute, 68 | Usage: "copy files from a source to a destination", 69 | Flags: mergeFlags(globalFlags, transferFlags, dryRunFlags), 70 | }, 71 | { 72 | Name: "move", Aliases: []string{"m", "mv"}, Action: action.NewActionFactory("move").Execute, 73 | Usage: "move files from a source to a destination", 74 | Flags: mergeFlags(globalFlags, dryRunFlags, transferFlags), 75 | }, 76 | { 77 | Name: "delete", Aliases: []string{"d", "rm"}, Action: action.NewActionFactory("delete").Execute, 78 | Usage: "delete files recursively", 79 | Flags: mergeFlags(globalFlags, dryRunFlags), 80 | }, 81 | { 82 | Name: "receive", Aliases: []string{"r"}, Action: action.NewActionFactory("receive").Execute, 83 | Usage: "receive files from a graft server", 84 | Flags: mergeFlags(globalFlags, dryRunFlags, transferFlags, networkFlags), 85 | }, 86 | } 87 | 88 | app.Run(os.Args) 89 | } 90 | 91 | func mergeFlags(flagsToMerge ...[]cli.Flag) []cli.Flag { 92 | mergedFlags := []cli.Flag{} 93 | for _,flags := range flagsToMerge { 94 | mergedFlags = append(mergedFlags, flags...) 95 | } 96 | return mergedFlags 97 | } 98 | -------------------------------------------------------------------------------- /sftpd/sftpd.go: -------------------------------------------------------------------------------- 1 | package sftpd 2 | 3 | import ( 4 | "golang.org/x/crypto/ssh" 5 | "log" 6 | "net" 7 | "fmt" 8 | "os" 9 | "crypto/subtle" 10 | "io/ioutil" 11 | "crypto/rsa" 12 | "encoding/pem" 13 | "crypto/x509" 14 | "io" 15 | "crypto/rand" 16 | "github.com/pkg/sftp" 17 | "strconv" 18 | ) 19 | 20 | func NewSimpleSftpServer(homePath, listenAddress string, listenPort int, username, password string, pathMapper *PathMapper) (net.Listener, error) { 21 | config := &ssh.ServerConfig{ 22 | PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { 23 | log.Printf("Login: %s\n", c.User()) 24 | if subtle.ConstantTimeCompare([]byte(username), []byte(c.User())) == 1 && subtle.ConstantTimeCompare(pass, []byte(password)) == 1 { 25 | return nil, nil 26 | } 27 | return nil, fmt.Errorf("password rejected for %q", c.User()) 28 | }, 29 | } 30 | 31 | generateKeysIfNotExist(homePath) 32 | 33 | privateBytes, err := ioutil.ReadFile(homePath + "/id_rsa") 34 | if err != nil { 35 | log.Fatal("Failed to load private key", err) 36 | return nil, err 37 | } 38 | private, err := ssh.ParsePrivateKey(privateBytes) 39 | if err != nil { 40 | log.Fatal("Failed to parse private key", err) 41 | return nil, err 42 | } 43 | config.AddHostKey(private) 44 | 45 | listener, err := net.Listen("tcp", listenAddress+":"+strconv.Itoa(listenPort)) 46 | if err != nil { 47 | log.Fatal("failed to listen for connection", err) 48 | return nil, err 49 | } 50 | log.Printf("Listening on %v\n", listener.Addr()) 51 | 52 | for { 53 | conn, e := listener.Accept() 54 | 55 | if e != nil { 56 | os.Exit(2) 57 | } 58 | go HandleConn(conn, config, pathMapper) 59 | } 60 | } 61 | 62 | func HandleConn(conn net.Conn, config *ssh.ServerConfig, pathMapper *PathMapper) { 63 | defer conn.Close() 64 | e := handleConn(conn, config, pathMapper) 65 | if e != nil { 66 | log.Println("sftpd connection errored:", e) 67 | } 68 | } 69 | func handleConn(conn net.Conn, config *ssh.ServerConfig, pathMapper *PathMapper) error { 70 | sconn, chans, reqs, e := ssh.NewServerConn(conn, config) 71 | if e != nil { 72 | return e 73 | } 74 | defer sconn.Close() 75 | 76 | // The incoming Request channel must be serviced. 77 | log.Println( "login detected:", sconn.User()) 78 | 79 | // The incoming Request channel must be serviced. 80 | go ssh.DiscardRequests(reqs) 81 | 82 | // Service the incoming Channel channel. 83 | for newChannel := range chans { 84 | if newChannel.ChannelType() != "session" { 85 | newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") 86 | continue 87 | } 88 | channel, requests, err := newChannel.Accept() 89 | if err != nil { 90 | return err 91 | } 92 | 93 | go func(in <-chan *ssh.Request) { 94 | for req := range in { 95 | log.Printf( "Request: %v\n", req.Type) 96 | ok := false 97 | switch req.Type { 98 | case "subsystem": 99 | log.Printf( "Subsystem: %s\n", req.Payload[4:]) 100 | if string(req.Payload[4:]) == "sftp" { 101 | ok = true 102 | } 103 | } 104 | log.Printf( " - accepted: %v\n", ok) 105 | req.Reply(ok, nil) 106 | } 107 | }(requests) 108 | 109 | 110 | root := VfsHandler(pathMapper) 111 | server := sftp.NewRequestServer(channel, root) 112 | if err := server.Serve(); err == io.EOF { 113 | server.Close() 114 | log.Print("sftp client exited session.") 115 | } else if err != nil { 116 | log.Fatal("sftp server completed with error:", err) 117 | } 118 | 119 | } 120 | return nil 121 | } 122 | 123 | 124 | 125 | 126 | func generateKeysIfNotExist(homeDir string) { 127 | 128 | privateKeyFile := homeDir + "/id_rsa" 129 | publicKeyFile := homeDir + "/id_rsa.pub" 130 | 131 | if _, err := os.Stat(privateKeyFile); os.IsNotExist(err) { 132 | makeSSHKeyPair(publicKeyFile, privateKeyFile) 133 | } 134 | } 135 | 136 | func makeSSHKeyPair(pubKeyPath, privateKeyPath string) error { 137 | 138 | privateKey, err := rsa.GenerateKey(rand.Reader, 1024) 139 | if err != nil { 140 | return err 141 | } 142 | 143 | // generate and write private key as PEM 144 | privateKeyFile, err := os.Create(privateKeyPath) 145 | defer privateKeyFile.Close() 146 | if err != nil { 147 | return err 148 | } 149 | privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)} 150 | if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil { 151 | return err 152 | } 153 | 154 | // generate and write public key 155 | pub, err := ssh.NewPublicKey(&privateKey.PublicKey) 156 | if err != nil { 157 | return err 158 | } 159 | return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0655) 160 | } -------------------------------------------------------------------------------- /file/compare/stitch_test.go: -------------------------------------------------------------------------------- 1 | package compare_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/sandreas/graft/file/compare" 7 | "github.com/spf13/afero" 8 | "github.com/stretchr/testify/assert" 9 | "os" 10 | ) 11 | 12 | func prepareFileSystem() afero.Fs { 13 | appFS := afero.NewMemMapFs() 14 | appFS.Mkdir("src", 0644) 15 | appFS.Mkdir("dst", 0644) 16 | 17 | // not existing 18 | afero.WriteFile(appFS, "file1-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 19 | 20 | // not resumable - destination bigger 21 | afero.WriteFile(appFS, "file2-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 22 | afero.WriteFile(appFS, "file2-dst.txt", []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), 0755) 23 | 24 | //// already completed 25 | afero.WriteFile(appFS, "file3-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 26 | afero.WriteFile(appFS, "file3-dst.txt", []byte("0123456789012345678901234567890123456789"), 0755) 27 | // 28 | // resumable 29 | afero.WriteFile(appFS, "file4-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 30 | afero.WriteFile(appFS, "file4-dst.txt", []byte("0123456789012345"), 0755) 31 | 32 | // not resumable - diff at start 33 | afero.WriteFile(appFS, "file5-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 34 | afero.WriteFile(appFS, "file5-dst.txt", []byte("a123456789012345678901234567"), 0755) 35 | // 36 | // not resumable - diff at end 37 | afero.WriteFile(appFS, "file6-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 38 | afero.WriteFile(appFS, "file6-dst.txt", []byte("0123456789012345678a"), 0755) 39 | 40 | // not resumable - diff at middle 41 | afero.WriteFile(appFS, "file7-src.txt", []byte("0123456789012345678901234567890123456789"), 0755) 42 | afero.WriteFile(appFS, "file7-dst.txt", []byte("01234567890123aaaaaaaa234567890123"), 0755) 43 | 44 | 45 | // resumable with spaces 46 | afero.WriteFile(appFS, "file8-src.txt", []byte("this is the full content of a file with a partial existing destination"), 0755) 47 | afero.WriteFile(appFS, "file8-dst.txt", []byte("this is the full content of a file with a partial"), 0755) 48 | 49 | 50 | return appFS 51 | } 52 | 53 | func prepareTestSubect(fileNamePrefix string, bufferSize int64) (*compare.Stitch, error) { 54 | fs := prepareFileSystem() 55 | src, _ := fs.Open(fileNamePrefix+"-src.txt") 56 | dst, _ := fs.OpenFile(fileNamePrefix+"-dst.txt", os.O_RDWR|os.O_CREATE, 0755) 57 | 58 | return compare.NewStich(src, dst, bufferSize) 59 | } 60 | 61 | func TestNotExistingFile(t *testing.T) { 62 | expect := assert.New(t) 63 | 64 | subject, err := prepareTestSubect("file1",2) 65 | 66 | expect.NoError(err) 67 | expect.NotNil(subject) 68 | expect.False(subject.IsComplete()) 69 | } 70 | 71 | func TestBiggerDestinationFile(t *testing.T) { 72 | expect := assert.New(t) 73 | 74 | subject, err := prepareTestSubect("file2",2) 75 | 76 | expect.Error(err) 77 | expect.Equal("source is smaller than destination", err.Error()) 78 | expect.Nil(subject) 79 | } 80 | 81 | func TestAlreadyCompletedFile(t *testing.T) { 82 | expect := assert.New(t) 83 | 84 | subject, err := prepareTestSubect("file3",2 ) 85 | 86 | expect.NoError(err) 87 | expect.NotNil(subject) 88 | expect.True(subject.IsComplete()) 89 | } 90 | 91 | func TestPartialFileCanBeResumed(t *testing.T) { 92 | expect := assert.New(t) 93 | 94 | subject, err := prepareTestSubect("file4", 2) 95 | 96 | expect.NoError(err) 97 | expect.NotNil(subject) 98 | expect.False(subject.IsComplete()) 99 | } 100 | 101 | func TestFileDiffAtStartCannotBeResumed(t *testing.T) { 102 | expect := assert.New(t) 103 | 104 | subject, err := prepareTestSubect("file5",2 ) 105 | 106 | expect.Error(err) 107 | expect.Equal("source file does not match destination file", err.Error()) 108 | expect.Nil(subject) 109 | } 110 | 111 | func TestFileDiffAtEndCannotBeResumed(t *testing.T) { 112 | expect := assert.New(t) 113 | 114 | subject, err := prepareTestSubect("file6", 2) 115 | 116 | expect.Error(err) 117 | expect.Equal("source file does not match destination file", err.Error()) 118 | expect.Nil(subject) 119 | } 120 | 121 | func TestFileDiffAtMiddleCannotBeResumed(t *testing.T) { 122 | expect := assert.New(t) 123 | 124 | subject, err := prepareTestSubect("file7",2 ) 125 | 126 | expect.Error(err) 127 | expect.Equal("source file does not match destination file", err.Error()) 128 | expect.Nil(subject) 129 | } 130 | 131 | func TestFileCanBeResumedWithHigherBufferSize(t *testing.T) { 132 | expect := assert.New(t) 133 | 134 | subject, err := prepareTestSubect("file8",1024 * 32 ) 135 | 136 | expect.NoError(err) 137 | expect.NotNil(subject) 138 | expect.False(subject.IsComplete()) 139 | } 140 | -------------------------------------------------------------------------------- /file/locator_test.go: -------------------------------------------------------------------------------- 1 | package file_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/sandreas/graft/designpattern/observer" 8 | "github.com/sandreas/graft/file" 9 | "github.com/sandreas/graft/matcher" 10 | "github.com/sandreas/graft/pattern" 11 | "github.com/sandreas/graft/testhelpers" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | const sep = string(os.PathSeparator) 16 | 17 | type FakeObserver struct { 18 | designpattern.ObserverInterface 19 | increaseItemsCalls int64 20 | increaseMatchesCalls int64 21 | finishCalls int64 22 | } 23 | 24 | func (ph *FakeObserver) Notify(a ...interface{}) { 25 | if a[0] == file.LocatorIncreaseMatches { 26 | ph.increaseMatchesCalls++ 27 | return 28 | } 29 | 30 | if a[0] == file.LocatorIncreaseItems { 31 | ph.increaseItemsCalls++ 32 | return 33 | } 34 | 35 | if a[0] == file.LocatorFinish { 36 | ph.finishCalls++ 37 | return 38 | } 39 | } 40 | 41 | func preparePattern(patternString string) (*file.Locator, *FakeObserver, *matcher.CompositeMatcher) { 42 | mockFs := testhelpers.MockFileSystem(map[string]string{ 43 | "fixtures" + sep + "global" + sep + "": "", 44 | "fixtures" + sep + "global" + sep + "file.txt": "", 45 | "fixtures" + sep + "file" + sep + "WalkPathByPattern" + sep + "dir" + sep + "": "", 46 | "fixtures" + sep + "file" + sep + "WalkPathByPattern" + sep + "dir" + sep + "dirfile.txt": "", 47 | "fixtures" + sep + "file" + sep + "WalkPathByPattern" + sep + "dir" + sep + "subdir" + sep + "": "", 48 | "fixtures" + sep + "file" + sep + "WalkPathByPattern" + sep + "dir" + sep + "subdir" + sep + "subdirfile.log": "", 49 | }) 50 | 51 | sourcePattern := pattern.NewSourcePattern(mockFs, patternString) 52 | compiledRegex, err := sourcePattern.Compile() 53 | if err != nil { 54 | println("error compiling regex: ", err.Error()) 55 | } 56 | m := matcher.NewRegexMatcher(compiledRegex) 57 | composite := matcher.NewCompositeMatcher() 58 | composite.Add(m) 59 | fakeObserver := &FakeObserver{} 60 | 61 | subject := file.NewLocator(sourcePattern) 62 | subject.RegisterObserver(fakeObserver) 63 | 64 | return subject, fakeObserver, composite 65 | } 66 | 67 | func TestFindWithFile(t *testing.T) { 68 | expect := assert.New(t) 69 | 70 | subject, fakeObserver, composite := preparePattern("fixtures/global/file.txt") 71 | 72 | subject.Find(composite) 73 | 74 | expect.Equal(1, len(subject.SourceFiles)) 75 | expect.Equal("fixtures"+sep+"global"+sep+"file.txt", subject.SourceFiles[0]) 76 | 77 | expect.Equal(int64(0), fakeObserver.increaseItemsCalls) 78 | expect.Equal(int64(1), fakeObserver.increaseMatchesCalls) 79 | expect.Equal(int64(1), fakeObserver.finishCalls) 80 | } 81 | 82 | func TestFindFilesWithDirectory(t *testing.T) { 83 | expect := assert.New(t) 84 | 85 | subject, fakeObserver, composite := preparePattern("fixtures/file/WalkPathByPattern/dir") 86 | 87 | subject.Find(composite) 88 | 89 | expect.Equal(4, len(subject.SourceFiles)) 90 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"", subject.SourceFiles[0]) 91 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"dirfile.txt", subject.SourceFiles[1]) 92 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"subdir"+sep+"", subject.SourceFiles[2]) 93 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"subdir"+sep+"subdirfile.log", subject.SourceFiles[3]) 94 | 95 | expect.Equal(int64(0), fakeObserver.increaseItemsCalls) 96 | expect.Equal(int64(4), fakeObserver.increaseMatchesCalls) 97 | expect.Equal(int64(1), fakeObserver.finishCalls) 98 | } 99 | 100 | func TestFindFilesWithGlob(t *testing.T) { 101 | expect := assert.New(t) 102 | 103 | subject, fakeObserver, composite := preparePattern("fixtures/file/WalkPathByPattern/dir/*irfile*") 104 | 105 | subject.Find(composite) 106 | 107 | expect.Equal(2, len(subject.SourceFiles)) 108 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"dirfile.txt", subject.SourceFiles[0]) 109 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"subdir"+sep+"subdirfile.log", subject.SourceFiles[1]) 110 | 111 | expect.Equal(int64(1), fakeObserver.increaseItemsCalls) 112 | expect.Equal(int64(2), fakeObserver.increaseMatchesCalls) 113 | expect.Equal(int64(1), fakeObserver.finishCalls) 114 | } 115 | 116 | func TestFind(t *testing.T) { 117 | expect := assert.New(t) 118 | 119 | subject, fakeObserver, composite := preparePattern("fixtures/file/WalkPathByPattern/dir/") 120 | 121 | subject.Find(composite) 122 | 123 | expect.Equal(4, len(subject.SourceFiles)) 124 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"", subject.SourceFiles[0]) 125 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"dirfile.txt", subject.SourceFiles[1]) 126 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"subdir"+sep+"", subject.SourceFiles[2]) 127 | expect.Equal("fixtures"+sep+"file"+sep+"WalkPathByPattern"+sep+"dir"+sep+"subdir"+sep+"subdirfile.log", subject.SourceFiles[3]) 128 | // 129 | expect.Equal(int64(0), fakeObserver.increaseItemsCalls) 130 | expect.Equal(int64(4), fakeObserver.increaseMatchesCalls) 131 | expect.Equal(int64(1), fakeObserver.finishCalls) 132 | } 133 | -------------------------------------------------------------------------------- /pattern/source_test.go: -------------------------------------------------------------------------------- 1 | package pattern_test 2 | 3 | import ( 4 | "testing" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/sandreas/graft/testhelpers" 7 | "regexp" 8 | "github.com/sandreas/graft/pattern" 9 | "os" 10 | "runtime" 11 | ) 12 | var sep = string(os.PathSeparator) 13 | 14 | func TestNewSourcePattern(t *testing.T) { 15 | expect := assert.New(t) 16 | 17 | mockFs := testhelpers.MockFileSystem(map[string]string{ 18 | "fixtures/global/": "", 19 | "fixtures/global/file.txt": "", 20 | }) 21 | 22 | sourcePattern := pattern.NewSourcePattern(mockFs, "fixtures/global/*") 23 | expect.Equal("fixtures"+sep+"global"+sep, sourcePattern.Path) 24 | expect.Equal("*", sourcePattern.Pattern) 25 | expect.True(sourcePattern.IsDir()) 26 | expect.False(sourcePattern.IsFile()) 27 | 28 | sourcePattern = pattern.NewSourcePattern(mockFs, "fixtures/non-existing/*.*") 29 | expect.Equal("fixtures"+sep, sourcePattern.Path) 30 | expect.Equal("non-existing/*.*", sourcePattern.Pattern) 31 | expect.True(sourcePattern.IsDir()) 32 | expect.False(sourcePattern.IsFile()) 33 | 34 | sourcePattern = pattern.NewSourcePattern(mockFs, "fixtures/global/file.txt") 35 | expect.Equal("fixtures"+sep+"global"+sep+"file.txt", sourcePattern.Path) 36 | expect.Equal("", sourcePattern.Pattern) 37 | expect.False(sourcePattern.IsDir()) 38 | expect.True(sourcePattern.IsFile()) 39 | 40 | sourcePattern = pattern.NewSourcePattern(mockFs, "fixtures/global/") 41 | expect.Equal("fixtures"+sep+"global"+sep, sourcePattern.Path) 42 | expect.Equal("", sourcePattern.Pattern) 43 | expect.True(sourcePattern.IsDir()) 44 | expect.False(sourcePattern.IsFile()) 45 | 46 | sourcePattern = pattern.NewSourcePattern(mockFs, "*") 47 | expect.Equal("."+sep, sourcePattern.Path) 48 | expect.Equal("*", sourcePattern.Pattern) 49 | expect.True(sourcePattern.IsDir()) 50 | expect.False(sourcePattern.IsFile()) 51 | 52 | sourcePattern = pattern.NewSourcePattern(mockFs, "./*") 53 | expect.Equal("."+sep, sourcePattern.Path) 54 | expect.Equal("*", sourcePattern.Pattern) 55 | expect.True(sourcePattern.IsDir()) 56 | expect.False(sourcePattern.IsFile()) 57 | 58 | sourcePattern = pattern.NewSourcePattern(mockFs, "fixtures/global/(.*)") 59 | expect.Equal("fixtures"+sep+"global"+sep, sourcePattern.Path) 60 | expect.Equal("(.*)", sourcePattern.Pattern) 61 | expect.True(sourcePattern.IsDir()) 62 | expect.False(sourcePattern.IsFile()) 63 | 64 | sourcePattern = pattern.NewSourcePattern(mockFs, ".") 65 | expect.Equal("."+sep, sourcePattern.Path) 66 | expect.Equal("", sourcePattern.Pattern) 67 | expect.True(sourcePattern.IsDir()) 68 | expect.False(sourcePattern.IsFile()) 69 | 70 | sourcePattern = pattern.NewSourcePattern(mockFs, "./") 71 | expect.Equal("."+sep, sourcePattern.Path) 72 | expect.Equal("", sourcePattern.Pattern) 73 | expect.True(sourcePattern.IsDir()) 74 | expect.False(sourcePattern.IsFile()) 75 | } 76 | 77 | func TestCompileSimple(t *testing.T) { 78 | quotedSep := "/" 79 | sep := "/" 80 | expect := assert.New(t) 81 | mockFs := testhelpers.MockFileSystem(map[string]string{ 82 | "fixtures/global/": "", 83 | "fixtures/global/file.txt": "", 84 | }) 85 | 86 | var compiled *regexp.Regexp 87 | compiled, _ = pattern.NewSourcePattern(mockFs, "fixtures/global/file.txt").Compile() 88 | expect.Equal("(?i)fixtures"+quotedSep+"global"+quotedSep+"file\\.txt$", compiled.String()) 89 | expect.Regexp(compiled, "fixtures"+sep+"global"+sep+"file.txt") 90 | 91 | } 92 | 93 | func TestCompileGlob(t *testing.T) { 94 | quotedSep := "/" 95 | sep := "/" 96 | expect := assert.New(t) 97 | mockFs := testhelpers.MockFileSystem(map[string]string{ 98 | "fixtures/global/": "", 99 | "fixtures/global/file.txt": "", 100 | }) 101 | var compiled *regexp.Regexp 102 | compiled, _ = pattern.NewSourcePattern(mockFs, "fixtures/global/*").Compile() 103 | expect.Equal("(?i)fixtures"+quotedSep+"global"+quotedSep+"(.*)$", compiled.String()) 104 | expect.Regexp(compiled, "fixtures"+sep+"global"+sep+"test.txt") 105 | 106 | compiled, _ = pattern.NewSourcePattern(mockFs, "fixtures/global/").Compile() 107 | expect.Equal("(?i)fixtures"+quotedSep+"global"+quotedSep+"(.*)$", compiled.String()) 108 | expect.Regexp(compiled, "fixtures"+sep+"global"+sep+"test.txt") 109 | 110 | compiled, _ = pattern.NewSourcePattern(mockFs, "fixtures/global/t(*)t.(txt)").Compile() 111 | expect.Equal("(?i)fixtures"+quotedSep+"global"+quotedSep+"t(.*)t\\.(txt)$", compiled.String()) 112 | expect.Regexp(compiled, "fixtures"+sep+"global"+sep+"Test.txt") 113 | 114 | compiled, _ = pattern.NewSourcePattern(mockFs, "fixtures/global/t(*)t.(txt)", pattern.CASE_SENSITIVE).Compile() 115 | expect.Equal("fixtures"+quotedSep+"global"+quotedSep+"t(.*)t\\.(txt)$", compiled.String()) 116 | expect.NotRegexp(compiled, "fixtures"+sep+"global"+sep+"Test.txt") 117 | 118 | sourcePattern := pattern.NewSourcePattern(mockFs, "fixtures/global/.*.?", pattern.CASE_SENSITIVE|pattern.USE_REAL_REGEX) 119 | compiled, _ = sourcePattern.Compile() 120 | expect.Equal("fixtures"+quotedSep+"global"+quotedSep+"(.*.?)$", compiled.String()) 121 | 122 | sourcePattern = pattern.NewSourcePattern(mockFs, ".") 123 | compiled, _ = sourcePattern.Compile() 124 | expect.Equal("(?i)(.*)$", compiled.String()) 125 | 126 | sourcePattern = pattern.NewSourcePattern(mockFs, "./") 127 | compiled, _ = sourcePattern.Compile() 128 | expect.Equal("(?i)(.*)$", compiled.String()) 129 | 130 | 131 | sourcePattern = pattern.NewSourcePattern(mockFs, "(*)\\\\global\\\\(*)") 132 | compiled, _ = sourcePattern.Compile() 133 | if runtime.GOOS != "windows" { 134 | expect.Equal("(?i)(.*)\\\\global\\\\(.*)$", compiled.String()) 135 | } else { 136 | expect.Equal("(?i)(.*)/global/(.*)$", compiled.String()) 137 | } 138 | 139 | sourcePattern = pattern.NewSourcePattern(mockFs, "(*)//global//(*)") 140 | compiled, _ = sourcePattern.Compile() 141 | expect.Equal("(?i)(.*)/global/(.*)$", compiled.String()) 142 | 143 | } 144 | -------------------------------------------------------------------------------- /pattern/base_test.go: -------------------------------------------------------------------------------- 1 | package pattern_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "os" 7 | 8 | "github.com/sandreas/graft/pattern" 9 | "github.com/sandreas/graft/testhelpers" 10 | "github.com/stretchr/testify/assert" 11 | "runtime" 12 | "strings" 13 | "github.com/spf13/afero" 14 | ) 15 | 16 | func TestBase(t *testing.T) { 17 | sep := string(os.PathSeparator) 18 | expect := assert.New(t) 19 | 20 | // long paths on windows have to be converted to absolute ones 21 | veryLongRelativePath := "inetpub/wwwroot/something_4.0/node_modules/babel-preset-es2015/node_modules/babel-plugin-transform-es2015-block-scoping/node_modules/babel-traverse/node_modules/babel-code-frame/node_modules/chalk/node_modules/strip-ansi/node_modules/ansi-regex/node_modules/fake-sub-module" 22 | veryLongRelativePathFile := veryLongRelativePath + "/package.json" 23 | 24 | uncPath := sep + sep + "unc-server" + sep + "unc-share" + sep + "file.txt" 25 | 26 | mockFs := testhelpers.MockFileSystem(map[string]string{ 27 | "fixtures/global/": "", 28 | "fixtures/global/file.txt": "", 29 | veryLongRelativePathFile: "{}", 30 | "C:" + sep + "Temp": "", 31 | uncPath: "file-content", 32 | }) 33 | 34 | var basePattern *pattern.BasePattern 35 | 36 | basePattern = pattern.NewBasePattern(mockFs, "fixtures/global/*") 37 | expect.Equal("fixtures"+sep+"global"+sep, basePattern.Path) 38 | expect.Equal("*", basePattern.Pattern) 39 | expect.True(basePattern.IsDir()) 40 | expect.False(basePattern.IsFile()) 41 | 42 | basePattern = pattern.NewBasePattern(mockFs, "fixtures/global/(.*)") 43 | expect.Equal("fixtures"+sep+"global"+sep, basePattern.Path) 44 | expect.Equal("(.*)", basePattern.Pattern) 45 | expect.True(basePattern.IsDir()) 46 | expect.False(basePattern.IsFile()) 47 | 48 | basePattern = pattern.NewBasePattern(mockFs, "fixtures/global/file.txt") 49 | expect.Equal("fixtures"+sep+"global"+sep+"file.txt", basePattern.Path) 50 | expect.Equal("", basePattern.Pattern) 51 | expect.False(basePattern.IsDir()) 52 | expect.True(basePattern.IsFile()) 53 | 54 | basePattern = pattern.NewBasePattern(mockFs, "fixtures/global/") 55 | expect.Equal("fixtures"+sep+"global"+sep, basePattern.Path) 56 | expect.Equal("", basePattern.Pattern) 57 | expect.True(basePattern.IsDir()) 58 | expect.False(basePattern.IsFile()) 59 | 60 | basePattern = pattern.NewBasePattern(mockFs, "fixtures/non-existing/*.*") 61 | expect.Equal("fixtures"+sep, basePattern.Path) 62 | expect.Equal("non-existing/*.*", basePattern.Pattern) 63 | expect.True(basePattern.IsDir()) 64 | expect.False(basePattern.IsFile()) 65 | 66 | basePattern = pattern.NewBasePattern(mockFs, "*") 67 | expect.Equal("."+sep, basePattern.Path) 68 | expect.Equal("*", basePattern.Pattern) 69 | expect.True(basePattern.IsDir()) 70 | expect.False(basePattern.IsFile()) 71 | 72 | basePattern = pattern.NewBasePattern(mockFs, "") 73 | expect.Equal("."+sep, basePattern.Path) 74 | expect.Equal("", basePattern.Pattern) 75 | expect.True(basePattern.IsDir()) 76 | expect.False(basePattern.IsFile()) 77 | 78 | basePattern = pattern.NewBasePattern(mockFs, "./*") 79 | expect.Equal("."+sep, basePattern.Path) 80 | expect.Equal("*", basePattern.Pattern) 81 | expect.True(basePattern.IsDir()) 82 | expect.False(basePattern.IsFile()) 83 | 84 | basePattern = pattern.NewBasePattern(mockFs, ".") 85 | expect.Equal("."+sep, basePattern.Path) 86 | expect.Equal("", basePattern.Pattern) 87 | expect.True(basePattern.IsDir()) 88 | expect.False(basePattern.IsFile()) 89 | 90 | basePattern = pattern.NewBasePattern(mockFs, "./") 91 | expect.Equal("."+sep, basePattern.Path) 92 | expect.Equal("", basePattern.Pattern) 93 | expect.True(basePattern.IsDir()) 94 | expect.False(basePattern.IsFile()) 95 | 96 | basePattern = pattern.NewBasePattern(mockFs, veryLongRelativePath+"/*") 97 | expect.Equal("*", basePattern.Pattern) 98 | expect.True(basePattern.IsDir()) 99 | expect.False(basePattern.IsFile()) 100 | 101 | basePattern = pattern.NewBasePattern(mockFs, "C:/TestMissingDir/") 102 | expect.Equal("C:"+sep, basePattern.Path) 103 | expect.Equal("TestMissingDir/", basePattern.Pattern) 104 | expect.True(basePattern.IsDir()) 105 | expect.False(basePattern.IsFile()) 106 | 107 | basePattern = pattern.NewBasePattern(mockFs, "(*)fi(*).txt") 108 | expect.Equal("."+sep, basePattern.Path) 109 | expect.Equal("(*)fi(*).txt", basePattern.Pattern) 110 | expect.True(basePattern.IsDir()) 111 | expect.False(basePattern.IsFile()) 112 | 113 | basePattern = pattern.NewBasePattern(mockFs, uncPath) 114 | if runtime.GOOS == "windows" { 115 | expect.Equal("\\\\unc-server\\unc-share\\file.txt", basePattern.Path) 116 | expect.Equal("", basePattern.Pattern) 117 | expect.False(basePattern.IsDir()) 118 | expect.True(basePattern.IsFile()) 119 | } else { 120 | expect.Equal("/unc-server/unc-share/file.txt", basePattern.Path) 121 | expect.Equal("", basePattern.Pattern) 122 | expect.False(basePattern.IsDir()) 123 | expect.True(basePattern.IsFile()) 124 | } 125 | 126 | slashedUncPath := strings.Replace(uncPath, "\\", "/", -1) 127 | basePattern = pattern.NewBasePattern(mockFs, slashedUncPath) 128 | if runtime.GOOS == "windows" { 129 | expect.Equal("\\\\unc-server\\unc-share\\file.txt", basePattern.Path) 130 | expect.Equal("", basePattern.Pattern) 131 | expect.False(basePattern.IsDir()) 132 | expect.True(basePattern.IsFile()) 133 | } else { 134 | expect.Equal("/unc-server/unc-share/file.txt", basePattern.Path) 135 | expect.Equal("", basePattern.Pattern) 136 | expect.False(basePattern.IsDir()) 137 | expect.True(basePattern.IsFile()) 138 | } 139 | 140 | basePattern = pattern.NewBasePattern(mockFs, "/") 141 | expect.Equal(sep, basePattern.Path) 142 | expect.Equal("", basePattern.Pattern) 143 | expect.True(basePattern.IsDir()) 144 | expect.False(basePattern.IsFile()) 145 | 146 | } 147 | 148 | func TestBaseOsFs(t *testing.T) { 149 | expect := assert.New(t) 150 | 151 | basePattern := pattern.NewBasePattern(afero.OsFs{}, "") 152 | expect.Equal(".\\", basePattern.Path) 153 | expect.Equal("", basePattern.Pattern) 154 | expect.True(basePattern.IsDir()) 155 | expect.False(basePattern.IsFile()) 156 | } 157 | -------------------------------------------------------------------------------- /action/receive.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/sandreas/graft/transfer" 8 | "github.com/urfave/cli" 9 | 10 | "context" 11 | "errors" 12 | 13 | "fmt" 14 | 15 | "bufio" 16 | "net" 17 | "os" 18 | "strconv" 19 | "strings" 20 | 21 | "github.com/grandcat/zeroconf" 22 | ) 23 | 24 | type ReceiveAction struct { 25 | AbstractTransferAction 26 | serverEntries []*zeroconf.ServiceEntry 27 | } 28 | 29 | func (action *ReceiveAction) Execute(c *cli.Context) error { 30 | action.serverEntries = []*zeroconf.ServiceEntry{} 31 | if err := action.PrepareExecution(c, 2, "*", "."); err != nil { 32 | return err 33 | } 34 | 35 | if action.shouldLookup() { 36 | return action.lookupServiceAndReceive() 37 | } else { 38 | return action.receive() 39 | } 40 | return nil 41 | } 42 | func (action *ReceiveAction) shouldLookup() bool { 43 | return action.CliContext.String("host") == "" 44 | } 45 | func (action *ReceiveAction) receive() error { 46 | 47 | action.CliParameters.Client = true 48 | 49 | action.suppressablePrintf("receive from %s@%s:%d\n", action.CliContext.String("username"), action.CliParameters.Host, action.CliParameters.Port) 50 | 51 | if action.CliContext.String("password") == "" { 52 | password, err := action.promptPassword("\nEnter password:") 53 | if err != nil { 54 | return err 55 | } 56 | action.CliContext.Set("password", password) 57 | } 58 | 59 | if err := action.locateSourceFiles(); err != nil { 60 | return cli.NewExitError(err.Error(), ErrorLocateSourceFiles) 61 | } 62 | 63 | if err := action.prepareDestination(); err != nil { 64 | return cli.NewExitError(err.Error(), ErrorPrepareDestination) 65 | } 66 | 67 | messagePrinter := transfer.NewMessagePrinterObserver(action.suppressablePrintf) 68 | transferStrategy, err := transfer.NewTransferStrategy(transfer.CopyResumed, action.sourcePattern, action.destinationPattern) 69 | if err != nil { 70 | return err 71 | } 72 | transferStrategy.ProgressHandler = transfer.NewCopyProgressHandler(int64(32*1024), 1*time.Second) 73 | transferStrategy.RegisterObserver(messagePrinter) 74 | transferStrategy.DryRun = action.CliContext.Bool("dry-run") 75 | transferStrategy.KeepTimes = action.CliContext.Bool("times") 76 | return transferStrategy.Perform(action.locator.SourceFiles) 77 | } 78 | 79 | func (action *ReceiveAction) lookupServiceAndReceive() error { 80 | var err error 81 | action.suppressablePrintf("hostname parameter is not set, trying to find graft servers...\n") 82 | 83 | service := "_graft._tcp" 84 | domain := "" 85 | 86 | waitTime := 1 87 | 88 | // Discover all services on the network (e.g. _workstation._tcp) 89 | resolver, err := zeroconf.NewResolver(nil) 90 | if err != nil { 91 | log.Fatalln("Failed to initialize resolver:", err.Error()) 92 | } 93 | 94 | entries := make(chan *zeroconf.ServiceEntry) 95 | go func(results <-chan *zeroconf.ServiceEntry) { 96 | for entry := range results { 97 | action.serverEntries = append(action.serverEntries, entry) 98 | println("found new server: " + fmt.Sprintf("%s:%d", entry.HostName, entry.Port)) 99 | } 100 | }(entries) 101 | 102 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(waitTime)) 103 | defer cancel() 104 | err = resolver.Browse(ctx, service, domain, entries) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | <-ctx.Done() 110 | // Wait some additional time to see debug messages on go routine shutdown. 111 | time.Sleep(1 * time.Second) 112 | action.chooseServerAndReceive() 113 | return nil 114 | } 115 | func (action *ReceiveAction) chooseServerAndReceive() error { 116 | serverCount := len(action.serverEntries) 117 | log.Printf("server entries found: %d", serverCount) 118 | 119 | if serverCount == 0 { 120 | return errors.New("graft did not find a server instance to receive from") 121 | } 122 | var selectedServer *zeroconf.ServiceEntry 123 | if serverCount == 1 { 124 | selectedServer = action.serverEntries[0] 125 | } else { 126 | action.suppressablePrintf("found %d servers, choose the one to receive from:\n", serverCount) 127 | 128 | for i := 0; i < serverCount; i++ { 129 | fmt.Printf("%d.) %s:%d\n", i+1, action.serverEntries[i].HostName, action.serverEntries[i].Port) 130 | } 131 | 132 | for { 133 | reader := bufio.NewReader(os.Stdin) 134 | fmt.Print("Choose server: ") 135 | text, _ := reader.ReadString('\n') 136 | 137 | chosenServerNum, err := strconv.Atoi(strings.Trim(text, "\r\n")) 138 | if err != nil || chosenServerNum < 1 || chosenServerNum > len(action.serverEntries) { 139 | fmt.Println("Invalid choice, please specify a valid number") 140 | } else { 141 | selectedServer = action.serverEntries[chosenServerNum-1] 142 | break 143 | } 144 | } 145 | } 146 | 147 | action.suppressablePrintf("selected server %s:%d\n", selectedServer.HostName, selectedServer.Port) 148 | 149 | lookupSuccess := false 150 | addr, err := net.LookupIP(selectedServer.HostName) 151 | if err != nil { 152 | log.Printf("Could not lookup host %s\n", selectedServer.HostName) 153 | } else { 154 | for _, ip := range addr { 155 | if resolvedIp := action.resolveIpConnection(ip, selectedServer.Port); resolvedIp != "" { 156 | action.CliParameters.Host = resolvedIp 157 | lookupSuccess = true 158 | break 159 | } 160 | } 161 | } 162 | 163 | if !lookupSuccess { 164 | log.Printf("Initial lookup for host %s failed\n", selectedServer.HostName) 165 | for _, ip := range selectedServer.AddrIPv4 { 166 | if resolvedIp := action.resolveIpConnection(ip, selectedServer.Port); resolvedIp != "" { 167 | action.CliParameters.Host = resolvedIp 168 | lookupSuccess = true 169 | break 170 | } 171 | } 172 | } 173 | if !lookupSuccess { 174 | log.Printf("IPv4 lookup for host %s failed\n", selectedServer.HostName) 175 | for _, ip := range selectedServer.AddrIPv6 { 176 | if resolvedIp := action.resolveIpConnection(ip, selectedServer.Port); resolvedIp != "" { 177 | action.CliParameters.Host = resolvedIp 178 | lookupSuccess = true 179 | break 180 | } 181 | } 182 | } 183 | 184 | if lookupSuccess { 185 | log.Printf("Lookup for host %s successful: %s\n", selectedServer.HostName, action.CliParameters.Host) 186 | } else { 187 | log.Printf("Lookup for host %s failed, using hostname instead", selectedServer.HostName) 188 | action.CliParameters.Host = selectedServer.HostName 189 | } 190 | 191 | action.CliParameters.Port = selectedServer.Port 192 | 193 | action.suppressablePrintf("connecting to %s:%d\n", action.CliParameters.Host, selectedServer.Port) 194 | 195 | action.receive() 196 | return nil 197 | 198 | } 199 | 200 | func (action *ReceiveAction) resolveIpConnection(netIp net.IP, port int) string { 201 | ip := netIp.To4() 202 | if ip == nil { 203 | ip = netIp.To16() 204 | } 205 | if ip == nil { 206 | return "" 207 | } 208 | 209 | conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", ip.String(), port)) 210 | if err != nil { 211 | return "" 212 | } 213 | defer conn.Close() 214 | return ip.String() 215 | } 216 | -------------------------------------------------------------------------------- /sftpd/path_mapper_test.go: -------------------------------------------------------------------------------- 1 | package sftpd_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/sandreas/graft/sftpd" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var files = []string{ 14 | "data/fixtures/global/", 15 | "data/fixtures/global/dir/", 16 | "data/fixtures/global/dir/dirfile.txt", 17 | "data/fixtures/global/dir/subdir/", 18 | "data/fixtures/global/dir/subdir/subdirfile.log", 19 | "data/fixtures/global/documents (2010)/", 20 | "data/fixtures/global/documents (2010)/document (2010).txt", 21 | "data/fixtures/global/file.txt", 22 | "data/fixtures/global/textfile.txt", 23 | } 24 | 25 | var filesWithDot = []string{ 26 | "../data/fixtures/global/", 27 | "../data/fixtures/global/dir/", 28 | "../data/fixtures/global/dir/dirfile.txt", 29 | "../data/fixtures/global/dir/subdir/", 30 | "../data/fixtures/global/dir/subdir/subdirfile.log", 31 | "../data/fixtures/global/documents (2010)/", 32 | "../data/fixtures/global/documents (2010)/document (2010).txt", 33 | "../data/fixtures/global/file.txt", 34 | "../data/fixtures/global/textfile.txt", 35 | } 36 | 37 | var filesOnly = []string{ 38 | "data/fixtures/file/CopyResumed/test2-dst-larger.txt", 39 | "data/fixtures/file/CopyResumed/test3-dst-partial.txt", 40 | "data/fixtures/file/CopyResumed/test4-dst-exists.txt", 41 | } 42 | 43 | var filesMulti = []string{ 44 | "data/fixtures/file/", 45 | "data/fixtures/file/AreFilesEqual/", 46 | "data/fixtures/file/AreFilesEqual/equal1.txt", 47 | "data/fixtures/file/AreFilesEqual/equal2.txt", 48 | "data/fixtures/file/AreFilesEqual/not-equal.txt", 49 | "data/fixtures/file/CopyResumed/", 50 | "data/fixtures/file/CopyResumed/test1-src.txt", 51 | "data/fixtures/file/CopyResumed/test2-dst-larger.txt", 52 | "data/fixtures/file/CopyResumed/test2-src.txt", 53 | "data/fixtures/file/CopyResumed/test3-dst-partial.txt", 54 | "data/fixtures/file/CopyResumed/test3-src.txt", 55 | "data/fixtures/file/CopyResumed/test4-dst-exists.txt", 56 | "data/fixtures/file/CopyResumed/test4-src.txt", 57 | "data/fixtures/file/ReadAllLines/", 58 | "data/fixtures/file/ReadAllLines/10-lines.txt", 59 | "data/fixtures/file/ReadAllLines/10-with-empty.txt", 60 | "data/fixtures/file/WalkPathByPattern/", 61 | "data/fixtures/file/WalkPathByPattern/dir/", 62 | "data/fixtures/file/WalkPathByPattern/dir/dirfile.txt", 63 | "data/fixtures/file/WalkPathByPattern/dir/subdir/", 64 | "data/fixtures/file/WalkPathByPattern/dir/subdir/subdirfile.log", 65 | "data/fixtures/file/WalkPathByPattern/documents (2010)/", 66 | "data/fixtures/file/WalkPathByPattern/documents (2010)/document (2010).txt", 67 | "data/fixtures/file/WalkPathByPattern/file.txt", 68 | "data/fixtures/file/WalkPathByPattern/test.part1.rar", 69 | "data/fixtures/file/WalkPathByPattern/test.part2.rar", 70 | "data/fixtures/file/WalkPathByPattern/test.part3.rar", 71 | "data/fixtures/file/WalkPathByPattern/textfile.txt", 72 | "data/fixtures/global/", 73 | "data/fixtures/global/dir/", 74 | "data/fixtures/global/dir/dirfile.txt", 75 | "data/fixtures/global/dir/subdir/", 76 | "data/fixtures/global/dir/subdir/subdirfile.log", 77 | "data/fixtures/global/documents (2010)/", 78 | "data/fixtures/global/documents (2010)/document (2010).txt", 79 | "data/fixtures/global/file.txt", 80 | "data/fixtures/global/textfile.txt", 81 | } 82 | 83 | var filesSpecial = []string{ 84 | "data/fixtures/file/AreFilesEqual/equal2.txt", 85 | "data/fixtures/file/WalkPathByPattern/documents (2010)/document (2010).txt", 86 | "data/fixtures/global/documents (2010)/document (2010).txt", 87 | } 88 | 89 | var singleFile = []string{ 90 | "./test.txt", 91 | } 92 | 93 | var singleFileServe = []string{ 94 | "../data/fixtures/file/AreFilesEqual/equal2.txt", 95 | } 96 | 97 | func TestFiles(t *testing.T) { 98 | expect := assert.New(t) 99 | 100 | mapper := sftpd.NewPathMapper(files, "data/fixtures") 101 | 102 | result, ok := mapper.List("global") 103 | want := []string{"/global/dir", "/global/documents (2010)", "/global/file.txt", "/global/textfile.txt"} 104 | expect.True(ok) 105 | expect.Equal(want, result) 106 | 107 | resultWithLeadingLash, ok2 := mapper.List("/global") 108 | expect.True(ok2) 109 | expect.Equal(want, resultWithLeadingLash) 110 | } 111 | 112 | func TestFilesDoubleDotSlash(t *testing.T) { 113 | expect := assert.New(t) 114 | 115 | mapper := sftpd.NewPathMapper(filesWithDot, "../data/fixtures") 116 | 117 | result, ok := mapper.List("global") 118 | want := []string{"/global/dir", "/global/documents (2010)", "/global/file.txt", "/global/textfile.txt"} 119 | expect.True(ok) 120 | expect.Equal(want, result) 121 | 122 | resultWithLeadingLash, ok2 := mapper.List("/global") 123 | expect.True(ok2) 124 | expect.Equal(want, resultWithLeadingLash) 125 | } 126 | 127 | func TestDotSlash(t *testing.T) { 128 | expect := assert.New(t) 129 | 130 | mapper := sftpd.NewPathMapper(files, "./data/fixtures") 131 | 132 | result, ok := mapper.List("global") 133 | want := []string{"/global/dir", "/global/documents (2010)", "/global/file.txt", "/global/textfile.txt"} 134 | expect.True(ok) 135 | expect.Equal(want, result) 136 | 137 | resultWithLeadingLash, ok2 := mapper.List("/global") 138 | expect.True(ok2) 139 | expect.Equal(want, resultWithLeadingLash) 140 | } 141 | 142 | func TestFilesOnly(t *testing.T) { 143 | expect := assert.New(t) 144 | 145 | mapper := sftpd.NewPathMapper(filesOnly, "data/fixtures") 146 | result, ok := mapper.List(string(os.PathSeparator)) 147 | want := []string{"/file"} 148 | expect.True(ok) 149 | expect.Equal(want, result) 150 | 151 | } 152 | 153 | func TestFilesMulti(t *testing.T) { 154 | expect := assert.New(t) 155 | 156 | mapper := sftpd.NewPathMapper(filesMulti, "data/fixtures") 157 | result, ok := mapper.List("/global") 158 | want := []string{"/global/dir", "/global/documents (2010)", "/global/file.txt", "/global/textfile.txt"} 159 | expect.True(ok) 160 | expect.Equal(want, result) 161 | 162 | } 163 | 164 | func TestFilesSpecial(t *testing.T) { 165 | expect := assert.New(t) 166 | 167 | mapper := sftpd.NewPathMapper(filesSpecial, "data/fixtures") 168 | result, ok := mapper.List("/") 169 | want := []string{"/file", "/global"} 170 | expect.True(ok) 171 | expect.Equal(want, result) 172 | 173 | } 174 | 175 | func TestPathTo(t *testing.T) { 176 | expect := assert.New(t) 177 | 178 | mapper := sftpd.NewPathMapper(files, "./data/fixtures") 179 | path, err := mapper.PathTo("global") 180 | expect.Equal(filepath.FromSlash("data/fixtures/global"), path) 181 | expect.Nil(err) 182 | 183 | path, err = mapper.PathTo("non-existing") 184 | expect.Equal("", path) 185 | expect.Error(err) 186 | 187 | mapper = sftpd.NewPathMapper(filesWithDot, "../data/fixtures") 188 | path, err = mapper.PathTo("global") 189 | expect.Equal(filepath.FromSlash("../data/fixtures/global"), path) 190 | expect.Nil(err) 191 | } 192 | 193 | func TestPathToSingleFile(t *testing.T) { 194 | expect := assert.New(t) 195 | 196 | mapper := sftpd.NewPathMapper(singleFile, ".") 197 | result, ok := mapper.PathTo("/test.txt") 198 | want := "test.txt" 199 | expect.NoError(ok) 200 | expect.Equal(want, result) 201 | } 202 | 203 | func TestPathToSingleFileDirectory(t *testing.T) { 204 | expect := assert.New(t) 205 | 206 | mapper := sftpd.NewPathMapper(singleFile, ".") 207 | result, ok := mapper.PathTo("/") 208 | want := "." 209 | expect.NoError(ok) 210 | expect.Equal(want, result) 211 | } 212 | 213 | func TestStat(t *testing.T) { 214 | expect := assert.New(t) 215 | 216 | mapper := sftpd.NewPathMapper(filesWithDot, "../data/fixtures") 217 | stat, err := mapper.Stat("global") 218 | expect.Nil(err) 219 | expect.True(stat.IsDir()) 220 | 221 | stat, err = mapper.Stat("/") 222 | expect.Nil(err) 223 | expect.True(stat.IsDir()) 224 | 225 | stat, err = mapper.Stat("non-existing") 226 | expect.Error(err) 227 | expect.Nil(stat) 228 | } 229 | 230 | func TestStatForServeFileOnly(t *testing.T) { 231 | expect := assert.New(t) 232 | 233 | mapper := sftpd.NewPathMapper(singleFileServe, "../data/fixtures/file/AreFilesEqual") 234 | stat, err := mapper.Stat("/") 235 | expect.Nil(err) 236 | expect.True(stat.IsDir()) 237 | 238 | stat, err = mapper.Stat("/equal2.txt") 239 | expect.Nil(err) 240 | expect.False(stat.IsDir()) 241 | } 242 | -------------------------------------------------------------------------------- /transfer/strategy.go: -------------------------------------------------------------------------------- 1 | package transfer 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "sort" 11 | "strings" 12 | "time" 13 | 14 | "github.com/sandreas/graft/designpattern/observer" 15 | "github.com/sandreas/graft/pattern" 16 | "github.com/sandreas/graft/filesystem" 17 | "github.com/sandreas/graft/file/compare" 18 | ) 19 | 20 | const ( 21 | CopyResumed = 1 22 | Move = 2 23 | ) 24 | 25 | type Strategy struct { 26 | designpattern.Observable 27 | 28 | SourcePattern *pattern.SourcePattern 29 | DestinationPattern *pattern.DestinationPattern 30 | CompiledSourcePattern *regexp.Regexp 31 | TransferredDirectories []string 32 | KeepTimes bool 33 | DryRun bool 34 | 35 | transferMethod int 36 | ProgressHandler *CopyProgressHandler 37 | bufferSize int64 38 | } 39 | 40 | func NewTransferStrategy(transferMethod int, src *pattern.SourcePattern, dst *pattern.DestinationPattern) (*Strategy, error) { 41 | var err error 42 | if transferMethod < CopyResumed || transferMethod > Move { 43 | return nil, errors.New("invalid transfer method" + string(transferMethod)) 44 | } 45 | strategy := &Strategy{ 46 | ProgressHandler: nil, 47 | bufferSize: 1024 * 32, 48 | transferMethod: transferMethod, 49 | } 50 | strategy.SourcePattern = src 51 | strategy.DestinationPattern = dst 52 | 53 | strategy.CompiledSourcePattern, err = strategy.SourcePattern.Compile() 54 | return strategy, err 55 | } 56 | 57 | func (strategy *Strategy) PerformFileTransfer(src string, dst string, srcStat os.FileInfo) error { 58 | if strategy.transferMethod == CopyResumed { 59 | return strategy.CopyResumed(src, dst, srcStat) 60 | } 61 | 62 | if strategy.transferMethod == Move { 63 | return strategy.Move(src, dst, srcStat) 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func (strategy *Strategy) CopyResumed(s, d string, srcStats os.FileInfo) error { 70 | src, err := strategy.SourcePattern.Fs.OpenFile(s, os.O_RDONLY, srcStats.Mode()) 71 | if err != nil { 72 | return err 73 | } 74 | defer src.Close() 75 | 76 | dst, err := strategy.DestinationPattern.Fs.OpenFile(d, os.O_RDWR|os.O_CREATE, srcStats.Mode()) 77 | if err != nil { 78 | return err 79 | } 80 | defer dst.Close() 81 | 82 | compareStrategy, err := compare.NewStich(src, dst, strategy.bufferSize) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | srcSize := compareStrategy.SourceFileStat.Size() 88 | dstSize := compareStrategy.DestinationFileStat.Size() 89 | strategy.handleProgress(dstSize, srcSize, strategy.bufferSize) 90 | 91 | if compareStrategy.IsComplete() { 92 | return nil 93 | } 94 | 95 | src.Seek(dstSize, 0) 96 | dst.Seek(dstSize, 0) 97 | 98 | buf := make([]byte, strategy.bufferSize) 99 | bytesTransferred := dstSize 100 | for { 101 | n, err := src.Read(buf) 102 | if err != nil && err != io.EOF { 103 | return err 104 | } 105 | if n == 0 { 106 | break 107 | } 108 | 109 | if _, err := dst.Write(buf[:n]); err != nil { 110 | return err 111 | } 112 | bytesTransferred += int64(n) 113 | newBufferSize := strategy.handleProgress(bytesTransferred, srcSize, strategy.bufferSize) 114 | if newBufferSize != strategy.bufferSize { 115 | strategy.bufferSize = newBufferSize 116 | buf = make([]byte, strategy.bufferSize) 117 | } 118 | } 119 | dst.Sync() 120 | 121 | return nil 122 | } 123 | 124 | func (strategy *Strategy) Move(src string, dst string, srcStat os.FileInfo) error { 125 | return strategy.SourcePattern.Fs.Rename(src, dst) 126 | } 127 | 128 | func (strategy *Strategy) handleProgress(bytesTransferred, srcSize, bufferSize int64) int64 { 129 | if strategy.ProgressHandler == nil { 130 | return bufferSize 131 | } 132 | newBufferSize, message := strategy.ProgressHandler.Update(bytesTransferred, srcSize, bufferSize, time.Now()) 133 | strategy.NotifyObservers(message) 134 | return newBufferSize 135 | } 136 | 137 | func (strategy *Strategy) Cleanup() error { 138 | if strategy.transferMethod == CopyResumed { 139 | return nil 140 | } 141 | 142 | if strategy.transferMethod == Move { 143 | sort.Strings(strategy.TransferredDirectories) 144 | sliceLen := len(strategy.TransferredDirectories) 145 | lastDir := "" 146 | for i := sliceLen - 1; i >= 0; i-- { 147 | if strategy.TransferredDirectories[i] == lastDir { 148 | continue 149 | } 150 | err := strategy.SourcePattern.Fs.Remove(strategy.TransferredDirectories[i]) 151 | lastDir = strategy.TransferredDirectories[i] 152 | if err != nil { 153 | str := err.Error() 154 | println(str) 155 | return err 156 | } 157 | } 158 | return nil 159 | } 160 | return nil 161 | } 162 | 163 | func (strategy *Strategy) Perform(strings []string) error { 164 | var err, returnError error 165 | 166 | strategy.NotifyObservers("\n") 167 | for _, src := range strings { 168 | err = strategy.PerformSingleTransfer(src) 169 | if err != nil { 170 | strategy.NotifyObservers("\n - failed (" + err.Error() + ")\n") 171 | returnError = errors.New("some files failed to transfer") 172 | } 173 | } 174 | strategy.Cleanup() 175 | return returnError 176 | } 177 | 178 | func (strategy *Strategy) DestinationFor(src string) string { 179 | cleanedSrc := filesystem.CleanPath(strategy.SourcePattern.Fs, src) 180 | 181 | sep := string(os.PathSeparator) 182 | 183 | if strategy.SourcePattern.IsFile() { 184 | if strategy.DestinationPattern.IsFile() { 185 | return strategy.DestinationPattern.Path 186 | } 187 | 188 | if strategy.DestinationPattern.Pattern == "" { 189 | return strategy.DestinationPattern.Path + filepath.Base(cleanedSrc) 190 | } 191 | 192 | l:=len(strategy.DestinationPattern.Pattern) 193 | if l > 0 && !os.IsPathSeparator(strategy.DestinationPattern.Pattern[l-1]) { 194 | return strategy.DestinationPattern.Path + strategy.DestinationPattern.Pattern 195 | } 196 | cleanedPattern := strings.TrimRight(strategy.DestinationPattern.Pattern, "\\/") 197 | return filesystem.CleanPath(strategy.DestinationPattern.Fs, strategy.DestinationPattern.Path+cleanedPattern+sep+filepath.Base(cleanedSrc)) 198 | } 199 | 200 | srcPatternPath := strings.TrimRight(strategy.SourcePattern.Path, "\\/") 201 | srcPatternPathLen := len(srcPatternPath) 202 | if strategy.SourcePattern.Path == "." + sep { 203 | srcPatternPathLen = 0 204 | } 205 | 206 | // source pattern points to an existing file or directory 207 | if strategy.SourcePattern.Pattern == "" { 208 | sourceParentDir := filepath.Dir(srcPatternPath) 209 | destinationPathParts := []string{ 210 | strings.TrimRight(strategy.DestinationPattern.Path, sep), 211 | } 212 | 213 | if strategy.DestinationPattern.Pattern != "" { 214 | destinationPathParts = append(destinationPathParts, strings.TrimRight(strategy.DestinationPattern.Pattern, "\\/")) 215 | } 216 | 217 | sourcePartAppendToDestination := strings.Trim(strings.TrimPrefix(cleanedSrc, sourceParentDir), "\\/") 218 | destinationPathParts = append(destinationPathParts, sourcePartAppendToDestination) 219 | 220 | return filesystem.CleanPath(strategy.DestinationPattern.Fs, strings.Join(destinationPathParts, sep)) 221 | } 222 | 223 | // destination pattern points to an existing file or directory 224 | if strategy.DestinationPattern.Pattern == "" { 225 | return filesystem.CleanPath(strategy.DestinationPattern.Fs, strings.TrimRight(strategy.DestinationPattern.Path + strings.TrimLeft(cleanedSrc[srcPatternPathLen:], "\\/"), "\\/")) 226 | } 227 | 228 | toSlashSrc := filepath.ToSlash(cleanedSrc) 229 | toSlashDst := filepath.ToSlash(strategy.DestinationPattern.Path)+"/"+strings.TrimLeft(strategy.DestinationPattern.Pattern, "\\/") 230 | result := strategy.CompiledSourcePattern.ReplaceAllString(toSlashSrc, toSlashDst) 231 | return filesystem.CleanPath(strategy.DestinationPattern.Fs, result) 232 | } 233 | 234 | func (strategy *Strategy) PerformSingleTransfer(src string) error { 235 | 236 | srcStat, err := strategy.SourcePattern.Fs.Stat(src) 237 | if err != nil { 238 | return err 239 | } 240 | 241 | dst := strategy.DestinationFor(src) 242 | 243 | strategy.NotifyObservers(src + " => " + dst + "\n") 244 | 245 | if strategy.DryRun { 246 | return nil 247 | } 248 | 249 | if srcStat.IsDir() { 250 | return strategy.PerformDirectoryTransfer(src, dst, srcStat, true) 251 | } 252 | 253 | if err := strategy.EnsureDirectoryOfFileExists(src, dst); err != nil { 254 | return err 255 | } 256 | 257 | if err := strategy.PerformFileTransfer(src, dst, srcStat); err != nil { 258 | return err 259 | } 260 | 261 | return nil 262 | } 263 | 264 | func (strategy *Strategy) EnsureDirectoryOfFileExists(src, dst string) error { 265 | _, err := strategy.DestinationPattern.Fs.Stat(dst) 266 | if os.IsNotExist(err) || strategy.KeepTimes { 267 | srcDirName := filepath.Dir(src) 268 | srcDirStat, err := strategy.SourcePattern.Fs.Stat(srcDirName) 269 | if err != nil { 270 | log.Printf("Could not stat directory %s of file %s", srcDirName, src) 271 | return err 272 | } 273 | 274 | dstDirName := filepath.Dir(dst) 275 | return strategy.PerformDirectoryTransfer(srcDirName, dstDirName, srcDirStat, false) 276 | } 277 | return nil 278 | 279 | } 280 | func (strategy *Strategy) PerformDirectoryTransfer(src, dst string, srcStat os.FileInfo, shouldRemoveAfterTransfer bool) error { 281 | err := strategy.DestinationPattern.Fs.MkdirAll(dst, srcStat.Mode()) 282 | if err == nil && shouldRemoveAfterTransfer { 283 | strategy.TransferredDirectories = append(strategy.TransferredDirectories, dst) 284 | } 285 | 286 | if err == nil && strategy.KeepTimes { 287 | err = strategy.keepTimes(dst, srcStat) 288 | } 289 | return err 290 | } 291 | 292 | func (strategy *Strategy) keepTimes(dst string, inStats os.FileInfo) error { 293 | return strategy.DestinationPattern.Fs.Chtimes(dst, inStats.ModTime(), inStats.ModTime()) 294 | } 295 | -------------------------------------------------------------------------------- /action/abstract.go: -------------------------------------------------------------------------------- 1 | package action 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "os/user" 11 | "regexp" 12 | "runtime" 13 | "strconv" 14 | "strings" 15 | "time" 16 | 17 | "github.com/howeyc/gopass" 18 | "github.com/sandreas/graft/bitflag" 19 | "github.com/sandreas/graft/file" 20 | "github.com/sandreas/graft/filesystem" 21 | "github.com/sandreas/graft/matcher" 22 | "github.com/sandreas/graft/pattern" 23 | "github.com/spf13/afero" 24 | "github.com/urfave/cli" 25 | ) 26 | 27 | const ( 28 | ErrorPreventUsingSingleQuotesOnWindows = 1 29 | ErrorPositionalArgumentCount = 2 30 | ErrorLocateSourceFiles = 3 31 | ErrorStartingServer = 4 32 | ErrorCopyFiles = 5 33 | ErrorMoveFiles = 6 34 | ErrorPrepareDestination = 7 35 | ErrorDeleteFiles = 10 36 | //ErrorNoGraftServerAvailable = 8 37 | //ErrorFailedToInitializeResolver = 9 38 | ) 39 | 40 | func NewActionFactory(action string) CliActionInterface { 41 | switch action { 42 | case "find": 43 | return new(FindAction) 44 | case "serve": 45 | return new(ServeAction) 46 | case "copy": 47 | return new(CopyAction) 48 | case "move": 49 | return new(MoveAction) 50 | case "receive": 51 | return new(ReceiveAction) 52 | case "delete": 53 | return new(DeleteAction) 54 | } 55 | 56 | return nil 57 | } 58 | 59 | type CliParameters struct { 60 | Quiet bool `arg:"help:do not show any output"` 61 | Force bool `arg:"help:force the requested action - even if it might be not a good idea"` 62 | Debug bool `arg:"-d,help:debug mode with logging to Stdout and into $HOME/.graft/application.log"` 63 | Regex bool `arg:"help:use a real regex instead of glob patterns (e.g. src/.*\\.jpg)"` 64 | CaseSensitive bool `arg:"--case-sensitive,help:be case sensitive when matching files and folders"` 65 | MaxAge string `arg:"--max-age,help:maximum age (e.g. 2d / 8w / 2016-12-24 / etc. )"` 66 | MinAge string `arg:"--min-age,help:minimum age (e.g. 2d / 8w / 2016-12-24 / etc. )"` 67 | MaxSize string `arg:"--max-size,help:maximum size in bytes or format string (e.g. 2G / 8M / 1000K etc. )"` 68 | MinSize string `arg:"--min-size,help:minimum size in bytes or format string (e.g. 2G / 8M / 1000K etc. )"` 69 | Type string `arg:"--type,help:only match items with given type (--type=f for files, --type=d for directories)"` 70 | ExportTo string `arg:"--export-to,help:export found matches to a text file - one line per item"` 71 | FilesFrom string `arg:"--files-from,help:import found matches from file - one line per item"` 72 | 73 | Client bool 74 | Host string 75 | Port int 76 | } 77 | 78 | type AbstractAction struct { 79 | CliParameters *CliParameters 80 | CliContext *cli.Context 81 | 82 | PositionalArguments cli.Args 83 | sourceFs afero.Fs 84 | sourcePattern *pattern.SourcePattern 85 | compiledRegex *regexp.Regexp 86 | locator *file.Locator 87 | } 88 | 89 | func (action *AbstractAction) PrepareExecution(c *cli.Context, positionalArgumentsCount int, positionalDefaultsIfUnset ...string) error { 90 | 91 | action.ParseCliContext(c) 92 | action.initLogging() 93 | 94 | if action.usedSingleQuotesAsQualifierOnWindows() { 95 | return cli.NewExitError("using single quotes as qualifier may lead to unexpected results - please use double quotes or --force", ErrorPreventUsingSingleQuotesOnWindows) 96 | } 97 | 98 | if err := action.assertPositionalArgumentsCount(positionalArgumentsCount, positionalDefaultsIfUnset); err != nil { 99 | return cli.NewExitError(err.Error(), ErrorPositionalArgumentCount) 100 | } 101 | 102 | return nil 103 | } 104 | func (action *AbstractAction) assertPositionalArgumentsCount(expectedPositionalCount int, defaults []string) error { 105 | 106 | givenPositionalCount := len(action.CliContext.Args()) 107 | 108 | var positionalStrings []string 109 | if givenPositionalCount != expectedPositionalCount { 110 | if len(defaults) == expectedPositionalCount { 111 | for i := 0; i < expectedPositionalCount; i++ { 112 | if i < givenPositionalCount { 113 | positionalStrings = append(positionalStrings, action.CliContext.Args().Get(i)) 114 | } else { 115 | positionalStrings = append(positionalStrings, defaults[i]) 116 | } 117 | } 118 | action.PositionalArguments = cli.Args(positionalStrings) 119 | return nil 120 | } 121 | 122 | suffix := "argument" 123 | if expectedPositionalCount != 1 { 124 | suffix += "s" 125 | } 126 | return errors.New(action.CliContext.Command.Name + " takes " + strconv.Itoa(expectedPositionalCount) + " " + suffix) 127 | } else { 128 | action.PositionalArguments = action.CliContext.Args() 129 | } 130 | return nil 131 | } 132 | 133 | func (action *AbstractAction) ParseCliContext(c *cli.Context) { 134 | action.CliContext = c 135 | action.CliParameters = &CliParameters{ 136 | Debug: c.Bool("debug"), 137 | Quiet: c.Bool("quiet"), 138 | FilesFrom: c.String("files-from"), 139 | ExportTo: c.String("export-to"), 140 | MinAge: c.String("min-age"), 141 | MaxAge: c.String("max-age"), 142 | MinSize: c.String("min-size"), 143 | MaxSize: c.String("max-size"), 144 | Type: c.String("type"), 145 | Client: c.IsSet("client") && c.Bool("client"), 146 | } 147 | 148 | for _, name := range c.FlagNames() { 149 | if name == "host" { 150 | action.CliParameters.Host = c.String("host") 151 | } 152 | if name == "port" { 153 | action.CliParameters.Port = c.Int("port") 154 | } 155 | } 156 | } 157 | 158 | func (action *AbstractAction) initLogging() { 159 | if !action.CliParameters.Debug { 160 | log.SetFlags(0) 161 | log.SetOutput(ioutil.Discard) 162 | return 163 | } 164 | log.SetOutput(os.Stdout) 165 | 166 | homeDir, err := action.createHomeDirectoryIfNotExists() 167 | if err != nil { 168 | log.Println("could not create home directory: ", homeDir, err) 169 | return 170 | } 171 | logFileName := homeDir + "/graft.log" 172 | os.Remove(logFileName) 173 | logFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 174 | if err != nil { 175 | log.Println("could not open logfile: ", logFile, err) 176 | return 177 | } 178 | defer logFile.Close() 179 | mw := io.MultiWriter(os.Stdout, logFile) 180 | log.SetOutput(mw) 181 | } 182 | 183 | func (action *AbstractAction) createHomeDirectoryIfNotExists() (string, error) { 184 | u, _ := user.Current() 185 | homeDir := u.HomeDir + "/.graft" 186 | if _, err := os.Stat(homeDir); err != nil { 187 | if err := os.Mkdir(homeDir, os.FileMode(0755)); err != nil { 188 | return homeDir, err 189 | } 190 | } 191 | return homeDir, nil 192 | } 193 | 194 | func (action *AbstractAction) usedSingleQuotesAsQualifierOnWindows() bool { 195 | if runtime.GOOS != "windows" { 196 | return false 197 | } 198 | for _, arg := range action.CliContext.Args() { 199 | if strings.HasPrefix(arg, "'") { 200 | return true 201 | } 202 | } 203 | return false 204 | } 205 | 206 | func (action *AbstractAction) locateSourceFiles() error { 207 | if err := action.prepareSourcePattern(); err != nil { 208 | return err 209 | } 210 | 211 | if err := action.prepareLocator(); err != nil { 212 | return err 213 | } 214 | 215 | return nil 216 | } 217 | 218 | func (action *AbstractAction) prepareSourcePattern() error { 219 | var err error 220 | if err = action.prepareSourceFileSystem(); err != nil { 221 | return err 222 | } 223 | action.sourcePattern = pattern.NewSourcePattern(action.sourceFs, action.PositionalArguments.First(), action.parseSourcePatternBitFlags()) 224 | return nil 225 | } 226 | 227 | func (action *AbstractAction) prepareSourceFileSystem() error { 228 | var err error 229 | if action.CliParameters.Client { 230 | host := action.CliParameters.Host 231 | port := action.CliParameters.Port 232 | username := action.CliContext.String("username") 233 | password := action.CliContext.String("password") 234 | action.sourceFs, err = filesystem.NewSftpFs(host, port, username, password) 235 | return err 236 | } 237 | action.sourceFs = filesystem.NewOsFs() 238 | return nil 239 | } 240 | 241 | func (action *AbstractAction) parseSourcePatternBitFlags() bitflag.Flag { 242 | var patternFlags bitflag.Flag 243 | if action.CliParameters.CaseSensitive { 244 | patternFlags |= pattern.CASE_SENSITIVE 245 | } 246 | if action.CliParameters.Regex { 247 | patternFlags |= pattern.USE_REAL_REGEX 248 | } 249 | return patternFlags 250 | } 251 | 252 | func (action *AbstractAction) prepareLocator() error { 253 | var err error 254 | locator := file.NewLocator(action.sourcePattern) 255 | locator.RegisterObserver(file.NewWalkObserver(action.suppressablePrintf)) 256 | 257 | if action.compiledRegex, err = action.sourcePattern.Compile(); err != nil { 258 | return errors.New(`Could not compile source pattern - did you quote all special chars with backslash? (special chars: \ . + * ? ( ) | [ ] { } ^ $, error: ` + err.Error() + `)`) 259 | } 260 | 261 | if action.CliParameters.FilesFrom != "" { 262 | locatorCache := file.NewLocatorCache(action.CliParameters.FilesFrom) 263 | if err = locatorCache.Load(); err != nil { 264 | return err 265 | } 266 | locator.SourceFiles = locatorCache.Items 267 | } else { 268 | compositeMatcher := matcher.NewCompositeMatcher() 269 | compositeMatcher.Add(matcher.NewRegexMatcher(action.compiledRegex)) 270 | 271 | minAge := time.Time{} 272 | maxAge := time.Time{} 273 | 274 | if action.CliParameters.MinAge != "" { 275 | if minAge, err = pattern.StrToAge(action.CliParameters.MinAge, time.Now()); err != nil { 276 | return err 277 | } 278 | } 279 | 280 | if action.CliParameters.MaxAge != "" { 281 | if maxAge, err = pattern.StrToAge(action.CliParameters.MaxAge, time.Now()); err != nil { 282 | return err 283 | } 284 | } 285 | 286 | if !minAge.IsZero() || !maxAge.IsZero() { 287 | compositeMatcher.Add(matcher.NewFileAgeMatcher(minAge, maxAge)) 288 | } 289 | 290 | minSize := int64(-1) 291 | maxSize := int64(-1) 292 | if action.CliParameters.MinSize != "" { 293 | if minSize, err = pattern.StrToSize(action.CliParameters.MinSize); err != nil { 294 | return err 295 | } 296 | } 297 | 298 | if action.CliParameters.MaxSize != "" { 299 | if maxSize, err = pattern.StrToSize(action.CliParameters.MaxSize); err != nil { 300 | return err 301 | } 302 | } 303 | 304 | if minSize > -1 || maxSize > -1 { 305 | compositeMatcher.Add(matcher.NewFileSizeMatcher(minSize, maxSize)) 306 | } 307 | 308 | if action.CliParameters.Type != "" { 309 | if action.CliParameters.Type != matcher.TypeFile && action.CliParameters.Type != matcher.TypeDirectory { 310 | return errors.New("invalid value for parameter --type specified") 311 | } 312 | 313 | compositeMatcher.Add(matcher.NewFileTypeMatcher(action.CliParameters.Type)) 314 | } 315 | 316 | locator.Find(compositeMatcher) 317 | if action.CliParameters.ExportTo != "" { 318 | locatorCache := file.NewLocatorCache(action.CliParameters.ExportTo) 319 | locatorCache.Items = locator.SourceFiles 320 | if err = locatorCache.Save(); err != nil { 321 | return err 322 | } 323 | } 324 | } 325 | 326 | action.locator = locator 327 | 328 | return nil 329 | } 330 | 331 | func (action *AbstractAction) suppressablePrintf(format string, a ...interface{}) (n int, err error) { 332 | if !action.CliParameters.Quiet { 333 | return fmt.Printf(format, a...) 334 | } 335 | log.Printf(format, a...) 336 | return 0, nil 337 | } 338 | 339 | func (action *AbstractAction) ShowMatchesForPath(path string) { 340 | elementMatches := pattern.BuildMatchList(action.compiledRegex, path) 341 | for i := 0; i < len(elementMatches); i++ { 342 | action.suppressablePrintf(" $" + strconv.Itoa(i+1) + ": " + elementMatches[i] + "\n") 343 | } 344 | } 345 | 346 | func (action *AbstractAction) promptPassword(message string) (string, error) { 347 | if message != "" { 348 | println(message) 349 | } 350 | 351 | if pass, err := gopass.GetPasswd(); err != nil { 352 | return "", err 353 | } else { 354 | return string(pass), nil 355 | } 356 | 357 | } 358 | -------------------------------------------------------------------------------- /transfer/strategy_test.go: -------------------------------------------------------------------------------- 1 | package transfer_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "os" 7 | 8 | "github.com/sandreas/graft/pattern" 9 | "github.com/sandreas/graft/transfer" 10 | "github.com/spf13/afero" 11 | "github.com/stretchr/testify/assert" 12 | "time" 13 | "github.com/sandreas/graft/designpattern/observer" 14 | ) 15 | 16 | var sep = string(os.PathSeparator) 17 | 18 | func prepareStrategy(src string, dst string) *transfer.Strategy { 19 | fs := prepareFileSystem() 20 | srcPattern := pattern.NewSourcePattern(fs, src) 21 | dstPattern := pattern.NewDestinationPattern(fs, dst) 22 | compiledSrcPattern, _ := srcPattern.Compile() 23 | 24 | return &transfer.Strategy{ 25 | SourcePattern: srcPattern, 26 | DestinationPattern: dstPattern, 27 | CompiledSourcePattern: compiledSrcPattern, 28 | KeepTimes: false, 29 | } 30 | 31 | } 32 | 33 | func prepareFileSystem() afero.Fs { 34 | appFS := afero.NewMemMapFs() 35 | appFS.Mkdir("src", 0644) 36 | afero.WriteFile(appFS, "src/src-file.txt", []byte(""), 0755) 37 | appFS.Mkdir("src/test-dir", 0644) 38 | appFS.Mkdir("dst", 0644) 39 | afero.WriteFile(appFS, "dst/overwrite.txt", []byte(""), 0755) 40 | afero.WriteFile(appFS, "src/test-dir/test-dir-file.txt", []byte(""), 0755) 41 | appFS.Mkdir("C:"+sep+"Temp", 0644) 42 | return appFS 43 | } 44 | 45 | func TestRelativeWildcardMapping(t *testing.T) { 46 | expect := assert.New(t) 47 | 48 | strategy := prepareStrategy("src/*", "dst") 49 | 50 | expect.Equal("dst", strategy.DestinationFor("src")) 51 | expect.Equal("dst"+sep+"src-file.txt", strategy.DestinationFor("src/src-file.txt")) 52 | expect.Equal("dst"+sep+"test-dir", strategy.DestinationFor("src/test-dir")) 53 | expect.Equal("dst"+sep+"test-dir"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 54 | 55 | } 56 | 57 | func TestComplexRelativeMapping(t *testing.T) { 58 | expect := assert.New(t) 59 | 60 | var strategy *transfer.Strategy 61 | strategy = prepareStrategy("src/test-dir/test-dir-file.txt", "../out/") 62 | expect.Equal(".."+sep+"out"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 63 | 64 | strategy = prepareStrategy("src/test-dir", "../out") 65 | expect.Equal(".."+sep+"out"+sep+"test-dir"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 66 | 67 | strategy = prepareStrategy("src/test-dir/test-dir-file.txt", "dst/overwrite.txt") 68 | expect.Equal("dst"+sep+"overwrite.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 69 | 70 | strategy = prepareStrategy("src/test-dir/test-dir-file.txt", "dst") 71 | expect.Equal("dst"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 72 | 73 | strategy = prepareStrategy("src/test-dir/test-dir-file.txt", "dst/new-file.txt") 74 | expect.Equal("dst"+sep+"new-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 75 | 76 | strategy = prepareStrategy("src/test-dir/test-dir-file.txt", "C:/TestMissingDir/") 77 | expect.Equal("C:"+sep+"TestMissingDir"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 78 | 79 | strategy = prepareStrategy("src/test-dir", "C:/TestMissingDir") 80 | expect.Equal("C:"+sep+"TestMissingDir"+sep+"test-dir"+sep+"test-dir-file.txt", strategy.DestinationFor("src/test-dir/test-dir-file.txt")) 81 | 82 | strategy = prepareStrategy("*", ".") 83 | expect.Equal("file", strategy.DestinationFor("file")) 84 | } 85 | 86 | func TestSingleTransferSourceNotExists(t *testing.T) { 87 | expect := assert.New(t) 88 | strategy := prepareStrategy("src/*", "dst") 89 | expect.Error(strategy.PerformSingleTransfer("non-existing-file.txt")) 90 | } 91 | 92 | func TestSingleTransferDir(t *testing.T) { 93 | expect := assert.New(t) 94 | strategy := prepareStrategy("src/*", "dst") 95 | expect.NoError(strategy.PerformSingleTransfer("src/test-dir")) 96 | stat, err := strategy.DestinationPattern.Fs.Stat("dst/test-dir") 97 | expect.True(stat.IsDir()) 98 | expect.NoError(err) 99 | expect.Len(strategy.TransferredDirectories, 1) 100 | } 101 | 102 | func TestSingleTransferFileDirectoryCreation(t *testing.T) { 103 | expect := assert.New(t) 104 | strategy := prepareStrategy("src/*", "dst") 105 | expect.Nil(strategy.PerformSingleTransfer("src/test-dir/test-dir-file.txt")) 106 | stat, err := strategy.DestinationPattern.Fs.Stat("dst/test-dir") 107 | expect.True(stat.IsDir()) 108 | expect.NoError(err) 109 | expect.Len(strategy.TransferredDirectories, 0) 110 | } 111 | 112 | func TestSingleTransferTimes(t *testing.T) { 113 | expect := assert.New(t) 114 | 115 | layout := "2006-01-02T15:04:05.000Z" 116 | timeAsStr := "2014-11-12T11:45:26.371Z" 117 | referenceTime, _ := time.Parse(layout, timeAsStr) 118 | 119 | strategy := prepareStrategy("src/*", "dst") 120 | strategy.KeepTimes = true 121 | strategy.SourcePattern.Fs.Chtimes("src/test-dir", referenceTime, referenceTime) 122 | 123 | expect.Nil(strategy.PerformSingleTransfer("src/test-dir/test-dir-file.txt")) 124 | 125 | stat, _ := strategy.DestinationPattern.Fs.Stat("dst/test-dir") 126 | expect.Equal(referenceTime, stat.ModTime()) 127 | } 128 | 129 | func TestMultiTransfer(t *testing.T) { 130 | expect := assert.New(t) 131 | strategy := prepareStrategy("src/*", "dst/$1") 132 | 133 | toTransfer := []string{"src", "src/test-dir"} 134 | 135 | expect.NoError(strategy.Perform(toTransfer)) 136 | stat, err := strategy.DestinationPattern.Fs.Stat("dst/test-dir") 137 | expect.True(stat.IsDir()) 138 | expect.NoError(err) 139 | expect.Len(strategy.TransferredDirectories, 2) 140 | } 141 | 142 | 143 | func TestDryRun(t *testing.T) { 144 | expect := assert.New(t) 145 | strategy := prepareStrategy("src/*", "dst/$1") 146 | strategy.DryRun = true 147 | toTransfer := []string{"src", "src/test-dir"} 148 | 149 | expect.NoError(strategy.Perform(toTransfer)) 150 | stat, err := strategy.DestinationPattern.Fs.Stat("dst/test-dir") 151 | expect.Nil(stat) 152 | expect.True(os.IsNotExist(err)) 153 | expect.Len(strategy.TransferredDirectories, 0) 154 | } 155 | 156 | /**** COPY TESTS ****/ 157 | 158 | type FakeObserver struct { 159 | designpattern.ObserverInterface 160 | messages []string 161 | } 162 | 163 | func (ph *FakeObserver) Notify(a ...interface{}) { 164 | ph.messages = append(ph.messages, a[0].(string)) 165 | } 166 | 167 | func prepareFilesystemTest(src, srcContent, dst, dstContent string) (*pattern.SourcePattern, *pattern.DestinationPattern) { 168 | appFS := afero.NewMemMapFs() 169 | afero.WriteFile(appFS, src, []byte(srcContent), 0644) 170 | 171 | if dstContent != "" { 172 | afero.WriteFile(appFS, dst, []byte(dstContent), 0644) 173 | } 174 | 175 | return pattern.NewSourcePattern(appFS, ""), pattern.NewDestinationPattern(appFS, "") 176 | } 177 | 178 | func TestCopyNewFile(t *testing.T) { 179 | expect := assert.New(t) 180 | 181 | srcFile := "test1-src.txt" 182 | srcContents := "this is a file without existing destination" 183 | destinationFile := "test1-dst.txt" 184 | 185 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, "") 186 | 187 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 188 | subject.ProgressHandler = transfer.NewCopyProgressHandler(2, 1*time.Nanosecond) 189 | observer := &FakeObserver{} 190 | subject.RegisterObserver(observer) 191 | 192 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 193 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 194 | expect.Equal(nil, err) 195 | 196 | dstContents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 197 | expect.Equal(srcContents, string(dstContents)) 198 | 199 | expect.Len(observer.messages, 2) 200 | expect.Equal("\r[> ] 0.00%", observer.messages[0]) 201 | expect.Equal("\r[====================>] 100.00%", observer.messages[1][0:32]) 202 | } 203 | 204 | func TestCopyLargerSourceError(t *testing.T) { 205 | expect := assert.New(t) 206 | 207 | srcFile := "test-src.txt" 208 | srcContents := "this is a small src with larger dst" 209 | destinationFile := "test-dst.txt" 210 | dstContents := "this is a dst that is larger than its source and therefore cannot be copied" 211 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, dstContents) 212 | 213 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 214 | 215 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 216 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 217 | expect.Error(err) 218 | contents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 219 | expect.Equal(dstContents, string(contents)) 220 | } 221 | 222 | func TestCopyPartial(t *testing.T) { 223 | expect := assert.New(t) 224 | 225 | srcFile := "test-src.txt" 226 | srcContents := "this is the full content of a file with a partial existing destination" 227 | destinationFile := "test-dst.txt" 228 | dstContents := "this is the full content of a file with a partial" 229 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, dstContents) 230 | 231 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 232 | 233 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 234 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 235 | expect.Equal(nil, err) 236 | contents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 237 | expect.Equal(srcContents, string(contents)) 238 | } 239 | 240 | func TestCopyExistingCompleted(t *testing.T) { 241 | expect := assert.New(t) 242 | srcFile := "test-src.txt" 243 | srcContents := "this is a file where src and dst are fully equal" 244 | destinationFile := "test-dst.txt" 245 | dstContents := "this is a file where src and dst are fully equal" 246 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, dstContents) 247 | 248 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 249 | 250 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 251 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 252 | expect.Equal(nil, err) 253 | contents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 254 | expect.Equal(srcContents, string(contents)) 255 | } 256 | 257 | func TestCopyZeroBytesFile(t *testing.T) { 258 | expect := assert.New(t) 259 | 260 | srcFile := "test-src.txt" 261 | srcContents := "" 262 | destinationFile := "test-dst.txt" 263 | dstContents := "" 264 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, dstContents) 265 | 266 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 267 | 268 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 269 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 270 | expect.Equal(nil, err) 271 | contents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 272 | expect.Equal(srcContents, string(contents)) 273 | _, err = subject.SourcePattern.Fs.Stat(destinationFile) 274 | expect.Nil(err) 275 | } 276 | 277 | func TestCleanupIsAlwaysNil(t *testing.T) { 278 | expect := assert.New(t) 279 | srcFile := "test-src.txt" 280 | srcContents := "" 281 | destinationFile := "test-dst.txt" 282 | dstContents := "" 283 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, dstContents) 284 | 285 | subject, _ := transfer.NewTransferStrategy(transfer.CopyResumed, sourcePattern, destinationPattern) 286 | 287 | expect.Nil(subject.Cleanup()) 288 | } 289 | 290 | func TestMoveFile(t *testing.T) { 291 | expect := assert.New(t) 292 | 293 | srcFile := "test1-src.txt" 294 | srcContents := "this is a file without existing destination" 295 | destinationFile := "test1-dst.txt" 296 | 297 | sourcePattern, destinationPattern := prepareFilesystemTest(srcFile, srcContents, destinationFile, "") 298 | 299 | subject, _ := transfer.NewTransferStrategy(transfer.Move, sourcePattern, destinationPattern) 300 | srcStats, _ := subject.SourcePattern.Fs.Stat(srcFile) 301 | err := subject.PerformFileTransfer(srcFile, destinationFile, srcStats) 302 | expect.Equal(nil, err) 303 | 304 | dstContents, _ := afero.ReadFile(subject.SourcePattern.Fs, destinationFile) 305 | expect.Equal(srcContents, string(dstContents)) 306 | 307 | srcStats, err = subject.SourcePattern.Fs.Stat(srcFile) 308 | 309 | expect.Nil(srcStats) 310 | expect.True(os.IsNotExist(err)) 311 | 312 | } 313 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # graft 2 | graft is a command line utility to search and transfer files including an adhoc sftp server. 3 | 4 | ## Download and Setup 5 | ### Binary releases 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
Windows DownloadMacOS DownloadLinux Download
Windows DownloadMacOS DownloadLinux Download
23 | 24 | #### [☑️ Checksums](https://github.com/sandreas/graft/releases/download/v0.2.1/graft_0.2.1_sha256_checksums.txt) 25 | 26 | #### [📋 Show all releases](https://github.com/sandreas/graft/releases/latest) 27 | 28 | ### Features 29 | - Finding and transferring files via glob like patterns (`graft find data/*.jp*g`) 30 | - Finding and transferring files via real regular expressions (`graft find data/\.*\.jpe?g --regex`) 31 | - Provide additional filters, e.g. skip files olter than 2 days (`graft find * --max-age=2d`) 32 | - Copy and resume partially transferred files automatically with no parameters needed (`graft copy src/*.jp*g dst/`) 33 | - Exporting and importing file lists (`graft find data/*.jp*g --export=listing.txt`) 34 | - Providing and receive files over network via sftp server (`graft serve *.jpg`) 35 | 36 | ### No release for your platform - go get graft 37 | If you would like to use **graft** on an unsupported platform, you can try the go package manager. 38 | After [installing go](https://golang.org/doc/install) and adding the go binary to your PATH, install graft with following command: 39 | 40 | ``` 41 | go get github.com/sandreas/graft 42 | ``` 43 | 44 | If compilation succeeds, you can use `graft` from the command line. 45 | 46 | ### Update via go get 47 | 48 | To force an update of the graft sources, simply add the `-u` flag 49 | ``` 50 | go get -u github.com/sandreas/graft 51 | ``` 52 | 53 | ## Quickstart 54 | 55 | ### Important notes: 56 | - Every action is performed **recursively by default**, so you **do NOT** need to provide any flags for this (e.g. `-R`) 57 | - For file transfer commands, it usually is a good idea to use the `--dry-run` option, to see what `graft` is going to do 58 | - Special chars `\ + * ? ( ) | [ ] { } ^ $` have to be quoted with backslash in patterns (e.g `graft find '/tmp/video*\(2016\)'`) 59 | - **Linux and Unix:** Use **single quotes (')** to encapsulate patterns to prevent shell expansion 60 | - **Windows:** Use **double quotes (")** to encapsulate patterns, since single quotes are treated as chars 61 | 62 | ### Examples 63 | ``` 64 | # recursively search all jpg files in current directory and export a textfile 65 | graft find '*.jpg' --export-to=all-jpg-files.txt 66 | ``` 67 | 68 | ``` 69 | # recursively copy all png files in data/ to /tmp 70 | graft copy 'data/*.png' '/tmp/' 71 | ``` 72 | 73 | ``` 74 | # move all jpeg files in /tmp/ to _new. (dry-run), e.g. /tmp/DSC0008.jpeg => /tmp/DSC0008_new.jpeg 75 | graft move '/tmp/(*).(jpeg)' '/home/johndoe/pictures/$1_new.$2' --dry-run 76 | ``` 77 | 78 | ``` 79 | # copy all jpeg files in /tmp/ to _new. 80 | graft copy '/tmp/(*).(jpeg)' '/home/johndoe/pictures/$1_new.$2' 81 | ``` 82 | 83 | ``` 84 | # start an sftp server promoting all txt files in data/ in a chroot environment via zeroconf 85 | graft serve 'data/*.txt' --password=graft 86 | ``` 87 | 88 | ``` 89 | # Receive all files from a graft server running on 192.168.0.150 90 | graft receive --host=192.168.0.150 --password=graft 91 | ``` 92 | 93 | ### Network transfer 94 | **graft** is designed for easy transferring files from one host to another. To achieve this, you can use `graft serve` and `graft receive`. 95 | 96 | To transfer all jpg files in `/tmp` from **host A** to **host B**, all you have to do is the following: 97 | 98 | Host A: 99 | ``` 100 | graft serve '/tmp/*.jpg' 101 | ``` 102 | 103 | **graft** will prompt for a password, run an sftp server and promote it via zeroconf. 104 | 105 | The sftp server uses following defaults: 106 | - Port: 2022 107 | - Username: graft 108 | - Listen address: 0.0.0.0 109 | 110 | 111 | To receive all files, all you have to do is: 112 | 113 | ``` 114 | graft receive 115 | ``` 116 | 117 | in the destination directory on any other host within the same network and type in your password. Partially copied files will be resumed. 118 | 119 | 120 | ## Usage Details 121 | 122 | **graft** internally uses a combination of glob pattern conversion and regular expressions for matching and replacing file names. 123 | 124 | 125 | ### ***find*** 126 | 127 | The find command is used to find files. In this mode **graft** recursively lists all matching files and directories in all subdirectories, so it can also be used as a search tool like `find` on unix systems. 128 | 129 | #### Examples 130 | 131 | Recursive listing of all jpg files in /tmp directory using a simple glob pattern: 132 | ``` 133 | graft find '/tmp/*.jpg' 134 | ``` 135 | 136 | Using some regex-logic to find jpeg files, too: 137 | ``` 138 | graft find '/tmp/*.jp[e]?g' 139 | ``` 140 | 141 | Exporting all results to a text file, one line for each find: 142 | ``` 143 | graft find '/tmp/*.jpg' --export-to="~/jpg-in-tmp.txt" 144 | ``` 145 | 146 | Finding all files that are between 3 and 5 days old: 147 | ``` 148 | graft find '/tmp/*.jpg' --min-age=3d --max-age=5d 149 | ``` 150 | 151 | See **[Option reference](#option-reference)** for more info. 152 | 153 | ### ***serve*** 154 | 155 | The serve command is used to provide files via sftp. Similar to find, all matching files are provided via sftp. You can now use a sftp client like `Filezilla` or `WinSCP` to download files from the serving host. 156 | 157 | Additionally, graft uses mdns/zeroconf by default to announce the sftp server within the current network, so that `graft receive` finds the graft server automatically and downloads all provided files. 158 | 159 | Provide all jpg files in /tmp: 160 | ``` 161 | graft serve '/tmp/*.jpg' 162 | ``` 163 | 164 | By default graft serve will provide all files in the current directory: 165 | ``` 166 | graft serve 167 | 168 | # is the same as 169 | 170 | graft serve . 171 | ``` 172 | 173 | To login with `FileZilla`, you have to use the correct protocol and port, e.g.: 174 | 175 | - Server: sftp://192.168.0.150 176 | - Username: graft (if you did not change the defaults) 177 | - Password: \ 178 | - Port: 2022 (if you did not change the defaults) 179 | 180 | See **[Option reference](#option-reference)** for more info. 181 | 182 | 183 | ### ***receive*** 184 | **graft** can receive files from a graft server. It should find its pairing host automatically via zeroconf, but you can also specify the host to receive from. 185 | 186 | Lookup host via zeroconf and receive files to current directory: 187 | 188 | ``` 189 | graft receive 190 | ``` 191 | 192 | Specify host and port: 193 | 194 | ``` 195 | graft receive --host=192.168.1.111 --port=2023 196 | ``` 197 | 198 | You can also specify a receive pattern and a destination pattern, to receive only matching files. 199 | 200 | Receive only jpg files and put them into /tmp/jpgs/ (directory structure is preserved): 201 | 202 | ``` 203 | graft receive '*.jpg' '/tmp/jpgs/' 204 | ``` 205 | 206 | See **[Option reference](#option-reference)** for more info. 207 | 208 | 209 | ### ***delete*** 210 | 211 | You can also delete files. Be careful with this command. **graft** takes no prisoners and offers no apologies. 212 | Because of this, by default you have to confirm the deletion process. With `--force` the confirmation is skipped. 213 | 214 | ``` 215 | graft delete '/tmp/*.jpg' --min-age=3d --max-age=5d 216 | ``` 217 | 218 | 219 | See **[Option reference](#option-reference)** for more info. 220 | 221 | ### ***copy*** 222 | **graft** is a powerful copy tool. It can copy files recursively and resumes partially transferred files by default. 223 | 224 | Recursive copy every jpg file from `/tmp` to `/home/johndoe/pictures` (dry-run) 225 | 226 | ``` 227 | graft copy '/tmp/*.jpg' '/home/johndoe/pictures/$1' --dry-run 228 | ``` 229 | 230 | #### Submatches and more complex examples 231 | 232 | As a result of using regular expressions internally, you can use `()` in combination with `$` to create submatches, e.g.: 233 | 234 | ``` 235 | graft copy '/tmp/(*).(jpeg)' '/home/johndoe/pictures/$1_new.$2' 236 | ``` 237 | 238 | will copy following source files to their destination: 239 | 240 | ``` 241 | /tmp/test.jpeg => /home/johndoe/pictures/test_new.jpeg 242 | /tmp/subdir/other.jpeg => /home/johndoe/pictures/subdir/other_new.jpeg 243 | ``` 244 | 245 | If you do not specify a submatch using `()`, the whole pattern is treated as submatch. 246 | 247 | ``` 248 | graft copy '/tmp/*.jpg' '/tmp/copy/' 249 | 250 | # is same as 251 | 252 | graft copy '/tmp/(*.jpg)' '/tmp/copy/' 253 | ``` 254 | 255 | You can also use pipes to match multiple variants of char combinations: 256 | ``` 257 | graft '/tmp/(*.)(jpg|png)' '/home/johndoe/pictures/$1$2' 258 | ``` 259 | 260 | This will copy following source files to their destination: 261 | ``` 262 | /tmp/test.jpg => /home/johndoe/pictures/test.jpg 263 | /tmp/subdir/other.PNG => /home/johndoe/pictures/subdir/other.PNG 264 | ``` 265 | 266 | If you would like to match on of these chars `\ + * ? ( ) | [ ] { } ^ $` in patterns, they have to be quoted via backslash: 267 | ``` 268 | graft copy '/tmp/*\(2016\)' '/home/johndoe/' 269 | ``` 270 | 271 | This means that on windows, you have to escape backslashes, when using patterns: 272 | ``` 273 | graft copy "folder\*example*\\*.jpg" "otherfolder\$1" 274 | ``` 275 | Will copy all jpg files of every subdirectory matching `example` to `otherfolder`. In existing directory names, backslashes need not to be escaped. 276 | Since **graft** also works with slashes on windows, it is recommended to use slashes to prevent unexpected behaviour. 277 | 278 | 279 | ### ***move*** 280 | **graft** can also move files recursively. It works exactly like copy except of moving files to its destination, instead of making a copy. 281 | 282 | ### Reference 283 | 284 | ``` 285 | Usage: graft SOURCE [OPTIONS] 286 | or graft SOURCE DESTINATION [OPTIONS] 287 | 288 | Positional arguments: 289 | SOURCE Source file, directory or pattern 290 | DESTINATION Destination file, directory or pattern (only available on transfer actions) 291 | 292 | COMMANDS: 293 | find, f find files 294 | serve, s serve files via sftp server 295 | copy, c, cp copy files from a source to a destination 296 | move, m, mv move files from a source to a destination 297 | delete, d, rm delete files recursively 298 | receive, r receive files from a graft server 299 | help, h Shows a list of commands or help for one command 300 | 301 | 302 | GLOBAL OPTIONS: 303 | --help, -h show help 304 | --version, -v print the version 305 | 306 | GLOBAL ACTION OPTIONS: 307 | --quiet, -q do not show any output 308 | --force, -f force the requested action - even if it might be not a good idea 309 | --debug debug mode with logging to Stdout and into $HOME/.graft/application.log 310 | --regex use a real regex instead of glob patterns (e.g. src/.*\.jpg) 311 | --case-sensitive be case sensitive when matching files and folders 312 | --max-age value maximum age (e.g. 2d / 8w / 2016-12-24 / etc. ) 313 | --min-age value minimum age (e.g. 2d / 8w / 2016-12-24 / etc. ) 314 | --max-size value maximum size in bytes or format string (e.g. 2G / 8M / 1000K etc. ) 315 | --min-size value minimum size in bytes or format string (e.g. 2G / 8M / 1000K etc. ) 316 | --export-to value export found matches to a text file - one line per item (can also be used as save cache for large scans) 317 | --files-from value import found matches from file - one line per item (can also be used as load cache for large scans) 318 | 319 | FIND OPTIONS: 320 | --show-matches show regex matches for search pattern ($1=filename) 321 | --client client mode - act as sftp client and search files remotely instead of local search 322 | --host value Specify the hostname for the server (client mode only) 323 | --port value Specifiy server port (used for server- and client mode) (default: 2022) 324 | --username value Specify server username (used in server- and client mode) (default: "graft") 325 | --password value Specify server password (used for server- and client mode) 326 | 327 | SERVE OPTIONS: 328 | --host value Specify the hostname for the server (client mode only) 329 | --port value Specifiy server port (used for server- and client mode) (default: 2022) 330 | --username value Specify server username (used in server- and client mode) (default: "graft") 331 | --password value Specify server password (used for server- and client mode) 332 | --no-zeroconf do not use mdns/zeroconf to publish multicast sftp server (graft receive will not work without parameters) 333 | 334 | COPY OPTIONS: 335 | --times transfer source modify times to destination 336 | --dry-run simulation mode - shows output but files remain unaffected 337 | 338 | MOVE OPTIONS: 339 | --times transfer source modify times to destination 340 | --dry-run simulation mode - shows output but files remain unaffected 341 | 342 | DELETE OPTIONS: 343 | --dry-run simulation mode - shows output but files remain unaffected 344 | 345 | RECEIVE OPTIONS: 346 | --dry-run simulation mode - shows output but files remain unaffected 347 | --times transfer source modify times to destination 348 | --host value Specify the hostname for the server (client mode only) 349 | --port value Specifiy server port (used for server- and client mode) (default: 2022) 350 | --username value Specify server username (used in server- and client mode) (default: "graft") 351 | --password value Specify server password (used for server- and client mode) 352 | ``` 353 | 354 | The parameters `--min-age` and `--max-age` take duration or date strings to specify the age. Valid formats for age parameters, used like --min-age=X are: 355 | 356 | ``` 357 | 1s => 1 second 358 | 2m => 2 minutes 359 | 3h => 3 hours 360 | 4d => 4 days 361 | 5w => 5 weeks 362 | 6mon => 6 months 363 | 7y => 7 years 364 | 2006-01-02 => exact date 2006-01-02 00:00:00 365 | 2006-01-02T15:04:05.000Z => exact date 2006-01-02 15:04:05 366 | ``` 367 | 368 | The parameters `--min-size` and `--max-size` take size in bytes or size strings. Valid formats for size parameters, used like --min-size=X are: 369 | 370 | ``` 371 | 1 => 1 byte 372 | 2K => 2 KiB 373 | 3M => 3 MiB 374 | 4G => 4 GiB 375 | 5T => 5 TiB 376 | ``` 377 | 378 | 379 | # development 380 | 381 | ***graft*** is developed in go and you can use the default `go build` command to create a working binary: 382 | 383 | ``` 384 | git clone https://github.com/sandreas/graft.git 385 | cd graft 386 | 387 | go build 388 | ``` 389 | 390 | If the build is successful, the directory should contain a binary named `graft` or `graft.exe` on windows systems 391 | 392 | 393 | If you prefer to do a full release for all supported plattforms, use goreleaser: 394 | 395 | ``` 396 | git clone https://github.com/sandreas/graft.git 397 | cd graft 398 | go get github.com/goreleaser/goreleaser 399 | 400 | # for stable releases 401 | goreleaser 402 | 403 | # for current branch releases 404 | goreleaser --snapshot 405 | ``` 406 | 407 | Your release files are placed in `dist`. 408 | --------------------------------------------------------------------------------