├── go.mod ├── stm ├── adapter.go ├── doc.go ├── utils_test.go ├── adapter_buffer.go ├── builder_test.go ├── builder_indexurl.go ├── ping.go ├── builder_indexfile.go ├── adapter_file.go ├── _adapter_s3.go ├── namer.go ├── builder.go ├── builder_file.go ├── consts.go ├── options.go ├── sitemap.go ├── builder_url.go ├── location.go ├── utils.go ├── sitemap_test.go └── builder_url_test.go ├── .travis.yml ├── go.sum ├── .gitignore ├── LICENSE └── README.md /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ikeikeikeike/go-sitemap-generator/v2 2 | 3 | go 1.9 4 | 5 | require ( 6 | github.com/beevik/etree v1.1.0 7 | github.com/clbanning/mxj v1.8.3 8 | github.com/fatih/structs v1.1.0 9 | ) 10 | -------------------------------------------------------------------------------- /stm/adapter.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import "regexp" 4 | 5 | // GzipPtn determines gzip file. 6 | var GzipPtn = regexp.MustCompile(".gz$") 7 | 8 | // Adapter provides interface for writes some kind of sitemap. 9 | type Adapter interface { 10 | Write(loc *Location, data []byte) 11 | Bytes() [][]byte 12 | } 13 | -------------------------------------------------------------------------------- /stm/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package stm has almost the same specific condition for sitemap xml. 3 | 4 | sitemap guidelines: https://support.google.com/webmasters/answer/183668 5 | 6 | Number of URLs = 50,000 7 | 8 | File size ( uncompressed ) = 50MB 9 | 10 | http://godoc.org/github.com/ikeikeikeike/go-sitemap-generator/stm 11 | */ 12 | package stm 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.9.x 5 | - 1.10.x 6 | - 1.11.x 7 | - tip 8 | 9 | matrix: 10 | allow_failures: 11 | - go: tip 12 | fast_finish: true 13 | 14 | install: 15 | - go get golang.org/x/tools/cmd/cover 16 | - go list -f '{{range .Imports}}{{.}} {{end}}' ./... | xargs go get -d -v 17 | - go list -f '{{range .TestImports}}{{.}} {{end}}' ./... | xargs go get -d -v 18 | 19 | gobuild_args: -v -race 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= 2 | github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= 3 | github.com/clbanning/mxj v1.8.3 h1:2r/KCJi52w2MRz+K+UMa/1d7DdCjnLqYJfnbr7dYNWI= 4 | github.com/clbanning/mxj v1.8.3/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= 5 | github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= 6 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | .DS_Store 27 | /vendor 28 | /tags 29 | /sitemap.rb 30 | /sitemap.go 31 | /.bundle/ 32 | /Gemfile 33 | /Gemfile.lock 34 | /public/ 35 | /requirements.txt 36 | /tmp 37 | -------------------------------------------------------------------------------- /stm/utils_test.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestMergeMap(t *testing.T) { 9 | var src, dst, expect [][]interface{} 10 | src = [][]interface{}{{"loc", "1"}, {"changefreq", "2"}, {"mobile", true}, {"host", "http://google.com"}} 11 | dst = [][]interface{}{{"host", "http://example.com"}} 12 | expect = [][]interface{}{{"loc", "1"}, {"changefreq", "2"}, {"mobile", true}, {"host", "http://google.com"}} 13 | 14 | src = MergeMap(src, dst) 15 | 16 | if !reflect.DeepEqual(src, expect) { 17 | t.Fatalf("Failed to maps merge: deferrent map \n%#v\n and \n%#v\n", src, expect) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /stm/adapter_buffer.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import "bytes" 4 | 5 | // NewBufferAdapter returns the created the BufferAdapter's pointer 6 | func NewBufferAdapter() *BufferAdapter { 7 | adapter := &BufferAdapter{} 8 | return adapter 9 | } 10 | 11 | // BufferAdapter provides implementation for the Adapter interface. 12 | type BufferAdapter struct { 13 | bufs []*bytes.Buffer // TODO: contains with filename 14 | } 15 | 16 | // Bytes gets written content. 17 | func (adp *BufferAdapter) Bytes() [][]byte { 18 | bufs := make([][]byte, len(adp.bufs)) 19 | 20 | for i, buf := range adp.bufs { 21 | bufs[i] = buf.Bytes() 22 | } 23 | return bufs 24 | } 25 | 26 | // Write will create sitemap xml file into the file systems. 27 | func (adp *BufferAdapter) Write(loc *Location, data []byte) { 28 | adp.bufs = append(adp.bufs, bytes.NewBuffer(data)) 29 | } 30 | -------------------------------------------------------------------------------- /stm/builder_test.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestURLType(t *testing.T) { 9 | url := URL{{"loc", "1"}, {"host", "http://example.com"}} 10 | expect := URL{{"loc", "http://example.com/1"}, {"host", "http://example.com"}} 11 | 12 | url = url.URLJoinBy("loc", "host", "loc") 13 | 14 | if !reflect.DeepEqual(url, expect) { 15 | t.Fatalf("Failed to join url in URL type: deferrent URL %v and %v", url, expect) 16 | } 17 | 18 | url = URL{{"loc", "1"}, {"host", "http://example.com"}, {"mobile", true}} 19 | expect = URL{{"loc", "http://example.com/1/true"}, {"host", "http://example.com"}, {"mobile", true}} 20 | 21 | url.BungURLJoinBy("loc", "host", "loc", "mobile") 22 | 23 | if !reflect.DeepEqual(url, expect) { 24 | t.Fatalf("Failed to join url in URL type: deferrent URL %v and %v", url, expect) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /stm/builder_indexurl.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/beevik/etree" 7 | ) 8 | 9 | // NewSitemapIndexURL and NewSitemapURL are almost the same behavior. 10 | func NewSitemapIndexURL(opts *Options, url URL) SitemapURL { 11 | return &sitemapIndexURL{opts: opts, data: url} 12 | } 13 | 14 | // sitemapIndexURL and sitemapURL are almost the same behavior. 15 | type sitemapIndexURL struct { 16 | opts *Options 17 | data URL 18 | } 19 | 20 | // XML and sitemapIndexURL.XML are almost the same behavior. 21 | func (su *sitemapIndexURL) XML() []byte { 22 | doc := etree.NewDocument() 23 | sitemap := doc.CreateElement("sitemap") 24 | 25 | SetBuilderElementValue(sitemap, su.data, "loc") 26 | 27 | if _, ok := SetBuilderElementValue(sitemap, su.data, "lastmod"); !ok { 28 | lastmod := sitemap.CreateElement("lastmod") 29 | lastmod.SetText(time.Now().Format(time.RFC3339)) 30 | } 31 | 32 | if su.opts.pretty { 33 | doc.Indent(2) 34 | } 35 | buf := poolBuffer.Get() 36 | doc.WriteTo(buf) 37 | 38 | bytes := buf.Bytes() 39 | poolBuffer.Put(buf) 40 | 41 | return bytes 42 | } 43 | -------------------------------------------------------------------------------- /stm/ping.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // PingSearchEngines requests some ping server from it calls Sitemap.PingSearchEngines. 10 | func PingSearchEngines(opts *Options, urls ...string) { 11 | urls = append(urls, []string{ 12 | "http://www.google.com/webmasters/tools/ping?sitemap=%s", 13 | "http://www.bing.com/webmaster/ping.aspx?siteMap=%s", 14 | }...) 15 | sitemapURL := opts.IndexLocation().URL() 16 | 17 | bufs := len(urls) 18 | does := make(chan string, bufs) 19 | client := http.Client{Timeout: time.Duration(5 * time.Second)} 20 | 21 | for _, url := range urls { 22 | go func(baseurl string) { 23 | url := fmt.Sprintf(baseurl, sitemapURL) 24 | println("Ping now:", url) 25 | 26 | resp, err := client.Get(url) 27 | if err != nil { 28 | does <- fmt.Sprintf("[E] Ping failed: %s (URL:%s)", 29 | err, url) 30 | return 31 | } 32 | defer resp.Body.Close() 33 | 34 | does <- fmt.Sprintf("Successful ping of `%s`", url) 35 | }(url) 36 | } 37 | 38 | for i := 0; i < bufs; i++ { 39 | println(<-does) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Tatsuo Ikeda / ikeikeikeike 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /stm/builder_indexfile.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import "bytes" 4 | 5 | // NewBuilderIndexfile returns the created the BuilderIndexfile's pointer 6 | func NewBuilderIndexfile(opts *Options, loc *Location) *BuilderIndexfile { 7 | return &BuilderIndexfile{opts: opts, loc: loc} 8 | } 9 | 10 | // BuilderIndexfile provides implementation for the Builder interface. 11 | type BuilderIndexfile struct { 12 | opts *Options 13 | loc *Location 14 | content []byte 15 | linkcnt int 16 | totalcnt int 17 | } 18 | 19 | // Add method joins old bytes with creates bytes by it calls from Sitemap.Finalize method. 20 | func (b *BuilderIndexfile) Add(link interface{}) BuilderError { 21 | bldr := link.(*BuilderFile) 22 | bldr.Write() 23 | 24 | smu := NewSitemapIndexURL(b.opts, URL{{"loc", bldr.loc.URL()}}) 25 | b.content = append(b.content, smu.XML()...) 26 | 27 | b.totalcnt += bldr.linkcnt 28 | b.linkcnt++ 29 | return nil 30 | } 31 | 32 | // Content and BuilderFile.Content are almost the same behavior. 33 | func (b *BuilderIndexfile) Content() []byte { 34 | return b.content 35 | } 36 | 37 | // XMLContent and BuilderFile.XMLContent share almost the same behavior. 38 | func (b *BuilderIndexfile) XMLContent() []byte { 39 | c := bytes.Join(bytes.Fields(IndexXMLHeader), []byte(" ")) 40 | c = append(append(c, b.Content()...), IndexXMLFooter...) 41 | 42 | return c 43 | } 44 | 45 | // Write and Builderfile.Write are almost the same behavior. 46 | func (b *BuilderIndexfile) Write() { 47 | c := b.XMLContent() 48 | 49 | b.loc.Write(c, b.linkcnt) 50 | } 51 | -------------------------------------------------------------------------------- /stm/adapter_file.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "compress/gzip" 5 | "log" 6 | "os" 7 | ) 8 | 9 | // NewFileAdapter returns the created the FileAdapter's pointer 10 | func NewFileAdapter() *FileAdapter { 11 | adapter := &FileAdapter{} 12 | return adapter 13 | } 14 | 15 | // FileAdapter provides implementation for the Adapter interface. 16 | type FileAdapter struct{} 17 | 18 | // Bytes gets written content. 19 | func (adp *FileAdapter) Bytes() [][]byte { 20 | // TODO 21 | return nil 22 | } 23 | 24 | // Write will create sitemap xml file into the file systems. 25 | func (adp *FileAdapter) Write(loc *Location, data []byte) { 26 | dir := loc.Directory() 27 | fi, err := os.Stat(dir) 28 | if err != nil { 29 | _ = os.MkdirAll(dir, 0755) 30 | } else if !fi.IsDir() { 31 | log.Fatalf("[F] %s should be a directory", dir) 32 | } 33 | 34 | file, _ := os.OpenFile(loc.Path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) 35 | fi, err = file.Stat() 36 | if err != nil { 37 | log.Fatalf("[F] %s file not exists", loc.Path()) 38 | } else if !fi.Mode().IsRegular() { 39 | log.Fatalf("[F] %s should be a filename", loc.Path()) 40 | } 41 | 42 | if GzipPtn.MatchString(loc.Path()) { 43 | adp.gzip(file, data) 44 | } else { 45 | adp.plain(file, data) 46 | } 47 | } 48 | 49 | // gzip will create sitemap file as a gzip. 50 | func (adp *FileAdapter) gzip(file *os.File, data []byte) { 51 | gz := gzip.NewWriter(file) 52 | defer gz.Close() 53 | gz.Write(data) 54 | } 55 | 56 | // plain will create uncompressed file. 57 | func (adp *FileAdapter) plain(file *os.File, data []byte) { 58 | file.Write(data) 59 | defer file.Close() 60 | } 61 | -------------------------------------------------------------------------------- /stm/_adapter_s3.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io" 7 | "log" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/credentials" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 13 | ) 14 | 15 | // S3Adapter provides implementation for the Adapter interface. 16 | type S3Adapter struct { 17 | Region string 18 | Bucket string 19 | ACL string 20 | Creds *credentials.Credentials 21 | } 22 | 23 | // Bytes gets written content. 24 | func (adp *S3Adapter) Bytes() [][]byte { 25 | // TODO 26 | return nil 27 | } 28 | 29 | // Write will create sitemap xml file into the s3. 30 | func (adp *S3Adapter) Write(loc *Location, data []byte) { 31 | var reader io.Reader = bytes.NewReader(data) 32 | 33 | if GzipPtn.MatchString(loc.Filename()) { 34 | var writer *io.PipeWriter 35 | 36 | reader, writer = io.Pipe() 37 | go func() { 38 | gz := gzip.NewWriter(writer) 39 | io.Copy(gz, bytes.NewReader(data)) 40 | 41 | gz.Close() 42 | writer.Close() 43 | }() 44 | } 45 | 46 | creds := adp.Creds 47 | if creds == nil { 48 | creds = credentials.NewEnvCredentials() 49 | } 50 | creds.Get() 51 | 52 | sess := session.New(&aws.Config{ 53 | Credentials: creds, Region: &adp.Region}) 54 | 55 | uploader := s3manager.NewUploader(sess) 56 | _, err := uploader.Upload(&s3manager.UploadInput{ 57 | Bucket: aws.String(adp.Bucket), 58 | Key: aws.String(loc.PathInPublic()), 59 | ACL: aws.String(adp.ACL), 60 | Body: reader, 61 | }) 62 | 63 | if err != nil { 64 | log.Fatal("[F] S3 Upload file Error:", err) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /stm/namer.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | // NewNamer returns created the Namer's pointer 9 | func NewNamer(opts *NOpts) *Namer { 10 | if opts.extension == "" { 11 | opts.extension = ".xml.gz" 12 | } 13 | 14 | namer := &Namer{opts: opts} 15 | namer.Reset() 16 | return namer 17 | } 18 | 19 | // NOpts Namer's option 20 | type NOpts struct { 21 | base string // filename base 22 | zero int 23 | extension string 24 | start int 25 | } 26 | 27 | // Namer provides sitemap's filename as a file number counter. 28 | type Namer struct { 29 | count int 30 | opts *NOpts 31 | } 32 | 33 | // String returns that combines filename base and file extension. 34 | func (n *Namer) String() string { 35 | ext := n.opts.extension 36 | if n.count == 0 { 37 | return fmt.Sprintf("%s%s", n.opts.base, ext) 38 | } 39 | return fmt.Sprintf("%s%d%s", n.opts.base, n.count, ext) 40 | } 41 | 42 | // Reset will initialize to zero value on Namer's counter. 43 | func (n *Namer) Reset() { 44 | n.count = n.opts.zero 45 | } 46 | 47 | // IsStart confirms that this struct has zero value. 48 | func (n *Namer) IsStart() bool { 49 | return n.count == n.opts.zero 50 | } 51 | 52 | // Next is going to go to next index for filename. 53 | func (n *Namer) Next() *Namer { 54 | if n.IsStart() { 55 | n.count = n.opts.start 56 | } else { 57 | n.count++ 58 | } 59 | return n 60 | } 61 | 62 | // Previous is going to go to previous index for filename. 63 | func (n *Namer) Previous() *Namer { 64 | if n.IsStart() { 65 | log.Fatal("[F] Already at the start of the series") 66 | } 67 | if n.count <= n.opts.start { 68 | n.count = n.opts.zero 69 | } else { 70 | n.count-- 71 | } 72 | return n 73 | } 74 | -------------------------------------------------------------------------------- /stm/builder.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import "fmt" 4 | 5 | var poolBuffer = NewBufferPool() 6 | 7 | // BuilderError provides interface for it can confirm the error in some difference. 8 | type BuilderError interface { 9 | error 10 | FullError() bool 11 | } 12 | 13 | // Builder provides interface for adds some kind of url sitemap. 14 | type Builder interface { 15 | XMLContent() []byte 16 | Content() []byte 17 | Add(interface{}) BuilderError 18 | Write() 19 | } 20 | 21 | // SitemapURL provides generated xml interface. 22 | type SitemapURL interface { 23 | XML() []byte 24 | } 25 | 26 | // Attrs defines for xml attribute. 27 | type Attrs []interface{} 28 | 29 | // Attr defines for xml attribute. 30 | type Attr map[string]string 31 | 32 | // URL User should use this typedef in main func. 33 | type URL [][]interface{} 34 | 35 | // URLJoinBy that's convenient. 36 | func (u URL) URLJoinBy(key string, joins ...string) URL { 37 | var values []string 38 | for _, k := range joins { 39 | var vals interface{} 40 | for _, v := range u { 41 | if v[0] == k { 42 | vals = v[1] 43 | break 44 | } 45 | } 46 | values = append(values, fmt.Sprint(vals)) 47 | } 48 | var index int 49 | var v []interface{} 50 | for index, v = range u { 51 | if v[0] == key { 52 | break 53 | } 54 | } 55 | u[index][1] = URLJoin("", values...) 56 | return u 57 | } 58 | 59 | // BungURLJoinBy that's convenient. Though, this is Bung method. 60 | func (u *URL) BungURLJoinBy(key string, joins ...string) { 61 | orig := *u 62 | 63 | var values []string 64 | for _, k := range joins { 65 | var vals interface{} 66 | for _, v := range *u { 67 | if v[0] == k { 68 | vals = v[1] 69 | break 70 | } 71 | } 72 | values = append(values, fmt.Sprint(vals)) 73 | } 74 | var index int 75 | var v []interface{} 76 | for index, v = range *u { 77 | if v[0] == key { 78 | break 79 | } 80 | } 81 | orig[index][1] = URLJoin("", values...) 82 | *u = orig 83 | } 84 | 85 | // type News map[string]interface{} 86 | -------------------------------------------------------------------------------- /stm/builder_file.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | ) 7 | 8 | // builderFileError is implementation for the BuilderError interface. 9 | type builderFileError struct { 10 | error 11 | full bool 12 | } 13 | 14 | // FullError returns true if a sitemap xml had been limit size. 15 | func (e *builderFileError) FullError() bool { 16 | return e.full 17 | } 18 | 19 | // NewBuilderFile returns the created the BuilderFile's pointer 20 | func NewBuilderFile(opts *Options, loc *Location) *BuilderFile { 21 | b := &BuilderFile{opts: opts, loc: loc} 22 | b.clear() 23 | return b 24 | } 25 | 26 | // BuilderFile provides implementation for the Builder interface. 27 | type BuilderFile struct { 28 | opts *Options 29 | loc *Location 30 | content []byte 31 | linkcnt int 32 | newscnt int 33 | } 34 | 35 | // Add method joins old bytes with creates bytes by it calls from Sitemap.Add method. 36 | func (b *BuilderFile) Add(url interface{}) BuilderError { 37 | u := MergeMap(url.(URL), 38 | URL{{"host", b.loc.opts.defaultHost}}, 39 | ) 40 | 41 | b.linkcnt++ 42 | 43 | smu, err := NewSitemapURL(b.opts, u) 44 | if err != nil { 45 | log.Fatalf("[F] Sitemap: %s", err) 46 | } 47 | 48 | bytes := smu.XML() 49 | 50 | if !b.isFileCanFit(bytes) { 51 | return &builderFileError{error: err, full: true} 52 | } 53 | 54 | b.content = append(b.content, bytes...) 55 | 56 | return nil 57 | } 58 | 59 | // isFileCanFit checks bytes to bigger than consts values. 60 | func (b *BuilderFile) isFileCanFit(bytes []byte) bool { 61 | r := len(append(b.content, bytes...)) < MaxSitemapFilesize 62 | r = r && b.linkcnt < MaxSitemapLinks 63 | return r && b.newscnt < MaxSitemapNews 64 | } 65 | 66 | // clear will initialize xml content. 67 | func (b *BuilderFile) clear() { 68 | b.content = make([]byte, 0, MaxSitemapFilesize) 69 | } 70 | 71 | // Content will return pooled bytes on content attribute. 72 | func (b *BuilderFile) Content() []byte { 73 | return b.content 74 | } 75 | 76 | // XMLContent will return an XML of the sitemap built 77 | func (b *BuilderFile) XMLContent() []byte { 78 | c := bytes.Join(bytes.Fields(XMLHeader), []byte(" ")) 79 | c = append(append(c, b.Content()...), XMLFooter...) 80 | 81 | return c 82 | } 83 | 84 | // Write will write pooled bytes with header and footer to 85 | // Location path for output sitemap file. 86 | func (b *BuilderFile) Write() { 87 | b.loc.ReserveName() 88 | 89 | c := b.XMLContent() 90 | 91 | b.loc.Write(c, b.linkcnt) 92 | b.clear() 93 | } 94 | -------------------------------------------------------------------------------- /stm/consts.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | const ( 4 | // MaxSitemapFiles defines max sitemap links per index file 5 | MaxSitemapFiles = 50000 6 | // MaxSitemapLinks defines max links per sitemap 7 | MaxSitemapLinks = 50000 8 | // MaxSitemapImages defines max images per url 9 | MaxSitemapImages = 1000 10 | // MaxSitemapNews defines max news sitemap per index_file 11 | MaxSitemapNews = 1000 12 | // MaxSitemapFilesize defines file size for sitemap. 13 | MaxSitemapFilesize = 50000000 // bytes 14 | ) 15 | 16 | const ( 17 | // SchemaGeo exists for geo sitemap 18 | SchemaGeo = "http://www.google.com/geo/schemas/sitemap/1.0" 19 | // SchemaImage exists for image sitemap 20 | SchemaImage = "http://www.google.com/schemas/sitemap-image/1.1" 21 | // SchemaMobile exists for mobile sitemap 22 | SchemaMobile = "http://www.google.com/schemas/sitemap-mobile/1.0" 23 | // SchemaNews exists for news sitemap 24 | SchemaNews = "http://www.google.com/schemas/sitemap-news/0.9" 25 | // SchemaPagemap exists for pagemap sitemap 26 | SchemaPagemap = "http://www.google.com/schemas/sitemap-pagemap/1.0" 27 | // SchemaVideo exists for video sitemap 28 | SchemaVideo = "http://www.google.com/schemas/sitemap-video/1.1" 29 | ) 30 | 31 | var ( 32 | // IndexXMLHeader exists for create sitemap xml as a specific sitemap document. 33 | IndexXMLHeader = []byte(` 34 | `) 40 | // IndexXMLFooter and IndexXMLHeader will used from user together . 41 | IndexXMLFooter = []byte("") 42 | ) 43 | 44 | var ( 45 | // XMLHeader exists for create sitemap xml as a specific sitemap document. 46 | XMLHeader = []byte(` 47 | `) 60 | // XMLFooter and XMLHeader will used from user together . 61 | XMLFooter = []byte("") 62 | ) 63 | -------------------------------------------------------------------------------- /stm/options.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | // NewOptions returns the created the Options's pointer 4 | func NewOptions() *Options { 5 | // Default values 6 | return &Options{ 7 | defaultHost: "http://www.example.com", 8 | sitemapsHost: "", // http://s3.amazonaws.com/sitemap-generator/, 9 | publicPath: "public/", 10 | sitemapsPath: "sitemaps/", 11 | filename: "sitemap", 12 | verbose: true, 13 | compress: true, 14 | pretty: false, 15 | adp: NewFileAdapter(), 16 | } 17 | } 18 | 19 | // Options exists for the Sitemap struct. 20 | type Options struct { 21 | defaultHost string 22 | sitemapsHost string 23 | publicPath string 24 | sitemapsPath string 25 | filename string 26 | verbose bool 27 | compress bool 28 | pretty bool 29 | adp Adapter 30 | nmr *Namer 31 | loc *Location 32 | } 33 | 34 | // SetDefaultHost sets that arg from Sitemap.Finalize method 35 | func (opts *Options) SetDefaultHost(host string) { 36 | opts.defaultHost = host 37 | } 38 | 39 | // SetSitemapsHost sets that arg from Sitemap.SetSitemapsHost method 40 | func (opts *Options) SetSitemapsHost(host string) { 41 | opts.sitemapsHost = host 42 | } 43 | 44 | // SetSitemapsPath sets that arg from Sitemap.SetSitemapsPath method. 45 | func (opts *Options) SetSitemapsPath(path string) { 46 | opts.sitemapsPath = path 47 | } 48 | 49 | // SetPublicPath sets that arg from Sitemap.SetPublicPath method 50 | func (opts *Options) SetPublicPath(path string) { 51 | opts.publicPath = path 52 | } 53 | 54 | // SetFilename sets that arg from Sitemap.SetFilename method 55 | func (opts *Options) SetFilename(filename string) { 56 | opts.filename = filename 57 | } 58 | 59 | // SetVerbose sets that arg from Sitemap.SetVerbose method 60 | func (opts *Options) SetVerbose(verbose bool) { 61 | opts.verbose = verbose 62 | } 63 | 64 | // SetCompress sets that arg from Sitemap.SetCompress method 65 | func (opts *Options) SetCompress(compress bool) { 66 | opts.compress = compress 67 | } 68 | 69 | // SetPretty option sets pretty option to Options struct which allows pretty formatting to output files. 70 | func (opts *Options) SetPretty(pretty bool) { 71 | opts.pretty = pretty 72 | } 73 | 74 | // SetAdapter sets that arg from Sitemap.SetAdapter method 75 | func (opts *Options) SetAdapter(adp Adapter) { 76 | opts.adp = adp 77 | } 78 | 79 | // SitemapsHost sets that arg from Sitemap.SitemapsHost method 80 | func (opts *Options) SitemapsHost() string { 81 | if opts.sitemapsHost != "" { 82 | return opts.sitemapsHost 83 | } 84 | return opts.defaultHost 85 | } 86 | 87 | // Location returns the Location's pointer with 88 | // set option to arguments for Builderfile struct. 89 | func (opts *Options) Location() *Location { 90 | return NewLocation(opts) 91 | } 92 | 93 | // IndexLocation returns the Location's pointer with 94 | // set option to arguments for BuilderIndexfile struct. 95 | func (opts *Options) IndexLocation() *Location { 96 | o := opts.Clone() 97 | o.nmr = NewNamer(&NOpts{base: opts.filename}) 98 | return NewLocation(o) 99 | } 100 | 101 | // Namer returns Namer's pointer cache. If didn't create that yet, 102 | // It also returns created Namer's pointer. 103 | func (opts *Options) Namer() *Namer { 104 | if opts.nmr == nil { 105 | opts.nmr = NewNamer(&NOpts{base: opts.filename, zero: 1, start: 2}) 106 | } 107 | return opts.nmr 108 | } 109 | 110 | // Clone method returns it copied myself. 111 | func (opts *Options) Clone() *Options { 112 | o := *opts 113 | return &o 114 | } 115 | -------------------------------------------------------------------------------- /stm/sitemap.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "log" 5 | "runtime" 6 | ) 7 | 8 | // NewSitemap returns the created the Sitemap's pointer 9 | func NewSitemap(maxProc int) *Sitemap { 10 | log.SetFlags(log.LstdFlags | log.Llongfile) 11 | if maxProc < 1 || maxProc > runtime.NumCPU() { 12 | maxProc = runtime.NumCPU() 13 | } 14 | log.Printf("Max processors %d\n", maxProc) 15 | runtime.GOMAXPROCS(maxProc) 16 | 17 | sm := &Sitemap{ 18 | opts: NewOptions(), 19 | } 20 | return sm 21 | } 22 | 23 | // Sitemap provides interface for create sitemap xml file and that has convenient interface. 24 | // And also needs to use first this struct if it wants to use this package. 25 | type Sitemap struct { 26 | opts *Options 27 | bldr Builder 28 | bldrs Builder 29 | } 30 | 31 | // SetDefaultHost is your website's host name 32 | func (sm *Sitemap) SetDefaultHost(host string) { 33 | sm.opts.SetDefaultHost(host) 34 | } 35 | 36 | // SetSitemapsHost is the remote host where your sitemaps will be hosted 37 | func (sm *Sitemap) SetSitemapsHost(host string) { 38 | sm.opts.SetSitemapsHost(host) 39 | } 40 | 41 | // SetSitemapsPath sets this to a directory/path if you don't 42 | // want to upload to the root of your `SitemapsHost` 43 | func (sm *Sitemap) SetSitemapsPath(path string) { 44 | sm.opts.SetSitemapsPath(path) 45 | } 46 | 47 | // SetPublicPath is the directory to write sitemaps to locally 48 | func (sm *Sitemap) SetPublicPath(path string) { 49 | sm.opts.SetPublicPath(path) 50 | } 51 | 52 | // SetAdapter can switch output file storage. 53 | // We have S3Adapter and FileAdapter (default: FileAdapter) 54 | func (sm *Sitemap) SetAdapter(adp Adapter) { 55 | sm.opts.SetAdapter(adp) 56 | } 57 | 58 | // SetVerbose can switch verbose output to console. 59 | func (sm *Sitemap) SetVerbose(verbose bool) { 60 | sm.opts.SetVerbose(verbose) 61 | } 62 | 63 | // SetCompress can switch compress for the output file. 64 | func (sm *Sitemap) SetCompress(compress bool) { 65 | sm.opts.SetCompress(compress) 66 | } 67 | 68 | // SetPretty option allows pretty formating to the output files. 69 | func (sm *Sitemap) SetPretty(pretty bool) { 70 | sm.opts.SetPretty(pretty) 71 | } 72 | 73 | // SetFilename can apply any name in this method if you wants to change output file name 74 | func (sm *Sitemap) SetFilename(filename string) { 75 | sm.opts.SetFilename(filename) 76 | } 77 | 78 | // Create method must be that calls first this method in that before call to Add method on this struct. 79 | func (sm *Sitemap) Create() *Sitemap { 80 | sm.bldrs = NewBuilderIndexfile(sm.opts, sm.opts.IndexLocation()) 81 | return sm 82 | } 83 | 84 | // Add Should call this after call to Create method on this struct. 85 | func (sm *Sitemap) Add(url interface{}) *Sitemap { 86 | if sm.bldr == nil { 87 | sm.bldr = NewBuilderFile(sm.opts, sm.opts.Location()) 88 | } 89 | 90 | err := sm.bldr.Add(url) 91 | if err != nil { 92 | if err.FullError() { 93 | sm.Finalize() 94 | return sm.Add(url) 95 | } 96 | } 97 | 98 | return sm 99 | } 100 | 101 | // XMLContent returns the XML content of the sitemap 102 | func (sm *Sitemap) XMLContent() []byte { 103 | return sm.bldr.XMLContent() 104 | } 105 | 106 | // Finalize writes sitemap and index files if it had some 107 | // specific condition in BuilderFile struct. 108 | func (sm *Sitemap) Finalize() *Sitemap { 109 | sm.bldrs.Add(sm.bldr) 110 | sm.bldrs.Write() 111 | sm.bldr = nil 112 | return sm 113 | } 114 | 115 | // PingSearchEngines requests some ping server. 116 | // It also has that includes PingSearchEngines function. 117 | func (sm *Sitemap) PingSearchEngines(urls ...string) { 118 | PingSearchEngines(sm.opts, urls...) 119 | } 120 | -------------------------------------------------------------------------------- /stm/builder_url.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/beevik/etree" 9 | "github.com/fatih/structs" 10 | ) 11 | 12 | // URLModel is specific sample model for valuedate. 13 | // http://www.sitemaps.org/protocol.html 14 | // https://support.google.com/webmasters/answer/178636 15 | type URLModel struct { 16 | Priority float64 `valid:"float,length(0.0|1.0)"` 17 | Changefreq string `valid:"alpha(always|hourly|daily|weekly|monthly|yearly|never)"` 18 | Lastmod time.Time `valid:"-"` 19 | Expires time.Time `valid:"-"` 20 | Host string `valid:"ipv4"` 21 | Loc string `valid:"url"` 22 | Image string `valid:"url"` 23 | Video string `valid:"url"` 24 | Tag string `valid:""` 25 | Geo string `valid:""` 26 | News string `valid:"-"` 27 | Mobile bool `valid:"-"` 28 | Alternate string `valid:"-"` 29 | Alternates map[string]interface{} `valid:"-"` 30 | Pagemap map[string]interface{} `valid:"-"` 31 | } 32 | 33 | // fieldnames []string{"priority" "changefreq" "lastmod" "expires" "host" "images" 34 | // "video" "geo" "news" "videos" "mobile" "alternate" "alternates" "pagemap"} 35 | var fieldnames = ToLowerString(structs.Names(&URLModel{})) 36 | 37 | // NewSitemapURL returns the created the SitemapURL's pointer 38 | // and it validates URL types error. 39 | func NewSitemapURL(opts *Options, url URL) (SitemapURL, error) { 40 | smu := &sitemapURL{opts: opts, data: url} 41 | err := smu.validate() 42 | return smu, err 43 | } 44 | 45 | // sitemapURL provides xml validator and xml builder. 46 | type sitemapURL struct { 47 | opts *Options 48 | data URL 49 | } 50 | 51 | // validate is checking correct keys and checks the existence. 52 | // TODO: Will create value's validator 53 | func (su *sitemapURL) validate() error { 54 | var key string 55 | var invalid bool 56 | var locOk, hostOk bool 57 | 58 | for _, value := range su.data { 59 | key = value[0].(string) 60 | switch key { 61 | case "loc": 62 | locOk = true 63 | case "host": 64 | hostOk = true 65 | } 66 | 67 | invalid = true 68 | for _, name := range fieldnames { 69 | if key == name { 70 | invalid = false 71 | break 72 | } 73 | } 74 | if invalid { 75 | break 76 | } 77 | } 78 | 79 | if invalid { 80 | msg := fmt.Sprintf("Unknown map's key `%s` in URL type", key) 81 | return errors.New(msg) 82 | } 83 | if !locOk { 84 | msg := fmt.Sprintf("URL type must have `loc` map's key") 85 | return errors.New(msg) 86 | } 87 | if !hostOk { 88 | msg := fmt.Sprintf("URL type must have `host` map's key") 89 | return errors.New(msg) 90 | } 91 | return nil 92 | } 93 | 94 | // XML is building xml. 95 | func (su *sitemapURL) XML() []byte { 96 | doc := etree.NewDocument() 97 | url := doc.CreateElement("url") 98 | 99 | SetBuilderElementValue(url, su.data.URLJoinBy("loc", "host", "loc"), "loc") 100 | if _, ok := SetBuilderElementValue(url, su.data, "lastmod"); !ok { 101 | lastmod := url.CreateElement("lastmod") 102 | lastmod.SetText(time.Now().Format(time.RFC3339)) 103 | } 104 | if _, ok := SetBuilderElementValue(url, su.data, "changefreq"); !ok { 105 | changefreq := url.CreateElement("changefreq") 106 | changefreq.SetText("weekly") 107 | } 108 | if _, ok := SetBuilderElementValue(url, su.data, "priority"); !ok { 109 | priority := url.CreateElement("priority") 110 | priority.SetText("0.5") 111 | } 112 | SetBuilderElementValue(url, su.data, "expires") 113 | SetBuilderElementValue(url, su.data, "mobile") 114 | SetBuilderElementValue(url, su.data, "news") 115 | SetBuilderElementValue(url, su.data, "video") 116 | SetBuilderElementValue(url, su.data, "image") 117 | SetBuilderElementValue(url, su.data, "geo") 118 | 119 | if su.opts.pretty { 120 | doc.Indent(2) 121 | } 122 | buf := poolBuffer.Get() 123 | doc.WriteTo(buf) 124 | 125 | bytes := buf.Bytes() 126 | poolBuffer.Put(buf) 127 | 128 | return bytes 129 | } 130 | -------------------------------------------------------------------------------- /stm/location.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | ) 11 | 12 | // NewLocation returns created the Location's pointer 13 | func NewLocation(opts *Options) *Location { 14 | loc := &Location{ 15 | opts: opts, 16 | } 17 | return loc 18 | } 19 | 20 | // Location provides sitemap's path and filename on file systems 21 | // and it provides proxy for Adapter interface also. 22 | type Location struct { 23 | opts *Options 24 | nmr *Namer 25 | filename string 26 | } 27 | 28 | // Directory returns path to combine publicPath and sitemapsPath on file systems. 29 | // It also indicates where sitemap files are. 30 | func (loc *Location) Directory() string { 31 | return filepath.Join( 32 | loc.opts.publicPath, 33 | loc.opts.sitemapsPath, 34 | ) 35 | } 36 | 37 | // Path returns path to combine publicPath, sitemapsPath and Filename on file systems. 38 | // It also indicates where sitemap name is. 39 | func (loc *Location) Path() string { 40 | return filepath.Join( 41 | loc.opts.publicPath, 42 | loc.opts.sitemapsPath, 43 | loc.Filename(), 44 | ) 45 | } 46 | 47 | // PathInPublic returns path to combine sitemapsPath and Filename on website. 48 | // It also indicates where url file path is. 49 | func (loc *Location) PathInPublic() string { 50 | return filepath.Join( 51 | loc.opts.sitemapsPath, 52 | loc.Filename(), 53 | ) 54 | } 55 | 56 | // URL returns path to combine SitemapsHost, sitemapsPath and 57 | // Filename on website with it uses ResolveReference. 58 | func (loc *Location) URL() string { 59 | base, _ := url.Parse(loc.opts.SitemapsHost()) 60 | 61 | for _, ref := range []string{ 62 | loc.opts.sitemapsPath + "/", loc.Filename(), 63 | } { 64 | base, _ = base.Parse(ref) 65 | } 66 | 67 | return base.String() 68 | } 69 | 70 | // Filesize returns file size this struct has. 71 | func (loc *Location) Filesize() int64 { 72 | f, _ := os.Open(loc.Path()) 73 | defer f.Close() 74 | 75 | fi, err := f.Stat() 76 | if err != nil { 77 | return 0 78 | } 79 | 80 | return fi.Size() 81 | } 82 | 83 | // reGzip determines gzip file. 84 | var reGzip = regexp.MustCompile(`\.gz$`) 85 | 86 | // Namer returns the Namer's pointer that Options struct has. 87 | func (loc *Location) Namer() *Namer { 88 | return loc.opts.Namer() 89 | } 90 | 91 | // Filename returns sitemap filename. 92 | func (loc *Location) Filename() string { 93 | nmr := loc.Namer() 94 | if loc.filename == "" && nmr == nil { 95 | log.Fatal("[F] No filename or namer set") 96 | } 97 | 98 | if loc.filename == "" { 99 | loc.filename = nmr.String() 100 | 101 | if !loc.opts.compress { 102 | newName := reGzip.ReplaceAllString(loc.filename, "") 103 | loc.filename = newName 104 | } 105 | } 106 | return loc.filename 107 | } 108 | 109 | // ReserveName returns that sets filename if this struct didn't keep filename and 110 | // it returns reserved filename if this struct keeps filename also. 111 | func (loc *Location) ReserveName() string { 112 | nmr := loc.Namer() 113 | if nmr != nil { 114 | loc.Filename() 115 | nmr.Next() 116 | } 117 | 118 | return loc.filename 119 | } 120 | 121 | // IsReservedName confirms that keeps filename on Location.filename. 122 | func (loc *Location) IsReservedName() bool { 123 | if loc.filename == "" { 124 | return false 125 | } 126 | return true 127 | } 128 | 129 | // IsVerbose returns boolean about verbosed summary. 130 | func (loc *Location) IsVerbose() bool { 131 | return loc.opts.verbose 132 | } 133 | 134 | // Write writes sitemap and index files that used from Adapter interface. 135 | func (loc *Location) Write(data []byte, linkCount int) { 136 | 137 | loc.opts.adp.Write(loc, data) 138 | if !loc.IsVerbose() { 139 | return 140 | } 141 | 142 | output := loc.Summary(linkCount) 143 | if output != "" { 144 | println(output) 145 | } 146 | } 147 | 148 | // Summary outputs to generated file summary for console. 149 | func (loc *Location) Summary(linkCount int) string { 150 | nmr := loc.Namer() 151 | if nmr.IsStart() { 152 | return "" 153 | } 154 | 155 | out := fmt.Sprintf("%s '%d' links", 156 | loc.PathInPublic(), linkCount) 157 | 158 | size := loc.Filesize() 159 | if size <= 0 { 160 | return out 161 | } 162 | 163 | return fmt.Sprintf("%s / %d bytes", out, size) 164 | } 165 | -------------------------------------------------------------------------------- /stm/utils.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/url" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/beevik/etree" 12 | ) 13 | 14 | // BufferPool is 15 | type BufferPool struct { 16 | sync.Pool 17 | } 18 | 19 | // NewBufferPool is 20 | func NewBufferPool() *BufferPool { 21 | return &BufferPool{ 22 | Pool: sync.Pool{New: func() interface{} { 23 | b := bytes.NewBuffer(make([]byte, 256)) 24 | b.Reset() 25 | return b 26 | }}, 27 | } 28 | } 29 | 30 | // Get is 31 | func (bp *BufferPool) Get() *bytes.Buffer { 32 | return bp.Pool.Get().(*bytes.Buffer) 33 | } 34 | 35 | // Put is 36 | func (bp *BufferPool) Put(b *bytes.Buffer) { 37 | b.Reset() 38 | bp.Pool.Put(b) 39 | } 40 | 41 | // SetBuilderElementValue if it will change to struct from map if the future's 42 | // author is feeling a bothersome in this function. 43 | func SetBuilderElementValue(elm *etree.Element, data [][]interface{}, basekey string) (*etree.Element, bool) { 44 | var child *etree.Element 45 | 46 | key := basekey 47 | ts, tk := spaceDecompose(elm.Tag) 48 | _, sk := spaceDecompose(elm.Space) 49 | 50 | if elm.Tag != "" && ts != "" && tk != "" { 51 | key = fmt.Sprintf("%s:%s", elm.Space, basekey) 52 | } else if sk != "" { 53 | key = fmt.Sprintf("%s:%s", sk, basekey) 54 | } 55 | 56 | var values interface{} 57 | var found bool 58 | for _, v := range data { 59 | if v[0] == basekey { 60 | values = v[1] 61 | found = true 62 | break 63 | } 64 | } 65 | if !found { 66 | return child, false 67 | } 68 | 69 | switch value := values.(type) { 70 | case nil: 71 | default: 72 | child = elm.CreateElement(key) 73 | child.SetText(fmt.Sprint(value)) 74 | case int: 75 | child = elm.CreateElement(key) 76 | child.SetText(fmt.Sprint(value)) 77 | case string: 78 | child = elm.CreateElement(key) 79 | child.SetText(value) 80 | case float64, float32: 81 | child = elm.CreateElement(key) 82 | child.SetText(fmt.Sprint(value)) 83 | case time.Time: 84 | child = elm.CreateElement(key) 85 | child.SetText(value.Format(time.RFC3339)) 86 | case bool: 87 | _ = elm.CreateElement(fmt.Sprintf("%s:%s", key, key)) 88 | case []int: 89 | for _, v := range value { 90 | child = elm.CreateElement(key) 91 | child.SetText(fmt.Sprint(v)) 92 | } 93 | case []string: 94 | for _, v := range value { 95 | child = elm.CreateElement(key) 96 | child.SetText(v) 97 | } 98 | case []Attr: 99 | for _, attr := range value { 100 | child = elm.CreateElement(key) 101 | for k, v := range attr { 102 | child.CreateAttr(k, v) 103 | } 104 | } 105 | case Attrs: 106 | val, attrs := value[0], value[1] 107 | 108 | child, _ = SetBuilderElementValue(elm, URL{[]interface{}{basekey, val}}, basekey) 109 | switch attr := attrs.(type) { 110 | case map[string]string: 111 | for k, v := range attr { 112 | child.CreateAttr(k, v) 113 | } 114 | // TODO: gotta remove below 115 | case Attr: 116 | for k, v := range attr { 117 | child.CreateAttr(k, v) 118 | } 119 | } 120 | 121 | case interface{}: 122 | var childkey string 123 | if sk == "" { 124 | childkey = fmt.Sprintf("%s:%s", key, key) 125 | } else { 126 | childkey = fmt.Sprint(key) 127 | } 128 | 129 | switch value := values.(type) { 130 | case []URL: 131 | for _, val := range value { 132 | child := elm.CreateElement(childkey) 133 | for _, v := range val { 134 | SetBuilderElementValue(child, val, v[0].(string)) 135 | } 136 | } 137 | case URL: 138 | child := elm.CreateElement(childkey) 139 | for _, v := range value { 140 | SetBuilderElementValue(child, value, v[0].(string)) 141 | } 142 | } 143 | } 144 | return child, true 145 | } 146 | 147 | // MergeMap TODO: Slow function: It wants to change fast function 148 | func MergeMap(src, dst [][]interface{}) [][]interface{} { 149 | for _, v := range dst { 150 | found := false 151 | for _, vSrc := range src { 152 | if v[0] == vSrc[0] { 153 | found = true 154 | break 155 | } 156 | } 157 | if !found { 158 | src = append(src, v) 159 | } 160 | } 161 | return src 162 | } 163 | 164 | // ToLowerString converts lower strings from including capital or upper strings. 165 | func ToLowerString(befores []string) (afters []string) { 166 | for _, name := range befores { 167 | afters = append(afters, strings.ToLower(name)) 168 | } 169 | return afters 170 | } 171 | 172 | // URLJoin TODO: Too slowly 173 | func URLJoin(src string, joins ...string) string { 174 | var u *url.URL 175 | lastnum := len(joins) 176 | base, _ := url.Parse(src) 177 | 178 | for i, j := range joins { 179 | if !strings.HasSuffix(j, "/") && lastnum > (i+1) { 180 | j = j + "/" 181 | } 182 | 183 | u, _ = url.Parse(j) 184 | base = base.ResolveReference(u) 185 | } 186 | 187 | return base.String() 188 | } 189 | 190 | // spaceDecompose is separating strings for the SetBuilderElementValue 191 | func spaceDecompose(str string) (space, key string) { 192 | colon := strings.IndexByte(str, ':') 193 | if colon == -1 { 194 | return "", str 195 | } 196 | return str[:colon], str[colon+1:] 197 | } 198 | -------------------------------------------------------------------------------- /stm/sitemap_test.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/clbanning/mxj" 9 | ) 10 | 11 | func TestSitemapGenerator(t *testing.T) { 12 | buf := BufferAdapter{} 13 | 14 | sm := NewSitemap(0) 15 | sm.SetPretty(true) 16 | sm.SetVerbose(false) 17 | sm.SetAdapter(&buf) 18 | 19 | sm.Create() 20 | for i := 1; i <= 10; i++ { 21 | sm.Add(URL{{"loc", "home"}, {"changefreq", "always"}, {"mobile", true}, {"lastmod", "2018-10-28T17:56:02+09:00"}}) 22 | sm.Add(URL{{"loc", "readme"}, {"lastmod", "2018-10-28T17:56:02+09:00"}}) 23 | sm.Add(URL{{"loc", "aboutme"}, {"priority", 0.1}, {"lastmod", "2018-10-28T17:56:02+09:00"}}) 24 | } 25 | sm.Finalize() 26 | 27 | buffers := buf.Bytes() 28 | 29 | data := buffers[len(buffers)-1] 30 | expect := []byte(` 31 | 32 | 33 | 34 | http://www.example.com/sitemaps//sitemap1.xml.gz 35 | 2018-10-28T17:37:21+09:00 36 | 37 | `) 38 | 39 | mdata, _ := mxj.NewMapXml(data) 40 | mexpect, _ := mxj.NewMapXml(expect) 41 | mdata.Remove("sitemapindex.sitemap.lastmod") 42 | mexpect.Remove("sitemapindex.sitemap.lastmod") 43 | 44 | if !reflect.DeepEqual(mdata, mexpect) { 45 | t.Error(`Failed to generate sitemapindex`) 46 | } 47 | 48 | bufs := bytes.Buffer{} 49 | for _, buf := range buffers[:len(buffers)-1] { 50 | bufs.Write(buf) 51 | } 52 | data = bufs.Bytes() 53 | expect = []byte(` 54 | 55 | http://www.example.com/home 56 | 2018-10-28T17:56:02+09:00 57 | always 58 | 0.5 59 | 60 | 61 | 62 | http://www.example.com/readme 63 | 2018-10-28T17:56:02+09:00 64 | weekly 65 | 0.5 66 | 67 | 68 | http://www.example.com/aboutme 69 | 2018-10-28T17:56:02+09:00 70 | weekly 71 | 0.1 72 | 73 | 74 | http://www.example.com/home 75 | 2018-10-28T17:56:02+09:00 76 | always 77 | 0.5 78 | 79 | 80 | 81 | http://www.example.com/readme 82 | 2018-10-28T17:56:02+09:00 83 | weekly 84 | 0.5 85 | 86 | 87 | http://www.example.com/aboutme 88 | 2018-10-28T17:56:02+09:00 89 | weekly 90 | 0.1 91 | 92 | 93 | http://www.example.com/home 94 | 2018-10-28T17:56:02+09:00 95 | always 96 | 0.5 97 | 98 | 99 | 100 | http://www.example.com/readme 101 | 2018-10-28T17:56:02+09:00 102 | weekly 103 | 0.5 104 | 105 | 106 | http://www.example.com/aboutme 107 | 2018-10-28T17:56:02+09:00 108 | weekly 109 | 0.1 110 | 111 | 112 | http://www.example.com/home 113 | 2018-10-28T17:56:02+09:00 114 | always 115 | 0.5 116 | 117 | 118 | 119 | http://www.example.com/readme 120 | 2018-10-28T17:56:02+09:00 121 | weekly 122 | 0.5 123 | 124 | 125 | http://www.example.com/aboutme 126 | 2018-10-28T17:56:02+09:00 127 | weekly 128 | 0.1 129 | 130 | 131 | http://www.example.com/home 132 | 2018-10-28T17:56:02+09:00 133 | always 134 | 0.5 135 | 136 | 137 | 138 | http://www.example.com/readme 139 | 2018-10-28T17:56:02+09:00 140 | weekly 141 | 0.5 142 | 143 | 144 | http://www.example.com/aboutme 145 | 2018-10-28T17:56:02+09:00 146 | weekly 147 | 0.1 148 | 149 | 150 | http://www.example.com/home 151 | 2018-10-28T17:56:02+09:00 152 | always 153 | 0.5 154 | 155 | 156 | 157 | http://www.example.com/readme 158 | 2018-10-28T17:56:02+09:00 159 | weekly 160 | 0.5 161 | 162 | 163 | http://www.example.com/aboutme 164 | 2018-10-28T17:56:02+09:00 165 | weekly 166 | 0.1 167 | 168 | 169 | http://www.example.com/home 170 | 2018-10-28T17:56:02+09:00 171 | always 172 | 0.5 173 | 174 | 175 | 176 | http://www.example.com/readme 177 | 2018-10-28T17:56:02+09:00 178 | weekly 179 | 0.5 180 | 181 | 182 | http://www.example.com/aboutme 183 | 2018-10-28T17:56:02+09:00 184 | weekly 185 | 0.1 186 | 187 | 188 | http://www.example.com/home 189 | 2018-10-28T17:56:02+09:00 190 | always 191 | 0.5 192 | 193 | 194 | 195 | http://www.example.com/readme 196 | 2018-10-28T17:56:02+09:00 197 | weekly 198 | 0.5 199 | 200 | 201 | http://www.example.com/aboutme 202 | 2018-10-28T17:56:02+09:00 203 | weekly 204 | 0.1 205 | 206 | 207 | http://www.example.com/home 208 | 2018-10-28T17:56:02+09:00 209 | always 210 | 0.5 211 | 212 | 213 | 214 | http://www.example.com/readme 215 | 2018-10-28T17:56:02+09:00 216 | weekly 217 | 0.5 218 | 219 | 220 | http://www.example.com/aboutme 221 | 2018-10-28T17:56:02+09:00 222 | weekly 223 | 0.1 224 | 225 | 226 | http://www.example.com/home 227 | 2018-10-28T17:56:02+09:00 228 | always 229 | 0.5 230 | 231 | 232 | 233 | http://www.example.com/readme 234 | 2018-10-28T17:56:02+09:00 235 | weekly 236 | 0.5 237 | 238 | 239 | http://www.example.com/aboutme 240 | 2018-10-28T17:56:02+09:00 241 | weekly 242 | 0.1 243 | 244 | 245 | `) 246 | 247 | mdata, _ = mxj.NewMapXml(data) 248 | mexpect, _ = mxj.NewMapXml(expect) 249 | 250 | if !reflect.DeepEqual(mdata, mexpect) { 251 | t.Error(`Failed to generate dataindex`) 252 | } 253 | 254 | } 255 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | A go-sitemap-generator is the easiest way to generate Sitemaps in Go. 3 | 4 | As of version 2.0.0, This Repo is available as a [Go module](https://github.com/golang/go/wiki/Modules). 5 | 6 | 7 | [![GoDoc](https://godoc.org/github.com/ikeikeikeike/go-sitemap-generator/stm?status.svg)](https://godoc.org/github.com/ikeikeikeike/go-sitemap-generator/stm) [![Build Status](https://travis-ci.org/ikeikeikeike/go-sitemap-generator.svg)](https://travis-ci.org/ikeikeikeike/go-sitemap-generator) 8 | 9 | ```go 10 | package main 11 | 12 | import ( 13 | "github.com/ikeikeikeike/go-sitemap-generator/v2/stm" 14 | ) 15 | 16 | 17 | func main() { 18 | sm := stm.NewSitemap(1) 19 | 20 | // Create method must be called first before adding entries to 21 | // the sitemap. 22 | sm.Create() 23 | 24 | sm.Add(stm.URL{{"loc", "home"}, {"changefreq", "always"}, {"mobile", true}}) 25 | sm.Add(stm.URL{{"loc", "readme"}}) 26 | sm.Add(stm.URL{{"loc", "aboutme"}, {"priority", 0.1}}) 27 | 28 | sm.Finalize().PingSearchEngines() 29 | } 30 | ``` 31 | 32 | Then 33 | 34 | ```console 35 | $ go build 36 | ``` 37 | 38 | #### Installation (Legacy download instead of a [Go module](https://github.com/golang/go/wiki/Modules).) 39 | 40 | ```console 41 | $ go get gopkg.in/ikeikeikeike/go-sitemap-generator.v1/stm 42 | $ go get gopkg.in/ikeikeikeike/go-sitemap-generator.v2/stm 43 | ``` 44 | 45 | ### Features 46 | 47 | Current Features or To-Do 48 | 49 | - [ ] Supports: generate kind of some sitemaps. 50 | - [x] [News sitemaps](#news-sitemaps) 51 | - [x] [Video sitemaps](#video-sitemaps) 52 | - [x] [Image sitemaps](#image-sitemaps) 53 | - [x] [Geo sitemaps](#geo-sitemaps) 54 | - [x] [Mobile sitemaps](#mobile-sitemaps) 55 | - [ ] PageMap sitemap 56 | - [x] Alternate Links 57 | - [ ] Supports: adapters for sitemap storage. 58 | - [x] Filesystem 59 | - [x] [S3](#upload-sitemap-to-s3) 60 | - [x] [Customizable sitemap working](#preventing-output) 61 | - [x] [Notifies search engines (Google, Bing) of new sitemaps](#pinging-search-engines) 62 | - [x] [Gives you complete control over your sitemap contents and naming scheme](#full-example) 63 | 64 | 65 | ## Getting Started 66 | 67 | ### Setting concurrency 68 | To disable concurrency, set number of CPUs to 1. 69 | ```go 70 | sm := stm.NewSitemap(1) 71 | ``` 72 | 73 | If you want to set max CPUs that are available, set number of CPUs <= 0. 74 | ```go 75 | sm := stm.NewSitemap(0) 76 | ``` 77 | 78 | ### Preventing Output 79 | 80 | To disable all non-essential output you can set `sm.SetVerbose` to `false`. 81 | To disable output inline use the following: 82 | 83 | ```go 84 | sm := stm.NewSitemap(1) 85 | sm.SetVerbose(false) 86 | ``` 87 | 88 | ### Pinging Search Engines 89 | 90 | PingSearchEngines notifies search engines of changes once a sitemap 91 | has been generated or changed. The library will append Google and Bing to any engines passed in to the function. 92 | 93 | ```go 94 | sm.Finalize().PingSearchEngines() 95 | ``` 96 | 97 | If you want to add `new search engine`, you can pass that in to the function: 98 | 99 | ```go 100 | sm.Finalize().PingSearchEngines("http://newengine.com/ping?url=%s") 101 | ``` 102 | 103 | ### Options 104 | 105 | ```go 106 | // Your website's host name 107 | sm.SetDefaultHost("http://www.example.com") 108 | 109 | // The remote host where your sitemaps will be hosted 110 | sm.SetSitemapsHost("http://s3.amazonaws.com/sitemap-generator/") 111 | 112 | // The directory to write sitemaps to locally 113 | sm.SetPublicPath("tmp/") 114 | 115 | // Set this to a directory/path if you don't want to upload to the root of your `SitemapsHost` 116 | sm.SetSitemapsPath("sitemaps/") 117 | 118 | // Struct of `S3Adapter` 119 | sm.SetAdapter(&stm.S3Adapter{Region: "ap-northeast-1", Bucket: "your-bucket", ACL: "public-read"}) 120 | 121 | // Change the output filename 122 | sm.SetFilename("new_filename") 123 | ``` 124 | 125 | ### Upload sitemap to S3 126 | 127 | Recently I disabled this module [here](https://github.com/ikeikeikeike/go-sitemap-generator/blob/master/stm/_adapter_s3.go). 128 | 129 | ```go 130 | package main 131 | 132 | import ( 133 | "github.com/aws/aws-sdk-go/aws/credentials" 134 | "github.com/ikeikeikeike/go-sitemap-generator/stm" 135 | ) 136 | 137 | func main() { 138 | sm := stm.NewSitemap(1) 139 | sm.SetDefaultHost("http://example.com") 140 | sm.SetSitemapsPath("sitemap-generator") // default: public 141 | sm.SetSitemapsHost("http://s3.amazonaws.com/sitemap-generator/") 142 | sm.SetAdapter(&stm.S3Adapter{ 143 | Region: "ap-northeast-1", 144 | Bucket: "your-bucket", 145 | ACL: "public-read", 146 | Creds: credentials.NewEnvCredentials(), 147 | }) 148 | 149 | sm.Create() 150 | 151 | sm.Add(stm.URL{{"loc", "home"}, {"changefreq", "always"}, {"mobile", true}}) 152 | sm.Add(stm.URL{{"loc", "readme"}}) 153 | sm.Add(stm.URL{{"loc", "aboutme"}, {"priority", 0.1}}) 154 | 155 | sm.Finalize().PingSearchEngines() 156 | } 157 | ``` 158 | 159 | ### News sitemaps 160 | 161 | ```go 162 | sm.Add(stm.URL{ 163 | {"loc", "/news"}, 164 | {"news", stm.URL{ 165 | {"publication", stm.URL{ 166 | {"name", "Example"}, 167 | {"language", "en"}, 168 | }, 169 | }, 170 | {"title", "My Article"}, 171 | {"keywords", "my article, articles about myself"}, 172 | {"stock_tickers", "SAO:PETR3"}, 173 | {"publication_date", "2011-08-22"}, 174 | {"access", "Subscription"}, 175 | {"genres", "PressRelease"}, 176 | },},}) 177 | ``` 178 | 179 | Look at [Creating a Google News Sitemap](https://support.google.com/news/publisher/answer/74288) as required. 180 | 181 | ### Video sitemaps 182 | 183 | ```go 184 | sm.Add(stm.URL{ 185 | {"loc", "/videos"}, 186 | {"video", stm.URL{ 187 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 188 | {"title", "Title"}, 189 | {"description", "Description"}, 190 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 191 | {"category", "Category"}, 192 | {"tag", []string{"one", "two", "three"}}, 193 | {"player_loc", stm.Attrs{"https://example.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26", map[string]string{"allow_embed": "Yes", "autoplay": "autoplay=1"}},}, 194 | }, 195 | }, 196 | }) 197 | ``` 198 | 199 | Look at [Video sitemaps](https://support.google.com/webmasters/answer/80471) as required. 200 | 201 | ### Image sitemaps 202 | 203 | ```go 204 | sm.Add(stm.URL{ 205 | {"loc", "/images"}, 206 | {"image", []stm.URL{ 207 | {{"loc", "http://www.example.com/image.png"}, {"title", "Image"}}, 208 | {{"loc", "http://www.example.com/image1.png"}, {"title", "Image1"}}, 209 | },}, 210 | }) 211 | 212 | ``` 213 | 214 | Look at [Image sitemaps](https://support.google.com/webmasters/answer/178636) as required. 215 | 216 | ### Geo sitemaps 217 | 218 | ```go 219 | sm.Add(stm.URL{ 220 | {"loc", "/geos"}, 221 | {"geo", stm.URL{ 222 | {"format", "kml"}, 223 | },}, 224 | }) 225 | ``` 226 | 227 | Couldn't find Geo sitemaps example, although it's similar to: 228 | 229 | ```xml 230 | 231 | /geos 232 | 233 | kml 234 | 235 | 236 | ``` 237 | 238 | ### Mobile sitemaps 239 | 240 | ```go 241 | sm.Add(stm.URL{{"loc", "mobiles"}, {"mobile", true}}) 242 | ``` 243 | 244 | Look at [Feature phone sitemaps](https://support.google.com/webmasters/answer/6082207) as required. 245 | 246 | 247 | ### Full example 248 | 249 | ```go 250 | package main 251 | 252 | import ( 253 | "github.com/ikeikeikeike/go-sitemap-generator/stm" 254 | ) 255 | 256 | func main() { 257 | sm := stm.NewSitemap(0) 258 | sm.SetDefaultHost("http://yourhost.com") 259 | sm.SetSitemapsHost("http://s3.amazonaws.com/sitemaps/") 260 | sm.SetSitemapsPath("sitemaps/") 261 | sm.SetFilename("anothername") 262 | sm.SetCompress(true) 263 | sm.SetVerbose(true) 264 | sm.SetAdapter(&stm.S3Adapter{Region: "ap-northeast-1", Bucket: "your-bucket"}) 265 | 266 | sm.Create() 267 | 268 | sm.Add(stm.URL{{"loc", "/home"}, {"changefreq", "daily"}}) 269 | 270 | sm.Add(stm.URL{{"loc", "/abouts"}, {"mobile", true}}) 271 | 272 | sm.Add(stm.URL{{"loc", "/news"}, 273 | {"news", stm.URL{ 274 | {"publication", stm.URL{ 275 | {"name", "Example"}, 276 | {"language", "en"}, 277 | }, 278 | }, 279 | {"title", "My Article"}, 280 | {"keywords", "my article, articles about myself"}, 281 | {"stock_tickers", "SAO:PETR3"}, 282 | {"publication_date", "2011-08-22"}, 283 | {"access", "Subscription"}, 284 | {"genres", "PressRelease"}, 285 | },}, 286 | }) 287 | 288 | sm.Add(stm.URL{{"loc", "/images"}, 289 | {"image", []stm.URL{ 290 | {{"loc", "http://www.example.com/image.png"}, {"title", "Image"}}, 291 | {{"loc", "http://www.example.com/image1.png"}, {"title", "Image1"}}, 292 | },}, 293 | }) 294 | 295 | sm.Add(stm.URL{{"loc", "/videos"}, 296 | {"video", stm.URL{ 297 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 298 | {"title", "Title"}, 299 | {"description", "Description"}, 300 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 301 | {"category", "Category"}, 302 | {"tag", []string{"one", "two", "three"}}, 303 | {"player_loc", stm.Attrs{"https://example.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26", map[string]string{"allow_embed": "Yes", "autoplay": "autoplay=1"}}}, 304 | },}, 305 | }) 306 | 307 | sm.Add(stm.URL{{"loc", "/geos"}, 308 | {"geo", stm.URL{ 309 | {"format", "kml"}, 310 | },}, 311 | }) 312 | 313 | sm.Finalize().PingSearchEngines("http://newengine.com/ping?url=%s") 314 | } 315 | ``` 316 | 317 | ### Webserver example 318 | 319 | 320 | ```go 321 | package main 322 | 323 | import ( 324 | "log" 325 | "net/http" 326 | 327 | "github.com/ikeikeikeike/go-sitemap-generator/stm" 328 | ) 329 | 330 | func buildSitemap() *stm.Sitemap { 331 | sm := stm.NewSitemap(1) 332 | sm.SetDefaultHost("http://example.com") 333 | 334 | sm.Create() 335 | sm.Add(stm.URL{{"loc", "/"}, {"changefreq", "daily"}}) 336 | 337 | // Note: Do not call `sm.Finalize()` because it flushes 338 | // the underlying data structure from memory to disk. 339 | 340 | return sm 341 | } 342 | 343 | func main() { 344 | sm := buildSitemap() 345 | 346 | mux := http.NewServeMux() 347 | mux.HandleFunc("/sitemap.xml", func(w http.ResponseWriter, r *http.Request) { 348 | // Go's webserver automatically sets the correct `Content-Type` header. 349 | w.Write(sm.XMLContent()) 350 | return 351 | }) 352 | 353 | log.Fatal(http.ListenAndServe(":8080", mux)) 354 | } 355 | ``` 356 | 357 | 358 | ### Documentation 359 | 360 | - [API Reference](https://godoc.org/github.com/ikeikeikeike/go-sitemap-generator/stm) 361 | - [sitemap_generator](http://github.com/kjvarga/sitemap_generator) 362 | 363 | ### How to test. 364 | 365 | Preparation: 366 | 367 | ```console 368 | $ go get github.com/clbanning/mxj 369 | ``` 370 | 371 | Run tests: 372 | 373 | ```console 374 | $ go test -v -cover -race ./... 375 | ``` 376 | 377 | #### Inspired by [sitemap_generator](http://github.com/kjvarga/sitemap_generator) 378 | -------------------------------------------------------------------------------- /stm/builder_url_test.go: -------------------------------------------------------------------------------- 1 | package stm 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/beevik/etree" 10 | "github.com/clbanning/mxj" 11 | ) 12 | 13 | func TestBlank(t *testing.T) { 14 | if _, err := NewSitemapURL(&Options{}, URL{}); err == nil { 15 | t.Errorf(`Failed to validate blank arg ( URL{} ): %v`, err) 16 | } 17 | } 18 | 19 | func TestItHasLocElement(t *testing.T) { 20 | if _, err := NewSitemapURL(&Options{}, URL{}); err == nil { 21 | t.Errorf(`Failed to validate about must have loc attribute in URL type ( URL{} ): %v`, err) 22 | } 23 | } 24 | 25 | func TestJustSetLocElement(t *testing.T) { 26 | smu, err := NewSitemapURL(&Options{}, URL{{"loc", "path"}, {"host", "http://example.com"}}) 27 | 28 | if err != nil { 29 | t.Fatalf(`Fatal to validate! This is a critical error: %v`, err) 30 | } 31 | 32 | doc := etree.NewDocument() 33 | doc.ReadFromBytes(smu.XML()) 34 | 35 | var elm *etree.Element 36 | url := doc.SelectElement("url") 37 | 38 | elm = url.SelectElement("loc") 39 | if elm == nil { 40 | t.Errorf(`Failed to generate xml that loc element is blank: %v`, elm) 41 | } 42 | if elm != nil && elm.Text() != "http://example.com/path" { 43 | t.Errorf(`Failed to generate xml thats deferrent value in loc element: %v`, elm.Text()) 44 | } 45 | } 46 | 47 | func TestJustSetLocElementAndThenItNeedsCompleteValues(t *testing.T) { 48 | smu, err := NewSitemapURL(&Options{}, URL{{"loc", "path"}, {"host", "http://example.com"}}) 49 | 50 | if err != nil { 51 | t.Fatalf(`Fatal to validate! This is a critical error: %v`, err) 52 | } 53 | 54 | doc := etree.NewDocument() 55 | doc.ReadFromBytes(smu.XML()) 56 | 57 | var elm *etree.Element 58 | url := doc.SelectElement("url") 59 | 60 | elm = url.SelectElement("loc") 61 | if elm == nil { 62 | t.Errorf(`Failed to generate xml that loc element is blank: %v`, elm) 63 | } 64 | if elm != nil && elm.Text() != "http://example.com/path" { 65 | t.Errorf(`Failed to generate xml thats deferrent value in loc element: %v`, elm.Text()) 66 | } 67 | 68 | elm = url.SelectElement("priority") 69 | if elm == nil { 70 | t.Errorf(`Failed to generate xml that priority element is nil: %v`, elm) 71 | } 72 | if elm != nil && elm.Text() != "0.5" { 73 | t.Errorf(`Failed to generate xml thats deferrent value in priority element: %v`, elm.Text()) 74 | } 75 | 76 | elm = url.SelectElement("changefreq") 77 | if elm == nil { 78 | t.Errorf(`Failed to generate xml that changefreq element is nil: %v`, elm) 79 | } 80 | if elm != nil && elm.Text() != "weekly" { 81 | t.Errorf(`Failed to generate xml thats deferrent value in changefreq element: %v`, elm.Text()) 82 | } 83 | 84 | elm = url.SelectElement("lastmod") 85 | if elm == nil { 86 | t.Errorf(`Failed to generate xml that lastmod element is nil: %v`, elm) 87 | } 88 | if elm != nil { 89 | if _, err := time.Parse(time.RFC3339, elm.Text()); err != nil { 90 | t.Errorf(`Failed to generate xml thats failed to parse datetime in lastmod element: %v`, err) 91 | } 92 | } 93 | } 94 | 95 | func TestSetNilValue(t *testing.T) { 96 | smu, err := NewSitemapURL(&Options{}, URL{{"loc", "path"}, {"priority", nil}, {"changefreq", nil}, {"lastmod", nil}, {"host", "http://example.com"}}) 97 | 98 | if err != nil { 99 | t.Fatalf(`Fatal to validate! This is a critical error: %v`, err) 100 | } 101 | 102 | doc := etree.NewDocument() 103 | doc.ReadFromBytes(smu.XML()) 104 | 105 | var elm *etree.Element 106 | url := doc.SelectElement("url") 107 | 108 | elm = url.SelectElement("loc") 109 | if elm == nil { 110 | t.Errorf(`Failed to generate xml that loc element is blank: %v`, elm) 111 | } 112 | if elm != nil && elm.Text() != "http://example.com/path" { 113 | t.Errorf(`Failed to generate xml thats deferrent value in loc element: %v`, elm.Text()) 114 | } 115 | 116 | elm = url.SelectElement("priority") 117 | if elm != nil { 118 | t.Errorf(`Failed to generate xml that priority element must be nil: %v`, elm) 119 | } 120 | 121 | elm = url.SelectElement("changefreq") 122 | if elm != nil { 123 | t.Errorf(`Failed to generate xml that changefreq element must be nil: %v`, elm) 124 | } 125 | 126 | elm = url.SelectElement("lastmod") 127 | if elm != nil { 128 | t.Errorf(`Failed to generate xml that lastmod element must be nil: %v`, elm) 129 | } 130 | } 131 | 132 | func TestAutoGenerateSitemapHost(t *testing.T) { 133 | smu, err := NewSitemapURL(&Options{}, URL{{"loc", "path"}, {"host", "http://example.com"}}) 134 | 135 | if err != nil { 136 | t.Fatalf(`Fatal to validate! This is a critical error: %v`, err) 137 | } 138 | 139 | doc := etree.NewDocument() 140 | doc.ReadFromBytes(smu.XML()) 141 | 142 | var elm *etree.Element 143 | url := doc.SelectElement("url") 144 | 145 | elm = url.SelectElement("loc") 146 | if elm == nil { 147 | t.Errorf(`Failed to generate xml that loc element is blank: %v`, elm) 148 | } 149 | if elm != nil && elm.Text() != "http://example.com/path" { 150 | t.Errorf(`Failed to generate xml thats deferrent value in loc element: %v`, elm.Text()) 151 | } 152 | } 153 | 154 | func TestNewsSitemaps(t *testing.T) { 155 | doc := etree.NewDocument() 156 | root := doc.CreateElement("root") 157 | 158 | data := URL{{"loc", "/news"}, {"news", URL{ 159 | {"publication", URL{ 160 | {"name", "Example"}, 161 | {"language", "en"}, 162 | }}, 163 | {"title", "My Article"}, 164 | {"keywords", "my article, articles about myself"}, 165 | {"stock_tickers", "SAO:PETR3"}, 166 | {"publication_date", "2011-08-22"}, 167 | {"access", "Subscription"}, 168 | {"genres", "PressRelease"}, 169 | }}} 170 | expect := []byte(` 171 | 172 | 173 | my article, articles about myself 174 | SAO:PETR3 175 | 2011-08-22 176 | Subscription 177 | PressRelease 178 | 179 | Example 180 | en 181 | 182 | My Article 183 | 184 | `) 185 | 186 | SetBuilderElementValue(root, data, "news") 187 | buf := &bytes.Buffer{} 188 | doc.WriteTo(buf) 189 | 190 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 191 | mexpect, _ := mxj.NewMapXml(expect) 192 | 193 | if !reflect.DeepEqual(mdata, mexpect) { 194 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 195 | } 196 | } 197 | 198 | func TestImageSitemaps(t *testing.T) { 199 | doc := etree.NewDocument() 200 | root := doc.CreateElement("root") 201 | 202 | data := URL{{"loc", "/images"}, {"image", []URL{ 203 | {{"loc", "http://www.example.com/image.png"}, {"title", "Image"}}, 204 | {{"loc", "http://www.example.com/image1.png"}, {"title", "Image1"}}, 205 | }}} 206 | expect := []byte(` 207 | 208 | 209 | http://www.example.com/image.png 210 | Image 211 | 212 | 213 | http://www.example.com/image1.png 214 | Image1 215 | 216 | `) 217 | 218 | SetBuilderElementValue(root, data, "image") 219 | buf := &bytes.Buffer{} 220 | doc.WriteTo(buf) 221 | 222 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 223 | mexpect, _ := mxj.NewMapXml(expect) 224 | 225 | if !reflect.DeepEqual(mdata, mexpect) { 226 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 227 | } 228 | } 229 | 230 | func TestVideoSitemaps(t *testing.T) { 231 | doc := etree.NewDocument() 232 | root := doc.CreateElement("root") 233 | 234 | data := URL{{"loc", "/videos"}, {"video", URL{ 235 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 236 | {"title", "Title"}, 237 | {"description", "Description"}, 238 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 239 | {"category", "Category"}, 240 | {"tag", []string{"one", "two", "three"}}, 241 | }}} 242 | 243 | expect := []byte(` 244 | 245 | 246 | http://www.example.com/video1_thumbnail.png 247 | Title 248 | Description 249 | http://www.example.com/cool_video.mpg 250 | one 251 | two 252 | three 253 | Category 254 | 255 | `) 256 | 257 | SetBuilderElementValue(root, data, "video") 258 | buf := &bytes.Buffer{} 259 | doc.WriteTo(buf) 260 | 261 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 262 | mexpect, _ := mxj.NewMapXml(expect) 263 | 264 | if !reflect.DeepEqual(mdata, mexpect) { 265 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 266 | } 267 | } 268 | 269 | func TestGeoSitemaps(t *testing.T) { 270 | doc := etree.NewDocument() 271 | root := doc.CreateElement("root") 272 | 273 | data := URL{{"loc", "/geos"}, {"geo", URL{{"format", "kml"}}}} 274 | 275 | expect := []byte(` 276 | 277 | 278 | kml 279 | 280 | `) 281 | 282 | SetBuilderElementValue(root, data, "geo") 283 | buf := &bytes.Buffer{} 284 | doc.WriteTo(buf) 285 | 286 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 287 | mexpect, _ := mxj.NewMapXml(expect) 288 | 289 | if !reflect.DeepEqual(mdata, mexpect) { 290 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 291 | } 292 | } 293 | 294 | func TestMobileSitemaps(t *testing.T) { 295 | doc := etree.NewDocument() 296 | root := doc.CreateElement("root") 297 | 298 | data := URL{{"loc", "/mobile"}, {"mobile", true}} 299 | 300 | expect := []byte(` 301 | 302 | /mobile 303 | 304 | `) 305 | 306 | SetBuilderElementValue(root, data.URLJoinBy("loc", "host", "loc"), "loc") 307 | SetBuilderElementValue(root, data, "mobile") 308 | 309 | buf := &bytes.Buffer{} 310 | doc.WriteTo(buf) 311 | 312 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 313 | mexpect, _ := mxj.NewMapXml(expect) 314 | 315 | if !reflect.DeepEqual(mdata, mexpect) { 316 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 317 | } 318 | } 319 | 320 | func TestPageMapSitemaps(t *testing.T) {} 321 | 322 | func TestAlternateLinks(t *testing.T) { 323 | doc := etree.NewDocument() 324 | root := doc.CreateElement("root") 325 | 326 | loc := "/alternates" 327 | data := URL{{"loc", loc}, {"xhtml:link", []Attr{ 328 | { 329 | "rel": "alternate", 330 | "hreflang": "zh-tw", 331 | "href": loc + "?locale=zh-tw", 332 | }, 333 | { 334 | "rel": "alternate", 335 | "hreflang": "en-us", 336 | "href": loc + "?locale=en-us", 337 | }, 338 | }}} 339 | 340 | expect := []byte(` 341 | 342 | /alternates 343 | 344 | 345 | `) 346 | 347 | SetBuilderElementValue(root, data.URLJoinBy("loc", "host", "loc"), "loc") 348 | SetBuilderElementValue(root, data, "xhtml:link") 349 | 350 | buf := &bytes.Buffer{} 351 | doc.WriteTo(buf) 352 | 353 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 354 | mexpect, _ := mxj.NewMapXml(expect) 355 | 356 | if !reflect.DeepEqual(mdata, mexpect) { 357 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 358 | } 359 | } 360 | 361 | func TestAttr(t *testing.T) { 362 | doc := etree.NewDocument() 363 | root := doc.CreateElement("root") 364 | 365 | data := URL{{"loc", "/videos"}, {"video", URL{ 366 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 367 | {"title", "Title"}, 368 | {"description", "Description"}, 369 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 370 | {"category", "Category"}, 371 | {"tag", []string{"one", "two", "three"}}, 372 | {"player_loc", Attrs{"https://f.vimeocdn.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26", Attr{"allow_embed": "Yes", "autoplay": "autoplay=1"}}}, 373 | }}} 374 | 375 | expect := []byte(` 376 | 377 | 378 | http://www.example.com/video1_thumbnail.png 379 | Title 380 | Description 381 | http://www.example.com/cool_video.mpg 382 | one 383 | two 384 | three 385 | Category 386 | https://f.vimeocdn.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26 387 | 388 | `) 389 | 390 | SetBuilderElementValue(root, data, "video") 391 | 392 | buf := &bytes.Buffer{} 393 | // doc.Indent(2) 394 | doc.WriteTo(buf) 395 | 396 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 397 | mexpect, _ := mxj.NewMapXml(expect) 398 | 399 | // print(string(buf.Bytes())) 400 | 401 | if !reflect.DeepEqual(mdata, mexpect) { 402 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 403 | } 404 | } 405 | 406 | func TestAttrWithoutTypedef(t *testing.T) { 407 | doc := etree.NewDocument() 408 | root := doc.CreateElement("root") 409 | 410 | data := URL{{"loc", "/videos"}, {"video", URL{ 411 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 412 | {"title", "Title"}, 413 | {"description", "Description"}, 414 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 415 | {"category", "Category"}, 416 | {"tag", []string{"one", "two", "three"}}, 417 | {"player_loc", Attrs{"https://f.vimeocdn.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26", map[string]string{"allow_embed": "Yes", "autoplay": "autoplay=1"}}}, 418 | }}} 419 | 420 | expect := []byte(` 421 | 422 | 423 | http://www.example.com/video1_thumbnail.png 424 | Title 425 | Description 426 | http://www.example.com/cool_video.mpg 427 | one 428 | two 429 | three 430 | Category 431 | https://f.vimeocdn.com/p/flash/moogaloop/6.2.9/moogaloop.swf?clip_id=26 432 | 433 | `) 434 | 435 | SetBuilderElementValue(root, data, "video") 436 | 437 | buf := &bytes.Buffer{} 438 | // doc.Indent(2) 439 | doc.WriteTo(buf) 440 | 441 | mdata, _ := mxj.NewMapXml(buf.Bytes()) 442 | mexpect, _ := mxj.NewMapXml(expect) 443 | 444 | // print(string(buf.Bytes())) 445 | 446 | if !reflect.DeepEqual(mdata, mexpect) { 447 | t.Error(`Failed to generate sitemap xml thats deferrent output value in URL type`) 448 | } 449 | } 450 | 451 | func BenchmarkGenerateXML(b *testing.B) { 452 | 453 | b.ReportAllocs() 454 | b.ResetTimer() 455 | 456 | forPerformance := 500 457 | 458 | for k := 0; k <= forPerformance; k++ { 459 | for i := 1; i <= forPerformance; i++ { 460 | 461 | var smu SitemapURL 462 | var data URL 463 | 464 | data = URL{{"loc", "/mobile"}, {"mobile", true}} 465 | smu, _ = NewSitemapURL(&Options{}, data) 466 | smu.XML() 467 | 468 | data = URL{{"loc", "/images"}, {"image", []URL{ 469 | {{"loc", "http://www.example.com/image.png"}, {"title", "Image"}}, 470 | {{"loc", "http://www.example.com/image1.png"}, {"title", "Image1"}}, 471 | }}} 472 | smu, _ = NewSitemapURL(&Options{}, data) 473 | smu.XML() 474 | 475 | data = URL{{"loc", "/videos"}, {"video", URL{ 476 | {"thumbnail_loc", "http://www.example.com/video1_thumbnail.png"}, 477 | {"title", "Title"}, 478 | {"description", "Description"}, 479 | {"content_loc", "http://www.example.com/cool_video.mpg"}, 480 | {"category", "Category"}, 481 | {"tag", []string{"one", "two", "three"}}, 482 | }}} 483 | smu, _ = NewSitemapURL(&Options{}, data) 484 | smu.XML() 485 | 486 | data = URL{{"loc", "/news"}, {"news", URL{ 487 | {"publication", URL{ 488 | {"name", "Example"}, 489 | {"language", "en"}, 490 | }}, 491 | {"title", "My Article"}, 492 | {"keywords", "my article, articles about myself"}, 493 | {"stock_tickers", "SAO:PETR3"}, 494 | {"publication_date", "2011-08-22"}, 495 | {"access", "Subscription"}, 496 | {"genres", "PressRelease"}, 497 | }}} 498 | 499 | smu, _ = NewSitemapURL(&Options{}, data) 500 | smu.XML() 501 | } 502 | } 503 | } 504 | --------------------------------------------------------------------------------