├── .gitignore ├── extract-config.toml ├── read ├── store.go ├── getter.go ├── config.go ├── filestore.go ├── read.go ├── httpgetter.go └── contentful.go ├── extract ├── doc.go ├── extractor.go └── extractor_test.go ├── write ├── store.go ├── filestore.go ├── write_test.go └── write.go ├── mapper ├── mapper.go ├── types.go └── items.go ├── translate ├── config.go ├── directory.go ├── filename.go ├── yaml.go ├── toml.go ├── translate.go └── translate_test.go ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── goreleaser.yml ├── main.go └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | content 2 | dist 3 | vendor -------------------------------------------------------------------------------- /extract-config.toml: -------------------------------------------------------------------------------- 1 | encoding = "toml" 2 | -------------------------------------------------------------------------------- /read/store.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | type Store interface { 4 | ReadFromFile(path string) (result []byte, err error) 5 | } 6 | -------------------------------------------------------------------------------- /extract/doc.go: -------------------------------------------------------------------------------- 1 | // Package extract orchestrates the full process for 2 | // command line use of the Contentful Hugo Extractor. 3 | package extract 4 | -------------------------------------------------------------------------------- /read/getter.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type Getter interface { 8 | Get(url string) (result io.ReadCloser, err error) 9 | } 10 | -------------------------------------------------------------------------------- /read/config.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | type ReadConfig struct { 4 | UsePreview bool 5 | SpaceID string 6 | AccessToken string 7 | Locale string 8 | // e.g. 200 items per page 9 | } 10 | -------------------------------------------------------------------------------- /write/store.go: -------------------------------------------------------------------------------- 1 | package write 2 | 3 | import "os" 4 | 5 | type Store interface { 6 | MkdirAll(path string, perm os.FileMode) error 7 | WriteFile(filename string, data []byte, perm os.FileMode) error 8 | } 9 | -------------------------------------------------------------------------------- /read/filestore.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | import ( 4 | "io/ioutil" 5 | ) 6 | 7 | type FileStore struct{} 8 | 9 | func (fs FileStore) ReadFromFile(filename string) (body []byte, err error) { 10 | bytes, err := ioutil.ReadFile(filename) 11 | 12 | return bytes, err 13 | } 14 | -------------------------------------------------------------------------------- /read/read.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | import "errors" 4 | 5 | type Reader struct { 6 | Store Store 7 | } 8 | 9 | func (r *Reader) ViewFromFile(fileName string) (body string, err error) { 10 | result, err := r.Store.ReadFromFile(fileName) 11 | if result == nil && err == nil { 12 | err = errors.New("File is Empty") 13 | } 14 | 15 | return string(result), err 16 | } 17 | -------------------------------------------------------------------------------- /write/filestore.go: -------------------------------------------------------------------------------- 1 | package write 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | ) 7 | 8 | type FileStore struct{} 9 | 10 | func (fs FileStore) MkdirAll(path string, perm os.FileMode) error { 11 | return os.MkdirAll(path, perm) 12 | } 13 | 14 | func (fs FileStore) WriteFile(filename string, data []byte, perm os.FileMode) error { 15 | return ioutil.WriteFile(filename, data, perm) 16 | } 17 | -------------------------------------------------------------------------------- /mapper/mapper.go: -------------------------------------------------------------------------------- 1 | package mapper 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | ) 7 | 8 | func MapTypes(rc io.ReadCloser) (typeResult TypeResult, err error) { 9 | defer rc.Close() 10 | err = json.NewDecoder(rc).Decode(&typeResult) 11 | 12 | return 13 | } 14 | 15 | func MapItems(rc io.ReadCloser) (itemResult ItemResult, err error) { 16 | defer rc.Close() 17 | err = json.NewDecoder(rc).Decode(&itemResult) 18 | 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /write/write_test.go: -------------------------------------------------------------------------------- 1 | package write 2 | 3 | import "testing" 4 | 5 | func TestDirForFile(t *testing.T) { 6 | tests := []struct { 7 | input string 8 | expected string 9 | }{ 10 | {"/basic/file.txt", "/basic"}, 11 | {"./one/more/file.txt", "./one/more"}, 12 | {"./one/less/", "./one/less"}, 13 | } 14 | for _, test := range tests { 15 | result := dirForFile(test.input) 16 | if result != test.expected { 17 | t.Errorf("dirForFile(%v) incorrect, expected %v, got %v", test.input, test.expected, result) 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /read/httpgetter.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | var myClient = &http.Client{Timeout: 10 * time.Second} 11 | 12 | type HttpGetter struct { 13 | } 14 | 15 | // Get makes an http get request and throws an Error if the response 16 | // statuscode is not 200 17 | func (hg HttpGetter) Get(url string) (result io.ReadCloser, err error) { 18 | resp, err := myClient.Get(url) 19 | if resp.StatusCode != 200 && err == nil { 20 | err = fmt.Errorf("Http request failed: %s", resp.Status) 21 | } 22 | return resp.Body, err 23 | } 24 | -------------------------------------------------------------------------------- /write/write.go: -------------------------------------------------------------------------------- 1 | package write 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | type Writer struct { 10 | Store Store 11 | } 12 | 13 | func (w *Writer) SaveToFile(fileName string, output string) { 14 | var fileMode os.FileMode 15 | fileMode = 0733 16 | 17 | err := w.Store.MkdirAll(dirForFile(fileName), fileMode) 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | 22 | err = w.Store.WriteFile(fileName, []byte(output), fileMode) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | } 27 | 28 | func dirForFile(filename string) string { 29 | index := strings.LastIndex(filename, "/") 30 | return filename[0:index] 31 | } 32 | -------------------------------------------------------------------------------- /mapper/types.go: -------------------------------------------------------------------------------- 1 | package mapper 2 | 3 | import "errors" 4 | 5 | type TypeResult struct { 6 | Total int 7 | Skip int 8 | Limit int 9 | Items []Type 10 | } 11 | 12 | func (t *TypeResult) GetType(name string) (result Type, err error) { 13 | for _, el := range t.Items { 14 | if el.Sys.ID == name { 15 | return el, nil 16 | } 17 | } 18 | return Type{}, errors.New("Type not found") 19 | } 20 | 21 | type Type struct { 22 | Sys Sys 23 | Name string 24 | Fields []TypeField 25 | } 26 | 27 | type TypeField struct { 28 | ID string 29 | Name string 30 | Type string 31 | Localized bool 32 | Required bool 33 | Disabled bool 34 | Omitted bool 35 | } 36 | -------------------------------------------------------------------------------- /mapper/items.go: -------------------------------------------------------------------------------- 1 | package mapper 2 | 3 | type ItemResult struct { 4 | Total int 5 | Skip int 6 | Limit int 7 | Items []Item 8 | Includes map[string][]Item 9 | } 10 | 11 | type Item struct { 12 | Sys Sys 13 | Fields map[string]interface{} 14 | } 15 | 16 | func (item *Item) ContentType() string { 17 | return item.Sys.ContentType.Sys.ID 18 | } 19 | 20 | type Sys struct { 21 | Type string 22 | LinkType string 23 | ID string 24 | Space map[string]interface{} 25 | CreatedAt string 26 | Locale string 27 | Revision int 28 | UpdatedAt string 29 | ContentType ContentType 30 | } 31 | 32 | type ContentType struct { 33 | Sys TypeDetails 34 | } 35 | 36 | type TypeDetails struct { 37 | Type string 38 | LinkType string 39 | ID string 40 | } 41 | -------------------------------------------------------------------------------- /translate/config.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | // TODO: TranslationConfig 4 | // e.g. /content as basedir 5 | // e.g. mainContent 6 | // e.g. slug 7 | 8 | // e.g. homepage -> _index.md 9 | // etc 10 | 11 | import ( 12 | "os" 13 | 14 | "github.com/naoina/toml" 15 | ) 16 | 17 | type TransConfig struct { 18 | Encoding string 19 | Section map[string]interface{} 20 | LeafBundle map[string]interface{} 21 | } 22 | 23 | func LoadConfig(config string) TransConfig { 24 | fileName := config 25 | if _, err := os.Stat(fileName); os.IsNotExist(err) { 26 | return TransConfig{} 27 | } 28 | 29 | f, err := os.Open(fileName) 30 | if err != nil { 31 | panic(err) 32 | } 33 | defer f.Close() 34 | var conf TransConfig 35 | if err := toml.NewDecoder(f).Decode(&conf); err != nil { 36 | panic(err) 37 | } 38 | 39 | return conf 40 | } 41 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/naoina/go-stringutil" 6 | packages = ["."] 7 | revision = "6b638e95a32d0c1131db0e7fe83775cbea4a0d0b" 8 | version = "v0.1.0" 9 | 10 | [[projects]] 11 | name = "github.com/naoina/toml" 12 | packages = [ 13 | ".", 14 | "ast" 15 | ] 16 | revision = "e6f5723bf2a66af014955e0888881314cf294129" 17 | version = "v0.1.1" 18 | 19 | [[projects]] 20 | name = "gopkg.in/yaml.v2" 21 | packages = ["."] 22 | revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" 23 | version = "v2.2.1" 24 | 25 | [solve-meta] 26 | analyzer-name = "dep" 27 | analyzer-version = 1 28 | inputs-digest = "38bd3280adab0af76ef5ef5d9f8ad60355f67163a2150d54358c327f6db8bcf6" 29 | solver-name = "gps-cdcl" 30 | solver-version = 1 31 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/naoina/toml" 30 | version = "0.1.1" 31 | 32 | [[constraint]] 33 | name = "gopkg.in/yaml.v2" 34 | version = "2.2.1" 35 | 36 | [prune] 37 | go-tests = true 38 | unused-packages = true 39 | -------------------------------------------------------------------------------- /translate/directory.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import "github.com/friends-of-hugo/contentful-export/mapper" 4 | 5 | // EstablishDirLevelConf provides the ability to augment content directories with with LeafBundle (index.md) 6 | // or Section level (_index.md) frontmatter during the export process. 7 | func EstablishDirLevelConf(t mapper.Type, tc TransConfig) (string, string) { 8 | var fileName string 9 | var content string 10 | if tc.Section[t.Sys.ID] != nil { 11 | fileName = SectionFilename(t) 12 | if tc.Encoding == "yaml" { 13 | content = WriteYamlFrontmatter(tc.Section[t.Sys.ID]) 14 | } else { 15 | content = WriteTomlFrontmatter(tc.Section[t.Sys.ID]) 16 | } 17 | } 18 | if tc.LeafBundle[t.Sys.ID] != nil { 19 | fileName = LeafBundleFilename(t) 20 | if tc.Encoding == "yaml" { 21 | content = WriteYamlFrontmatter(tc.LeafBundle[t.Sys.ID]) 22 | } else { 23 | content = WriteTomlFrontmatter(tc.LeafBundle[t.Sys.ID]) 24 | } 25 | } 26 | 27 | return fileName, content 28 | } 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Adriaan de Jonge 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /goreleaser.yml: -------------------------------------------------------------------------------- 1 | build: 2 | main: main.go 3 | binary: contentful-hugo 4 | ldflags: -s -w -X hugolib.BuildDate={{.Date}} 5 | goos: 6 | - darwin 7 | - linux 8 | - windows 9 | - freebsd 10 | - netbsd 11 | - openbsd 12 | - dragonfly 13 | goarch: 14 | - amd64 15 | - 386 16 | - arm 17 | - arm64 18 | ignore: 19 | - goos: openbsd 20 | goarch: arm 21 | goarm: 6 22 | fpm: 23 | formats: 24 | - deb 25 | vendor: "Young Industries" 26 | homepage: "https://adriaandejonge.github.io/" 27 | maintainer: "Adriaan de Jonge " 28 | description: "Export Contentful to Hugo" 29 | license: "Apache 2.0" 30 | archive: 31 | format: tar.gz 32 | format_overrides: 33 | - goos: windows 34 | format: zip 35 | name_template: "{{.Binary}}_{{.Version}}_{{.Os}}-{{.Arch}}" 36 | replacements: 37 | amd64: 64bit 38 | 386: 32bit 39 | arm: ARM 40 | arm64: ARM64 41 | darwin: macOS 42 | linux: Linux 43 | windows: Windows 44 | openbsd: OpenBSD 45 | netbsd: NetBSD 46 | freebsd: FreeBSD 47 | dragonfly: DragonFlyBSD 48 | files: 49 | - README.md 50 | release: 51 | draft: true 52 | -------------------------------------------------------------------------------- /read/contentful.go: -------------------------------------------------------------------------------- 1 | package read 2 | 3 | import ( 4 | "io" 5 | "strconv" 6 | ) 7 | 8 | const previewURL string = "https://preview.contentful.com" 9 | const URL string = "https://cdn.contentful.com" 10 | 11 | type Contentful struct { 12 | Getter Getter 13 | ReadConfig ReadConfig 14 | } 15 | 16 | // Types will use Contentful's content_types endpoint to retrieve all content types from contentful 17 | func (c *Contentful) Types() (rc io.ReadCloser, err error) { 18 | 19 | return c.get("/spaces/" + 20 | c.ReadConfig.SpaceID + "/content_types?access_token=" + 21 | c.ReadConfig.AccessToken + "&limit=200&order=sys.createdAt&locale=" + 22 | c.ReadConfig.Locale) 23 | } 24 | 25 | // Items will use Contentful's entires endpoint to retrieve all 'items' from contetnful 26 | func (c *Contentful) Items(skip int) (rc io.ReadCloser, err error) { 27 | 28 | return c.get("/spaces/" + 29 | c.ReadConfig.SpaceID + "/entries?access_token=" + 30 | c.ReadConfig.AccessToken + "&limit=200&order=sys.createdAt&locale=" + 31 | c.ReadConfig.Locale + "&skip=" + strconv.Itoa(skip)) 32 | } 33 | 34 | func (c *Contentful) get(endpoint string) (rc io.ReadCloser, err error) { 35 | urlBase := URL 36 | if c.ReadConfig.UsePreview { 37 | urlBase = previewURL 38 | } 39 | 40 | return c.Getter.Get(urlBase + endpoint) 41 | } 42 | -------------------------------------------------------------------------------- /translate/filename.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/friends-of-hugo/contentful-export/mapper" 7 | ) 8 | 9 | const baseArchetypeDir string = "./archetypes/" 10 | const baseContentDir string = "./content/" 11 | const idxFile string = "index.md" 12 | const sectionIdxFile string = "_" + idxFile 13 | 14 | func Dir(baseDir string, contentType string) string { 15 | dir := baseDir 16 | if contentType != "homepage" { 17 | dir += strings.ToLower(contentType) + "/" 18 | } 19 | 20 | return dir 21 | } 22 | 23 | func Filename(item mapper.Item) string { 24 | baseDir := baseContentDir 25 | dir := Dir(baseDir, item.ContentType()) 26 | if dir == baseDir { 27 | return dir + sectionIdxFile 28 | } 29 | 30 | return dir + item.Sys.ID + ".md" 31 | } 32 | 33 | func SectionFilename(t mapper.Type) string { 34 | dir := Dir(baseContentDir, t.Sys.ID) 35 | 36 | return dir + sectionIdxFile 37 | } 38 | 39 | func LeafBundleFilename(t mapper.Type) string { 40 | dir := Dir(baseContentDir, t.Sys.ID) 41 | 42 | return dir + idxFile 43 | } 44 | 45 | // ArcheTypeFilename takes a content-type's name and returns the file path to the corresponding archetype file. 46 | func GetArchetypeFilename(contentTypeName string) string { 47 | dir := baseArchetypeDir 48 | 49 | return dir + contentTypeName + ".md" 50 | } 51 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/friends-of-hugo/contentful-export/extract" 9 | "github.com/friends-of-hugo/contentful-export/read" 10 | "github.com/friends-of-hugo/contentful-export/translate" 11 | "github.com/friends-of-hugo/contentful-export/write" 12 | ) 13 | 14 | func main() { 15 | space := flag.String("space-id", os.Getenv("CONTENTFUL_API_SPACE"), "The contentful space id to export data from") 16 | key := flag.String("api-key", os.Getenv("CONTENTFUL_API_KEY"), "The contentful delivery API access token") 17 | config := flag.String("config-file", "extract-config.toml", "Path to the TOML config file to load for export config") 18 | preview := flag.Bool("p", false, "Use contentful's preview API so that draft content is downloaded") 19 | flag.Parse() 20 | 21 | fmt.Println("Begin contentful export : ", *space) 22 | extractor := extract.Extractor{ 23 | ReadConfig: read.ReadConfig{ 24 | UsePreview: *preview, 25 | SpaceID: *space, 26 | AccessToken: *key, 27 | Locale: "en-US", 28 | }, 29 | Getter: read.HttpGetter{}, 30 | TransConfig: translate.LoadConfig(*config), 31 | WStore: write.FileStore{}, 32 | RStore: read.FileStore{}, 33 | } 34 | 35 | err := extractor.ProcessAll() 36 | if err != nil { 37 | fmt.Println(err) 38 | } else { 39 | fmt.Println("finished") 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /translate/yaml.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | "gopkg.in/yaml.v2" 8 | ) 9 | 10 | const _YAMLdelimiter string = "---" 11 | 12 | // ToYaml .() -> string 13 | // Takes a Content struct and outputs it as YAML frontmatter followed by main-content. 14 | func (s Content) ToYaml() string { 15 | result := WriteYamlFrontmatter(s.Params) 16 | result += s.MainContent 17 | 18 | return result 19 | } 20 | 21 | // FromYaml reads in a *.yaml file and returns all mappings. 22 | func FromYaml(s string) (c map[string]interface{}, err error) { 23 | c = map[string]interface{}{} 24 | potentialFrontmatter := strings.SplitAfter(s, _YAMLdelimiter) 25 | 26 | if len(potentialFrontmatter) > 1 { 27 | frontmatter := []byte(strings.TrimRight(potentialFrontmatter[1], _YAMLdelimiter)) 28 | err = yaml.Unmarshal(frontmatter, &c) 29 | } else { 30 | err = errors.New("No parsable YAML found in: " + s) 31 | } 32 | 33 | return 34 | } 35 | 36 | // WriteYamlFrontmatter (fm Map[]) -> string 37 | // Converts a Map[] into a YAML string, pre and postfixing it with `---` to designate frontmatter. 38 | func WriteYamlFrontmatter(fm interface{}) string { 39 | result := _YAMLdelimiter + "\n" 40 | output, err := yaml.Marshal(fm) 41 | if err != nil { 42 | return "ERR" 43 | } 44 | 45 | result += string(output) 46 | result += _YAMLdelimiter + "\n" 47 | return result 48 | } 49 | -------------------------------------------------------------------------------- /translate/toml.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | "github.com/naoina/toml" 8 | ) 9 | 10 | const _TOMLdelimiter string = "+++" 11 | 12 | // ToToml .() -> string 13 | // Takes a Content struct and outputs it as TOML frontmatter followed by main-content. 14 | func (s Content) ToToml() string { 15 | result := WriteTomlFrontmatter(s.Params) 16 | result += s.MainContent 17 | 18 | return result 19 | } 20 | 21 | // FromToml reads in a *.toml file and returns all mappings. 22 | func FromToml(s string) (c map[string]interface{}, err error) { 23 | c = map[string]interface{}{} 24 | potentialFrontmatter := strings.SplitAfter(s, _TOMLdelimiter) 25 | 26 | if len(potentialFrontmatter) > 1 { 27 | frontmatter := []byte(strings.TrimRight(potentialFrontmatter[1], _TOMLdelimiter)) 28 | err = toml.Unmarshal(frontmatter, &c) 29 | } else { 30 | err = errors.New("No parsable TOML found in: " + s) 31 | } 32 | 33 | return 34 | } 35 | 36 | // WriteTomlFrontmatter (fm Map[]) -> string 37 | // Converts a Map[] into a TOML string, pre and postfixing it with `+++` to designate frontmatter. 38 | func WriteTomlFrontmatter(fm interface{}) string { 39 | result := _TOMLdelimiter + "\n" 40 | output, err := toml.Marshal(fm) 41 | if err != nil { 42 | return "ERR" 43 | } 44 | 45 | result += string(output) 46 | result += _TOMLdelimiter + "\n" 47 | 48 | return result 49 | } 50 | -------------------------------------------------------------------------------- /extract/extractor.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "github.com/friends-of-hugo/contentful-export/mapper" 5 | "github.com/friends-of-hugo/contentful-export/read" 6 | "github.com/friends-of-hugo/contentful-export/translate" 7 | "github.com/friends-of-hugo/contentful-export/write" 8 | 9 | "log" 10 | ) 11 | 12 | // Extractor enables the automated tests to replace key functionalities 13 | // with fakes, mocks and stubs by parameterizing the Reader Configuration, 14 | // the HTTP Getter and the File Store. 15 | type Extractor struct { 16 | ReadConfig read.ReadConfig 17 | Getter read.Getter 18 | RStore read.Store 19 | TransConfig translate.TransConfig 20 | WStore write.Store 21 | } 22 | 23 | // ProcessAll goes through all stages: Read, Map, Translate and Write. 24 | // Underwater, it uses private function processItems to allow reading 25 | // through multiple pages of items being returned from Contentful. 26 | func (e *Extractor) ProcessAll() error { 27 | 28 | cf := read.Contentful{ 29 | Getter: e.Getter, 30 | ReadConfig: e.ReadConfig, 31 | } 32 | typesReader, err := cf.Types() 33 | if err != nil { 34 | log.Fatal(err) 35 | return err 36 | } 37 | 38 | typeResult, err := mapper.MapTypes(typesReader) 39 | if err != nil { 40 | log.Fatal(err) 41 | return err 42 | } 43 | 44 | writer := write.Writer{Store: e.WStore} 45 | for _, t := range typeResult.Items { 46 | fileName, content := translate.EstablishDirLevelConf(t, e.TransConfig) 47 | if fileName != "" && content != "" { 48 | writer.SaveToFile(fileName, content) 49 | } 50 | } 51 | 52 | skip := 0 53 | 54 | var ids map[string]string 55 | ids = make(map[string]string) 56 | ids = e.processContentTypes(cf, typeResult, skip, ids) 57 | 58 | e.processItems(cf, typeResult, skip, ids) 59 | return nil 60 | } 61 | 62 | // processContentTypes is a recursive function which goes through all pages 63 | // to make a maping of ids to contentTypes 64 | func (e *Extractor) processContentTypes(cf read.Contentful, typeResult mapper.TypeResult, skip int, ids map[string]string) (map[string]string ) { 65 | itemsReader, err := cf.Items(skip) 66 | if err != nil { 67 | log.Fatal(err) 68 | } 69 | itemResult, err := mapper.MapItems(itemsReader) 70 | if err != nil { 71 | log.Fatal(err) 72 | } 73 | 74 | for _, item := range itemResult.Items { 75 | contentType := item.ContentType() 76 | ids[item.Sys.ID] = contentType 77 | } 78 | 79 | nextPage := itemResult.Skip + itemResult.Limit 80 | if nextPage < itemResult.Total { 81 | var nextPageIds = e.processContentTypes(cf, typeResult, nextPage, ids) 82 | for k, v := range nextPageIds { 83 | ids[k] = v 84 | } 85 | } 86 | return ids 87 | } 88 | 89 | // processItems is a recursive function which goes through all pages 90 | // returned by Contentful and creates a markdownfile for each. 91 | func (e *Extractor) processItems(cf read.Contentful, typeResult mapper.TypeResult, skip int, ids map[string]string) { 92 | itemsReader, err := cf.Items(skip) 93 | if err != nil { 94 | log.Fatal(err) 95 | } 96 | 97 | itemResult, err := mapper.MapItems(itemsReader) 98 | if err != nil { 99 | log.Fatal(err) 100 | } 101 | 102 | archetypeDataMap := make(map[string]map[string]interface{}) 103 | reader := read.Reader{Store: e.RStore} 104 | writer := write.Writer{Store: e.WStore} 105 | tc := translate.TranslationContext{Result: itemResult, TransConfig: e.TransConfig} 106 | for _, item := range itemResult.Items { 107 | contentType := item.ContentType() 108 | itemType, err := typeResult.GetType(contentType) 109 | if err != nil { 110 | log.Fatalln(err) 111 | } 112 | 113 | if archetypeDataMap[contentType] == nil { 114 | result, err := reader.ViewFromFile(translate.GetArchetypeFilename(contentType)) 115 | if err == nil { 116 | archeMap, err := tc.TranslateFromMarkdown(result) 117 | if err != nil { 118 | log.Fatalln(err) 119 | } 120 | 121 | archetypeDataMap[contentType] = archeMap 122 | } else { 123 | 124 | archetypeDataMap[contentType] = make(map[string]interface{}) 125 | } 126 | } 127 | 128 | contentMap := tc.MapContentValuesToTypeNames(item.Fields, itemType.Fields, ids) 129 | overriddenContentmap := tc.MergeMaps(archetypeDataMap[contentType], contentMap) 130 | contentMarkdown := tc.TranslateToMarkdown(tc.ConvertToContent(overriddenContentmap)) 131 | fileName := translate.Filename(item) 132 | writer.SaveToFile(fileName, contentMarkdown) 133 | } 134 | 135 | nextPage := itemResult.Skip + itemResult.Limit 136 | if nextPage < itemResult.Total { 137 | e.processItems(cf, typeResult, nextPage, ids) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Contentful Hugo Extractor 2 | 3 |   4 | 5 | This tool extracts all content from your Contentful space and makes it easily consumable by Hugo. You can run it locally or as part of a CI server like Travis. 6 | 7 | ## Install 8 | 9 | ### Go Install Method 10 | 11 | Assuming Go (1.10 +) is installed as well as [dep](https://golang.github.io/dep/) 12 | 13 | ``` sh 14 | go get -u "github.com/friends-of-hugo/contentful-export" 15 | cd "$GOPATH/src/github.com/friends-of-hugo/contentful-export" 16 | dep ensure 17 | go install 18 | ``` 19 | 20 | ## Usage 21 | 22 | ``` sh 23 | contentful-export [Flags] 24 | 25 | Flags: 26 | -space-id=value "Id of the contentful space from which to extract content. If not present will default to an environment variable named `$CONTENTFUL_API_SPACE`" 27 | -api-key=value "API Key used to authenticate with contentful for content delivery. If not present will default to an environment variable named `$CONTENTFUL_API_KEY`. The preview API key should be provided if -p is used." 28 | -config-file=value "Path to the config TOML file to load. Defauls to `./extract-config.tml`" 29 | -p "If present, the contentful preview API will be used so that draft content will be included as part of the export." 30 | ``` 31 | 32 | The tool requires two parameters to work, a contentful space id and API key. These can be provided as command line flags or as environment variables 33 | 34 | _As environment vars..._ 35 | 36 | ``` sh 37 | export CONTENTFUL_API_KEY=YOUR-CONTENT-DELIVERY-API-ACCESS-TOKEN-HERE 38 | export CONTENTFUL_API_SPACE=YOUR-SPACE-ID-HERE 39 | 40 | contentful-export 41 | ``` 42 | 43 | _As flags..._ 44 | 45 | ``` sh 46 | contentful-export -space-id=[YOUR-ID-HERE] -api-key=[YOUR-ACCESS-KEY-HERE] -config-file="./export-conf.toml" 47 | 48 | ``` 49 | 50 | ## Expected output 51 | 52 | Contentful Hugo Extractor stores all content under the /content directory. For each content type, it makes a subdirectory. For each item, it creates a markdown file with the all properties in TOML format. 53 | 54 | Special cases: 55 | 56 | - Items of type Homepage are stored as /content/_index 57 | - Note that there can only be one such item 58 | - Fields named mainContent are used for the main content of the markdown file 59 | - File names are based on the ID of the item to make it easily referencable from related items (for the machine, not humans) 60 | 61 | ## Configuration 62 | 63 | Use the `--config-file` command line flag to provide the location of a TOML configuration to laod or ensure that there is a `extract-config.toml` file in the work directory of contentful-hugo 64 | 65 | ### Configure YAML output 66 | 67 | While the default output is in TOML format, it is also possible to output content in YAML format. Use the following key in your config file: 68 | 69 | ``` yaml 70 | encoding = "yaml" 71 | ``` 72 | 73 | ### Configure Hugo Page Bundles 74 | 75 | `contentful-export` will export each content type in contentful into its own content directory `./content/` and, since hugo treats each rootlevel content directory as a [Section][1], you will end up having a hugo section for each contentful content type. Hugo allows you to provide [Section][1] level configuration for its [Page Bundles](https://gohugo.io/content-management/page-bundles) by dropping a file named `_index.md` in the section's content directory. It is likely that you'll want to provide such configuration for some sections. 76 | 77 | For example, let's say you need to make a section [headless](https://gohugo.io/content-management/page-bundles/#headless-bundle). Pretend that you have a contentful content type with the id `question` and you have some questions in your contentful content model which you intend to reference in a seperate `FAQ` page. After a `contentful-hugo` export, you might the following directory structure: 78 | 79 | ``` none 80 | ./ 81 | | content 82 | | | _index.md 83 | | | question 84 | | | | 12h3jk213n.md //question 1 85 | | | | sdfer343sn.md //question 2 86 | | | page 87 | | | | sdf234dd32.md //FAQ page - refs questions in its frontmatter 88 | | layouts 89 | | | _default 90 | | | | single.html 91 | | | page 92 | | | | single.html //question refs are loaded via .Site.GetPage 93 | ``` 94 | 95 | Without any further confuguration, hugo would generate a HTML file for the page using the `./layouts/page/single.html` layout template but it would aslo generate HTML files for the questions using the `./layouts/_default/single.html` layout template. To prevent this from happening you would create the following file under the path `./content/question/index.md`: 96 | 97 | ``` toml 98 | +++ 99 | headless = true 100 | +++ 101 | ``` 102 | 103 | If you need this kind of configuration, the `contentful-hugo` export process can generate this `index.md` file for you. Simply provide the TOML to use in your config file: 104 | 105 | ``` toml 106 | encoding = "toml" 107 | [section] 108 | [section.question] 109 | headless = true 110 | ``` 111 | 112 | You can nest as many tables as you need under the `[sections]` and if the nested table name matches a contentful content type id than the configuration provided will be propagated to the section's `index.md` frontmatter. 113 | 114 | [1]: https://gohugo.io/content-management/sections/ 115 | -------------------------------------------------------------------------------- /translate/translate.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/friends-of-hugo/contentful-export/mapper" 10 | ) 11 | 12 | type TranslationContext struct { 13 | Result mapper.ItemResult 14 | TransConfig TransConfig 15 | } 16 | 17 | type Content struct { 18 | Params map[string]interface{} 19 | MainContent string 20 | Slug string 21 | } 22 | 23 | func isZero(v reflect.Value) bool { 24 | switch v.Kind() { 25 | case reflect.Func, reflect.Map, reflect.Slice: 26 | return v.IsNil() 27 | case reflect.Array: 28 | z := true 29 | for i := 0; i < v.Len(); i++ { 30 | z = z && isZero(v.Index(i)) 31 | } 32 | return z 33 | case reflect.Struct: 34 | z := true 35 | for i := 0; i < v.NumField(); i++ { 36 | if v.Field(i).CanSet() { 37 | z = z && isZero(v.Field(i)) 38 | } 39 | } 40 | return z 41 | case reflect.Ptr: 42 | return isZero(reflect.Indirect(v)) 43 | } 44 | // Compare other types directly: 45 | z := reflect.Zero(v.Type()) 46 | result := v.Interface() == z.Interface() 47 | 48 | return result 49 | } 50 | 51 | // MergeMaps takes a defaults and an overrides map and assigns any missing 52 | // values from the defaults to the overrides map. 53 | func (tc *TranslationContext) MergeMaps(itemDefault map[string]interface{}, itemOverride map[string]interface{}) (combinedItem map[string]interface{}) { 54 | for k, v := range itemDefault { 55 | if isZero(reflect.ValueOf(itemOverride[k])) { 56 | itemOverride[k] = v 57 | } 58 | } 59 | 60 | return itemOverride 61 | } 62 | 63 | // MapContentValuesToTypeNames takes the values map and the typefield map from contentful and merges the two. 64 | func (tc *TranslationContext) MapContentValuesToTypeNames(Map map[string]interface{}, fields []mapper.TypeField, ids map[string]string) map[string]interface{} { 65 | fieldMap := map[string]interface{}{} 66 | for _, field := range fields { 67 | value := tc.translateField(Map[field.ID], field, ids) 68 | if value != nil { 69 | fieldMap[field.ID] = value 70 | } 71 | } 72 | 73 | return fieldMap 74 | } 75 | 76 | func removeItem(Map map[string]interface{}, toDelete string) interface{} { 77 | value := Map[toDelete] 78 | if value == nil { 79 | return "" 80 | } 81 | delete(Map, toDelete) 82 | 83 | return value 84 | } 85 | 86 | // ConvertToContent takes a map of values and converts it to a Content struct 87 | func (tc *TranslationContext) ConvertToContent(fieldMap map[string]interface{}) Content { 88 | mainContent := removeItem(fieldMap, "mainContent").(string) 89 | slug, _ := fieldMap["slug"].(string) 90 | 91 | return Content{ 92 | fieldMap, 93 | mainContent, 94 | slug, 95 | } 96 | } 97 | 98 | // TranslateFromMarkdown takes a markdown file's contents and converts it to a map. 99 | func (tc *TranslationContext) TranslateFromMarkdown(content string) (rawContent map[string]interface{}, err error) { 100 | switch tc.TransConfig.Encoding { 101 | case "yaml": 102 | return FromYaml(content) 103 | case "toml": 104 | return FromToml(content) 105 | default: 106 | return FromToml(content) 107 | } 108 | } 109 | 110 | // TranslateToMarkdown accepts a Content struct and converts it to markdown file contents. 111 | func (tc *TranslationContext) TranslateToMarkdown(rawContent Content) (content string) { 112 | switch tc.TransConfig.Encoding { 113 | case "yaml": 114 | return rawContent.ToYaml() 115 | case "toml": 116 | return rawContent.ToToml() 117 | default: 118 | return rawContent.ToToml() 119 | } 120 | } 121 | 122 | func (tc *TranslationContext) translateArrayField(value interface{}, ids map[string]string) interface{} { 123 | if value == nil { 124 | return []interface{}{} 125 | } 126 | items := value.([]interface{}) 127 | 128 | var array []string 129 | array = make([]string, len(items)) 130 | 131 | for i, el := range items { 132 | s, isString := el.(string) 133 | if isString { 134 | array[i] = s 135 | } else { 136 | if s, ok := tc.translateLinkField(el, ids).(string); ok { 137 | array[i] = s 138 | } 139 | } 140 | } 141 | 142 | return array 143 | } 144 | 145 | func (tc *TranslationContext) translateLinkField(value interface{}, ids map[string]string) interface{} { 146 | if value == nil { 147 | return value 148 | } 149 | item := value.(map[string]interface{}) 150 | sys := item["sys"].(map[string]interface{}) 151 | contentType := strings.ToLower(ids[sys["id"].(string)]) 152 | linkType := sys["linkType"] 153 | if linkType == "Entry" { 154 | return contentType + "/" + sys["id"].(string) + ".md" 155 | } else { 156 | assets := tc.Result.Includes["Asset"] 157 | for _, asset := range assets { 158 | if sys["id"].(string) == asset.Sys.ID { 159 | return asset.Fields 160 | } 161 | } 162 | // Look up asset - but from where??? 163 | } 164 | 165 | return "ERR" 166 | } 167 | 168 | func (tc *TranslationContext) translateDateField(value interface{}) interface{} { 169 | re, err := regexp.Compile(`([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2})(\+[0-9]{2}:[0-9]{2})?`) // want to know what is in front of 'at' 170 | if err != nil { 171 | fmt.Println(err) 172 | } 173 | 174 | res := re.FindAllStringSubmatch(value.(string), -1) 175 | if len(res) > 0 { 176 | value = fmt.Sprintf("%v:00%v", res[0][1], res[0][2]) 177 | } 178 | 179 | return value 180 | } 181 | 182 | func (tc *TranslationContext) translateField(value interface{}, field mapper.TypeField, ids map[string]string) interface{} { 183 | if field.Type == "Array" { 184 | return tc.translateArrayField(value, ids) 185 | 186 | } else if field.Type == "Link" { 187 | return tc.translateLinkField(value, ids) 188 | 189 | } else if field.Type == "Date" { 190 | return tc.translateDateField(value) 191 | } 192 | 193 | return value 194 | } 195 | -------------------------------------------------------------------------------- /translate/translate_test.go: -------------------------------------------------------------------------------- 1 | package translate 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/friends-of-hugo/contentful-export/mapper" 8 | ) 9 | 10 | func TestTranslateFromMarkdown(t *testing.T) { 11 | yamlMarkdown := ` 12 | --- 13 | overriddentestbool: false 14 | overriddentestint: 13 15 | overriddenteststring: "please, overide me" 16 | overriddentestslicenil: [] 17 | overriddentestslicebool: [false, false, false] 18 | overriddentestsliceint: [13, 13, 13] 19 | overriddentestslicestring: ["please", "overide", "me"] 20 | testbool: true 21 | testint: 42 22 | teststring: "test" 23 | testslicenil: [] 24 | testslicebool: [true, false, true] 25 | testsliceint: [1, 2, 3] 26 | testslicestring: ["one", "two", "thee"] 27 | --- 28 | ` 29 | tomlMarkdown := ` 30 | +++ 31 | overriddentestbool = false 32 | overriddentestint = 13 33 | overriddenteststring = "please, overide me" 34 | overriddentestslicenil = [] 35 | overriddentestslicebool = [false, false, false] 36 | overriddentestsliceint = [13, 13, 13] 37 | overriddentestslicestring = ["please", "overide", "me"] 38 | testbool = true 39 | testint = 42 40 | teststring = "test" 41 | testslicenil = [] 42 | testslicebool = [true, false, true] 43 | testsliceint = [1, 2, 3] 44 | testslicestring = ["one", "two", "thee"] 45 | +++ 46 | ` 47 | extraMarkdown := "\n#header\n_italics_\n__bold__" 48 | archetypeYaml := yamlMarkdown + extraMarkdown 49 | archetypeToml := tomlMarkdown + extraMarkdown 50 | 51 | tests := []struct { 52 | encoding string 53 | givenInput string 54 | }{ 55 | { 56 | encoding: "yaml", 57 | givenInput: archetypeYaml, 58 | }, 59 | { 60 | encoding: "toml", 61 | givenInput: archetypeToml, 62 | }, 63 | } 64 | 65 | for _, test := range tests { 66 | tc := TranslationContext{TransConfig: TransConfig{Encoding: test.encoding}} 67 | result, err := tc.TranslateFromMarkdown(test.givenInput) 68 | if err != nil || result == nil { 69 | t.Errorf("TranslateFromMarkdown() failed...\n\nInput:\n%s\n\nActual Output:\n%v\n\nInner Error:\n%v", test.givenInput, result, err) 70 | } 71 | } 72 | } 73 | 74 | func TestConvertContent(t *testing.T) { 75 | tests := []struct { 76 | Map map[string]interface{} 77 | fields []mapper.TypeField 78 | expected Content 79 | }{ 80 | { 81 | map[string]interface{}{ 82 | "key": "value", 83 | }, 84 | []mapper.TypeField{ 85 | mapper.TypeField{ID: "key", Name: "", Type: "String", Localized: false, Required: false, Disabled: false, Omitted: false}, 86 | }, 87 | Content{ 88 | map[string]interface{}{ 89 | "key": "value", 90 | }, 91 | "", 92 | "", 93 | }, 94 | }, 95 | { 96 | map[string]interface{}{ 97 | "key": "value", 98 | "mainContent": "This is test main content\nand one more line", 99 | "slug": "my-test-slug", 100 | }, 101 | []mapper.TypeField{ 102 | mapper.TypeField{ID: "key", Name: "", Type: "String", Localized: false, Required: false, Disabled: false, Omitted: false}, 103 | mapper.TypeField{ID: "mainContent", Name: "", Type: "String", Localized: false, Required: false, Disabled: false, Omitted: false}, 104 | mapper.TypeField{ID: "slug", Name: "", Type: "String", Localized: false, Required: false, Disabled: false, Omitted: false}, 105 | }, 106 | Content{ 107 | map[string]interface{}{ 108 | "key": "value", 109 | "slug": "my-test-slug", 110 | }, 111 | "This is test main content\nand one more line", 112 | "my-test-slug", 113 | }, 114 | }, 115 | } 116 | 117 | tc := TranslationContext{} 118 | 119 | for _, test := range tests { 120 | result := tc.ConvertToContent(tc.MapContentValuesToTypeNames(test.Map, test.fields)) 121 | if !reflect.DeepEqual(result, test.expected) { 122 | t.Errorf("_.ConvertToContent( _.MapContentValuesToTypeNames(%v, %v) ) incorrect, expected %v, got %v", test.Map, test.fields, test.expected, result) 123 | } 124 | 125 | } 126 | } 127 | 128 | func TestRemoveItem(t *testing.T) { 129 | tests := []struct { 130 | initial map[string]interface{} 131 | toDelete string 132 | expectedValue string 133 | expectedMap map[string]interface{} 134 | }{ 135 | { 136 | map[string]interface{}{ 137 | "one": "value-1", 138 | "two": "value-2", 139 | }, 140 | "one", 141 | "value-1", 142 | map[string]interface{}{ 143 | "two": "value-2", 144 | }, 145 | }, 146 | { 147 | map[string]interface{}{ 148 | "two": "value-2", 149 | "three": "value-3", 150 | }, 151 | "one", 152 | "", 153 | map[string]interface{}{ 154 | "two": "value-2", 155 | "three": "value-3", 156 | }, 157 | }, 158 | } 159 | 160 | for _, test := range tests { 161 | resultValue := removeItem(test.initial, test.toDelete) 162 | 163 | if !reflect.DeepEqual(resultValue, test.expectedValue) { 164 | t.Errorf("removeItem(%v, %v) return value incorrect, expected %v, got %v", test.initial, test.toDelete, test.expectedValue, resultValue) 165 | } 166 | if !reflect.DeepEqual(test.initial, test.expectedMap) { 167 | t.Errorf("removeItem(%v, %v) resulting map incorrect, expected %v, got %v", test.initial, test.toDelete, test.expectedMap, test.initial) 168 | } 169 | 170 | } 171 | } 172 | 173 | func TestTranslateField(t *testing.T) { 174 | tests := []struct { 175 | value interface{} 176 | field mapper.TypeField 177 | expected interface{} 178 | }{ 179 | { 180 | "Unchanged", 181 | mapper.TypeField{ID: "", Name: "", Type: "default", Localized: false, Required: false, Disabled: false, Omitted: false}, 182 | "Unchanged", 183 | }, 184 | { 185 | []interface{}{ 186 | map[string]interface{}{"sys": map[string]interface{}{"id": "test-id-1", "linkType": "Entry"}}, 187 | map[string]interface{}{"sys": map[string]interface{}{"id": "test-id-2", "linkType": "Entry"}}, 188 | map[string]interface{}{"sys": map[string]interface{}{"id": "test-id-3", "linkType": "Entry"}}, 189 | }, 190 | mapper.TypeField{ID: "", Name: "", Type: "Array", Localized: false, Required: false, Disabled: false, Omitted: false}, 191 | []string{"test-id-1.md", "test-id-2.md", "test-id-3.md"}, 192 | }, 193 | } 194 | 195 | tc := TranslationContext{} 196 | 197 | for _, test := range tests { 198 | result := tc.translateField(test.value, test.field) 199 | 200 | if !reflect.DeepEqual(result, test.expected) { 201 | t.Errorf("translateField(%v, %v) incorrect, expected %v, got %v", test.value, test.field.Type, test.expected, result) 202 | } 203 | } 204 | 205 | } 206 | -------------------------------------------------------------------------------- /extract/extractor_test.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/friends-of-hugo/contentful-export/read" 10 | ) 11 | 12 | type MockStore struct{} 13 | 14 | func (ms MockStore) MkdirAll(path string, perm os.FileMode) error { 15 | return nil 16 | } 17 | 18 | func (ms MockStore) WriteFile(filename string, data []byte, perm os.FileMode) error { 19 | return nil 20 | } 21 | 22 | func (ms MockStore) ReadFromFile(path string) (result []byte, err error) { 23 | return nil, nil 24 | } 25 | 26 | type MockGetter struct { 27 | JSON []string 28 | } 29 | 30 | type MockReaderCloser struct { 31 | Reader io.Reader 32 | } 33 | 34 | func (mrc MockReaderCloser) Read(p []byte) (n int, err error) { 35 | return mrc.Reader.Read(p) 36 | } 37 | 38 | func (mrc MockReaderCloser) Close() error { 39 | return nil 40 | } 41 | 42 | var count int 43 | 44 | func (mg MockGetter) Get(url string) (result io.ReadCloser, err error) { 45 | mrc := MockReaderCloser{strings.NewReader(mg.JSON[count])} 46 | count++ 47 | return mrc, nil 48 | } 49 | 50 | func TestExtractor(t *testing.T) { 51 | testContent := ` 52 | { 53 | "sys": { 54 | "type": "Array" 55 | }, 56 | "total": 4, 57 | "skip": 0, 58 | "limit": 200, 59 | "items": [ 60 | { 61 | "sys": { 62 | "space": { 63 | "sys": { 64 | "type": "Link", 65 | "linkType": "Space", 66 | "id": "fp8h0eoshqd0" 67 | } 68 | }, 69 | "id": "TU4YuzKtKC6QIqceaAOQM", 70 | "type": "Entry", 71 | "createdAt": "2017-08-04T13:01:18.942Z", 72 | "updatedAt": "2017-08-12T05:07:50.535Z", 73 | "revision": 6, 74 | "contentType": { 75 | "sys": { 76 | "type": "Link", 77 | "linkType": "ContentType", 78 | "id": "smallgroup" 79 | } 80 | }, 81 | "locale": "en-US" 82 | }, 83 | "fields": { 84 | "title": "Joe & Joanne", 85 | "slug": "joe-and-joanne", 86 | "description": "Weekly bible study at Joe & Joanne place", 87 | "locationText": "Deira", 88 | "locationCoordinates": { 89 | "lon": 55.318323969841, 90 | "lat": 25.26726106496277 91 | }, 92 | "weekday": "Sunday", 93 | "time": "19:30", 94 | "mainContent": "# Smallgroup\n\nJohn & Caroline host a group on Sunday evenings at 19:30.\n\nLocation: __Al Ghurair__\n\n\n---\n\n" 95 | } 96 | }, 97 | { 98 | "sys": { 99 | "space": { 100 | "sys": { 101 | "type": "Link", 102 | "linkType": "Space", 103 | "id": "fp8h0eoshqd0" 104 | } 105 | }, 106 | "id": "2ZZNV02YX6sI80Coc6MSwa", 107 | "type": "Entry", 108 | "createdAt": "2017-08-04T13:01:56.833Z", 109 | "updatedAt": "2017-08-12T04:55:40.707Z", 110 | "revision": 6, 111 | "contentType": { 112 | "sys": { 113 | "type": "Link", 114 | "linkType": "ContentType", 115 | "id": "smallgroup" 116 | } 117 | }, 118 | "locale": "en-US" 119 | }, 120 | "fields": { 121 | "title": "Jim & Jane", 122 | "slug": "jim-and-jane", 123 | "description": "To do", 124 | "locationText": "Silicon Oasis", 125 | "locationCoordinates": { 126 | "lon": 55.38626380000005, 127 | "lat": 25.1279484 128 | }, 129 | "weekday": "Tuesday", 130 | "time": "19:30", 131 | "mainContent": "To do: Add more text" 132 | } 133 | }, 134 | { 135 | "sys": { 136 | "space": { 137 | "sys": { 138 | "type": "Link", 139 | "linkType": "Space", 140 | "id": "fp8h0eoshqd0" 141 | } 142 | }, 143 | "id": "6CW1y3x6CWCKQoC8UwwAkC", 144 | "type": "Entry", 145 | "createdAt": "2017-08-04T13:05:35.498Z", 146 | "updatedAt": "2017-08-12T04:52:41.342Z", 147 | "revision": 14, 148 | "contentType": { 149 | "sys": { 150 | "type": "Link", 151 | "linkType": "ContentType", 152 | "id": "homepage" 153 | } 154 | }, 155 | "locale": "en-US" 156 | }, 157 | "fields": { 158 | "mainHeader": "Welcome to the \"Test\" Site of Tubia!", 159 | "test": 7, 160 | "listOfSmallgroups": [ 161 | { 162 | "sys": { 163 | "type": "Link", 164 | "linkType": "Entry", 165 | "id": "2ZZNV02YX6sI80Coc6MSwa" 166 | } 167 | }, 168 | { 169 | "sys": { 170 | "type": "Link", 171 | "linkType": "Entry", 172 | "id": "2P1NfVrIJ2yQQqOIiUewOW" 173 | } 174 | }, 175 | { 176 | "sys": { 177 | "type": "Link", 178 | "linkType": "Entry", 179 | "id": "TU4YuzKtKC6QIqceaAOQM" 180 | } 181 | } 182 | ], 183 | "mainContent": "This is the main content of the homepage!\n\nAnd here are a few more lines...\n\n# And this is a header" 184 | } 185 | }, 186 | { 187 | "sys": { 188 | "space": { 189 | "sys": { 190 | "type": "Link", 191 | "linkType": "Space", 192 | "id": "fp8h0eoshqd0" 193 | } 194 | }, 195 | "id": "2P1NfVrIJ2yQQqOIiUewOW", 196 | "type": "Entry", 197 | "createdAt": "2017-08-04T13:02:53.886Z", 198 | "updatedAt": "2017-08-17T14:57:48.379Z", 199 | "revision": 10, 200 | "contentType": { 201 | "sys": { 202 | "type": "Link", 203 | "linkType": "ContentType", 204 | "id": "smallgroup" 205 | } 206 | }, 207 | "locale": "en-US" 208 | }, 209 | "fields": { 210 | "title": "Jaideep & Jyoti", 211 | "slug": "jaideep-and-jyoti", 212 | "description": "Short descr", 213 | "locationText": "Garhoud", 214 | "locationCoordinates": { 215 | "lon": 55.35490795969963, 216 | "lat": 25.239319871287446 217 | }, 218 | "weekday": "Wednesday", 219 | "time": "20:00", 220 | "mainContent": "To do" 221 | } 222 | } 223 | ] 224 | } 225 | ` 226 | testTypes := ` 227 | { 228 | "sys": { 229 | "type": "Array" 230 | }, 231 | "total": 3, 232 | "skip": 0, 233 | "limit": 200, 234 | "items": [ 235 | { 236 | "sys": { 237 | "space": { 238 | "sys": { 239 | "type": "Link", 240 | "linkType": "Space", 241 | "id": "fp8h0eoshqd0" 242 | } 243 | }, 244 | "id": "2PqfXUJwE8qSYKuM0U6w8M", 245 | "type": "ContentType", 246 | "createdAt": "2017-03-16T17:51:06.624Z", 247 | "updatedAt": "2017-03-16T17:51:06.624Z", 248 | "revision": 1 249 | }, 250 | "displayField": "productName", 251 | "name": "Product", 252 | "description": null, 253 | "fields": [ 254 | { 255 | "id": "productName", 256 | "name": "Product name", 257 | "type": "Text", 258 | "localized": false, 259 | "required": true, 260 | "disabled": false, 261 | "omitted": false 262 | }, 263 | { 264 | "id": "slug", 265 | "name": "Slug", 266 | "type": "Symbol", 267 | "localized": false, 268 | "required": false, 269 | "disabled": false, 270 | "omitted": false 271 | }, 272 | { 273 | "id": "productDescription", 274 | "name": "Description", 275 | "type": "Text", 276 | "localized": false, 277 | "required": false, 278 | "disabled": false, 279 | "omitted": false 280 | }, 281 | { 282 | "id": "sizetypecolor", 283 | "name": "Size/Type/Color", 284 | "type": "Symbol", 285 | "localized": false, 286 | "required": false, 287 | "disabled": false, 288 | "omitted": false 289 | }, 290 | { 291 | "id": "image", 292 | "name": "Image", 293 | "type": "Array", 294 | "localized": false, 295 | "required": false, 296 | "disabled": false, 297 | "omitted": false, 298 | "items": { 299 | "type": "Link", 300 | "validations": [], 301 | "linkType": "Asset" 302 | } 303 | }, 304 | { 305 | "id": "tags", 306 | "name": "Tags", 307 | "type": "Array", 308 | "localized": false, 309 | "required": false, 310 | "disabled": false, 311 | "omitted": false, 312 | "items": { 313 | "type": "Symbol", 314 | "validations": [] 315 | } 316 | }, 317 | { 318 | "id": "categories", 319 | "name": "Categories", 320 | "type": "Array", 321 | "localized": false, 322 | "required": false, 323 | "disabled": false, 324 | "omitted": false, 325 | "items": { 326 | "type": "Link", 327 | "validations": [ 328 | { 329 | "linkContentType": [ 330 | "6XwpTaSiiI2Ak2Ww0oi6qa" 331 | ] 332 | } 333 | ], 334 | "linkType": "Entry" 335 | } 336 | }, 337 | { 338 | "id": "price", 339 | "name": "Price", 340 | "type": "Number", 341 | "localized": false, 342 | "required": false, 343 | "disabled": false, 344 | "omitted": false 345 | }, 346 | { 347 | "id": "brand", 348 | "name": "Brand", 349 | "type": "Link", 350 | "localized": false, 351 | "required": false, 352 | "disabled": false, 353 | "omitted": false, 354 | "linkType": "Entry" 355 | }, 356 | { 357 | "id": "quantity", 358 | "name": "Quantity", 359 | "type": "Integer", 360 | "localized": false, 361 | "required": false, 362 | "disabled": false, 363 | "omitted": false 364 | }, 365 | { 366 | "id": "sku", 367 | "name": "SKU", 368 | "type": "Symbol", 369 | "localized": false, 370 | "required": false, 371 | "disabled": false, 372 | "omitted": false 373 | }, 374 | { 375 | "id": "website", 376 | "name": "Available at", 377 | "type": "Symbol", 378 | "localized": false, 379 | "required": false, 380 | "disabled": false, 381 | "omitted": false 382 | } 383 | ] 384 | }, 385 | { 386 | "sys": { 387 | "space": { 388 | "sys": { 389 | "type": "Link", 390 | "linkType": "Space", 391 | "id": "fp8h0eoshqd0" 392 | } 393 | }, 394 | "id": "homepage", 395 | "type": "ContentType", 396 | "createdAt": "2017-08-04T12:59:45.163Z", 397 | "updatedAt": "2017-08-11T16:40:02.258Z", 398 | "revision": 2 399 | }, 400 | "displayField": "mainHeader", 401 | "name": "Homepage", 402 | "description": "One and only one", 403 | "fields": [ 404 | { 405 | "id": "mainHeader", 406 | "name": "Main header", 407 | "type": "Symbol", 408 | "localized": false, 409 | "required": true, 410 | "disabled": false, 411 | "omitted": false 412 | }, 413 | { 414 | "id": "test", 415 | "name": "test", 416 | "type": "Integer", 417 | "localized": false, 418 | "required": false, 419 | "disabled": false, 420 | "omitted": false 421 | }, 422 | { 423 | "id": "listOfSmallgroups", 424 | "name": "List of Smallgroups", 425 | "type": "Array", 426 | "localized": false, 427 | "required": false, 428 | "disabled": false, 429 | "omitted": false, 430 | "items": { 431 | "type": "Link", 432 | "validations": [ 433 | { 434 | "linkContentType": [ 435 | "smallgroup" 436 | ] 437 | } 438 | ], 439 | "linkType": "Entry" 440 | } 441 | }, 442 | { 443 | "id": "mainContent", 444 | "name": "Main Content", 445 | "type": "Text", 446 | "localized": false, 447 | "required": false, 448 | "disabled": false, 449 | "omitted": false 450 | } 451 | ] 452 | }, 453 | { 454 | "sys": { 455 | "space": { 456 | "sys": { 457 | "type": "Link", 458 | "linkType": "Space", 459 | "id": "fp8h0eoshqd0" 460 | } 461 | }, 462 | "id": "smallgroup", 463 | "type": "ContentType", 464 | "createdAt": "2017-08-04T11:42:20.401Z", 465 | "updatedAt": "2017-08-17T14:57:11.135Z", 466 | "revision": 9 467 | }, 468 | "displayField": "title", 469 | "name": "Smallgroup", 470 | "description": "", 471 | "fields": [ 472 | { 473 | "id": "title", 474 | "name": "Title", 475 | "type": "Symbol", 476 | "localized": false, 477 | "required": false, 478 | "disabled": false, 479 | "omitted": false 480 | }, 481 | { 482 | "id": "slug", 483 | "name": "Slug", 484 | "type": "Symbol", 485 | "localized": false, 486 | "required": true, 487 | "disabled": false, 488 | "omitted": false 489 | }, 490 | { 491 | "id": "description", 492 | "name": "Description", 493 | "type": "Text", 494 | "localized": true, 495 | "required": false, 496 | "disabled": false, 497 | "omitted": false 498 | }, 499 | { 500 | "id": "locationText", 501 | "name": "LocationText", 502 | "type": "Symbol", 503 | "localized": false, 504 | "required": true, 505 | "disabled": false, 506 | "omitted": false 507 | }, 508 | { 509 | "id": "locationCoordinates", 510 | "name": "Location Coordinates", 511 | "type": "Location", 512 | "localized": false, 513 | "required": false, 514 | "disabled": false, 515 | "omitted": false 516 | }, 517 | { 518 | "id": "weekday", 519 | "name": "Weekday", 520 | "type": "Symbol", 521 | "localized": false, 522 | "required": false, 523 | "disabled": false, 524 | "omitted": false 525 | }, 526 | { 527 | "id": "time", 528 | "name": "Time", 529 | "type": "Symbol", 530 | "localized": false, 531 | "required": false, 532 | "disabled": false, 533 | "omitted": false 534 | }, 535 | { 536 | "id": "mainContent", 537 | "name": "Main Content", 538 | "type": "Text", 539 | "localized": true, 540 | "required": false, 541 | "disabled": false, 542 | "omitted": false 543 | } 544 | ] 545 | } 546 | ] 547 | } 548 | ` 549 | 550 | extractor := Extractor{ 551 | ReadConfig: read.ReadConfig{ 552 | UsePreview: false, 553 | SpaceID: "my-fake-space-id", 554 | AccessToken: "my-fake-content-key", 555 | Locale: "en-US", 556 | }, 557 | Getter: MockGetter{[]string{testTypes, testContent}}, 558 | RStore: MockStore{}, 559 | WStore: MockStore{}, 560 | } 561 | 562 | extractor.ProcessAll() 563 | 564 | } 565 | --------------------------------------------------------------------------------