├── .gitignore ├── go.mod ├── utils.go ├── main_test.go ├── .github ├── dependabot.yml └── workflows │ └── release.yml ├── Gomfile ├── docs ├── BuildForWindows.md └── BuildForMacLinux.md ├── go.sum ├── import-with-bulkRequest_test.go ├── bulkRequest_test.go ├── export_test.go ├── README.md ├── main.go ├── import-with-bulkRequest.go ├── bulkRequest.go ├── LICENSE └── export.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | build/ 3 | cli-kintone.zip 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module cli-kintone 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c 7 | github.com/jessevdk/go-flags v1.5.0 8 | github.com/kintone-labs/go-kintone v0.4.3 9 | golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307 // indirect 10 | golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect 11 | golang.org/x/text v0.3.1-0.20180410181320-7922cc490dd5 12 | ) 13 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "log" 6 | "io" 7 | ) 8 | 9 | func removeBOMCharacter(reader io.Reader) io.Reader { 10 | bufferReader := bufio.NewReader(reader) 11 | r, _, err := bufferReader.ReadRune() 12 | if err != nil { 13 | log.Fatal(err) 14 | } 15 | 16 | if r != '\uFEFF' { 17 | bufferReader.UnreadRune() // Not a BOM -- put the rune back 18 | } 19 | return bufferReader 20 | } 21 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | 7 | "github.com/kintone-labs/go-kintone" 8 | ) 9 | 10 | func newApp() *kintone.App { 11 | appID, _ := strconv.ParseUint(os.Getenv("KINTONE_APP_ID"), 10, 64) 12 | 13 | return &kintone.App{ 14 | Domain: os.Getenv("KINTONE_DOMAIN"), 15 | User: os.Getenv("KINTONE_USERNAME"), 16 | Password: os.Getenv("KINTONE_PASSWORD"), 17 | AppId: appID, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /Gomfile: -------------------------------------------------------------------------------- 1 | group :production do 2 | gom 'golang.org/x/crypto', :commit => '613d6eafa307c6881a737a3c35c0e312e8d3a8c5' 3 | gom 'golang.org/x/text', :commit => '7922cc490dd5a7dbaa7fd5d6196b49db59ac042f' 4 | gom 'github.com/kintone-labs/go-kintone', :tag => 'v0.3.0' 5 | gom 'github.com/howeyc/gopass', :commit => 'bf9dde6d0d2c004a008c27aaee91170c786f6db8' 6 | gom 'github.com/jessevdk/go-flags', :tag => 'v1.4.0' 7 | end 8 | group :development do 9 | gom -u 'golang.org/x/crypto' 10 | gom -u 'golang.org/x/text' 11 | gom -u 'github.com/kintone-labs/go-kintone' 12 | gom -u 'github.com/howeyc/gopass' 13 | gom -u 'github.com/jessevdk/go-flags' 14 | end 15 | -------------------------------------------------------------------------------- /docs/BuildForWindows.md: -------------------------------------------------------------------------------- 1 | ## How to Build Windows 2 | #### Step 1: Creating folder to develop 3 | ``` 4 | mkdir -p c:\tmp\dev-cli-kintone\src 5 | ``` 6 | Note: "c:\tmp\dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. 7 | 8 | #### Step 2: Creating variable environment GOPATH 9 | 10 | ``` 11 | set GOPATH=c:\tmp\dev-cli-kintone 12 | ``` 13 | 14 | #### Step 3: Getting cli-kintone repository 15 | ``` 16 | cd %GOPATH%\src 17 | git clone https://github.com/kintone/cli-kintone.git 18 | ``` 19 | 20 | #### Step 4: Install dependencies 21 | ``` 22 | cd %GOPATH%\src\cli-kintone 23 | go get github.com/mattn/gom 24 | ..\..\bin\gom.exe -production install 25 | ``` 26 | 27 | #### Step 5: Build 28 | ``` 29 | ..\..\bin\gom.exe build 30 | ``` 31 | 32 | ## Copyright 33 | 34 | Copyright(c) Cybozu, Inc. 35 | -------------------------------------------------------------------------------- /docs/BuildForMacLinux.md: -------------------------------------------------------------------------------- 1 | ## How to Build Mac OS X/Linux 2 | #### Step 1: Creating folder to develop 3 | ``` 4 | mkdir -p /tmp/dev-cli-kintone/src 5 | ``` 6 | Note: "/tmp/dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. 7 | 8 | #### Step 2: Creating variable environment GOPATH 9 | 10 | ``` 11 | export GOPATH=/tmp/dev-cli-kintone 12 | ``` 13 | 14 | #### Step 3: Getting cli-kintone repository 15 | ``` 16 | cd ${GOPATH}/src 17 | git clone https://github.com/kintone/cli-kintone.git 18 | ``` 19 | 20 | #### Step 4: Install dependencies 21 | ``` 22 | cd ${GOPATH}/src/cli-kintone 23 | go get github.com/mattn/gom 24 | sudo ln -s $GOPATH/bin/gom /usr/local/bin/gom # Link package gom to directory "/usr/local/" to use globally 25 | gom -production install 26 | ``` 27 | 28 | #### Step 5: Build 29 | ``` 30 | mv vendor/ src 31 | gom build 32 | ``` 33 | 34 | ## Copyright 35 | 36 | Copyright(c) Cybozu, Inc. 37 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:kQWxfPIHVLbgLzphqk3QUflDy9QdksZR4ygR807bpy0= 2 | github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= 3 | github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= 4 | github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= 5 | github.com/kintone-labs/go-kintone v0.4.3 h1:b5wHLz6gRHsordcqAypcIjlMt1NoI8KDKG8f0NnzukE= 6 | github.com/kintone-labs/go-kintone v0.4.3/go.mod h1:fw3pW563k7QM1RY+uuymUcdJh3IJjafbiTQz43/Q0VI= 7 | golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307 h1:O5C+XK++apFo5B+Vq4ujc/LkLwHxg9fDdgjgoIikBdA= 8 | golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 9 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 10 | golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= 11 | golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 12 | golang.org/x/text v0.3.1-0.20180410181320-7922cc490dd5 h1:23hw054QGj0KDkhDTmeMTzaawNqHp/Q5B65f8TTG3vg= 13 | golang.org/x/text v0.3.1-0.20180410181320-7922cc490dd5/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 14 | -------------------------------------------------------------------------------- /import-with-bulkRequest_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/kintone-labs/go-kintone" 8 | ) 9 | 10 | func TestImport1(t *testing.T) { 11 | data := "Text,Text_Area,Rich_text\n11,22,
aaaaaa
\n111,22,
dddddqqddss
\n211,22,
aaaaaa
" 12 | 13 | app := newApp() 14 | 15 | config.DeleteAll = true 16 | err := importFromCSV(app, bytes.NewBufferString(data)) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | 21 | recs, err := app.GetRecords(nil, "order by $id desc") 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | if len(recs) != 3 { 26 | t.Error("Invalid record count") 27 | } 28 | 29 | fields := recs[0].Fields 30 | if _, ok := fields["Text"].(kintone.SingleLineTextField); !ok { 31 | t.Error("Not a SingleLineTextField") 32 | } 33 | if fields["Text"] != kintone.SingleLineTextField("211") { 34 | t.Error("Text mismatch") 35 | } 36 | if _, ok := fields["Text_Area"].(kintone.MultiLineTextField); !ok { 37 | t.Error("Not a MultiLineTextField") 38 | } 39 | if fields["Text_Area"] != kintone.MultiLineTextField("22") { 40 | t.Error("Text_Area mismatch") 41 | } 42 | if _, ok := fields["Rich_text"].(kintone.RichTextField); !ok { 43 | t.Error("Not a RichTextField") 44 | } 45 | if fields["Rich_text"] != kintone.RichTextField("
aaaaaa
") { 46 | t.Error("Rich_text mismatch") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | Release: 9 | name: Release for ${{ matrix.os }} 10 | runs-on: ubuntu-18.04 11 | strategy: 12 | matrix: 13 | include: 14 | - os: linux-amd64 15 | goos_name: linux 16 | goarch_name: amd64 17 | artifact_name: linux-x64 18 | bin_name: cli-kintone 19 | - os: darwin-amd64 20 | goos_name: darwin 21 | goarch_name: amd64 22 | artifact_name: macos-x64 23 | bin_name: cli-kintone 24 | - os: windows-amd64 25 | goos_name: windows 26 | goarch_name: amd64 27 | artifact_name: windows-x64 28 | bin_name: cli-kintone.exe 29 | steps: 30 | - uses: actions/checkout@v2 31 | - uses: actions/setup-go@v2 32 | with: 33 | go-version: '1.15.15' 34 | - name: Preparation 35 | run: | 36 | go vet -x ./... 37 | - name: Build ${{ matrix.goos_name }}/${{ matrix.goarch_name }} archive 38 | run: | 39 | export GOOS="${{ matrix.goos_name }}" 40 | export GOARCH="${{ matrix.goarch_name }}" 41 | go build -v -tags "forceposix" -o build/${{ matrix.artifact_name }}/${{ matrix.bin_name }} 42 | zip ${{ matrix.artifact_name }}.zip build/${{ matrix.artifact_name }}/${{ matrix.bin_name }} 43 | - name: Upload package to release[binaries] 44 | uses: svenstaro/upload-release-action@v1-release 45 | with: 46 | repo_token: ${{ secrets.CLI_KINTONE_WORKFLOW_TOKEN }} 47 | file: ${{ matrix.artifact_name }}.zip 48 | asset_name: ${{ matrix.artifact_name }}.zip 49 | tag: ${{ github.ref }} 50 | overwrite: true 51 | -------------------------------------------------------------------------------- /bulkRequest_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "os" 7 | "github.com/kintone-labs/go-kintone" 8 | ) 9 | 10 | func newAppTest(id uint64) *kintone.App { 11 | return &kintone.App{ 12 | Domain: os.Getenv("KINTONE_DOMAIN"), 13 | User: os.Getenv("KINTONE_USERNAME"), 14 | Password: os.Getenv("KINTONE_PASSWORD"), 15 | AppId: id, 16 | } 17 | } 18 | 19 | func TestRequest(t *testing.T) { 20 | 21 | bulkReq := &BulkRequests{} 22 | app := newAppTest(16) 23 | bulkReq.Requests = make([]*BulkRequestItem, 0) 24 | 25 | /// INSERT 26 | records := make([]*kintone.Record, 0) 27 | record1 := kintone.NewRecord(map[string]interface{}{ 28 | "Text": kintone.SingleLineTextField("test 11!"), 29 | "_2": kintone.SingleLineTextField("test 21!"), 30 | }) 31 | records = append(records, record1) 32 | record2 := kintone.NewRecord(map[string]interface{}{ 33 | "Text": kintone.SingleLineTextField("test 22!"), 34 | "_2": kintone.SingleLineTextField("test 22!"), 35 | }) 36 | records = append(records, record2) 37 | dataPOST := &DataRequestRecordsPOST{app.AppId, records} 38 | postRecords := &BulkRequestItem{"POST", "/k/v1/records.json", dataPOST} 39 | 40 | bulkReq.Requests = append(bulkReq.Requests, postRecords) 41 | 42 | /// UPDATE 43 | recordsUpdate := make([]interface{}, 0) 44 | recordsUpdate1 := kintone.NewRecordWithId(4902, map[string]interface{}{ 45 | "Text": kintone.SingleLineTextField("test NNN!"), 46 | "_2": kintone.SingleLineTextField("test MMM!"), 47 | }) 48 | fmt.Println(recordsUpdate1) 49 | recordsUpdate = append(recordsUpdate, &DataRequestRecordPUT{ID: recordsUpdate1.Id(), 50 | Record: recordsUpdate1}) 51 | 52 | recordsUpdate2 := kintone.NewRecordWithId(4903, map[string]interface{}{ 53 | "Text": kintone.SingleLineTextField("test 123!"), 54 | "_2": kintone.SingleLineTextField("test 234!"), 55 | }) 56 | recordsUpdate = append(recordsUpdate, &DataRequestRecordPUT{ 57 | ID: recordsUpdate2.Id(), Record: recordsUpdate2}) 58 | 59 | dataPUT := &DataRequestRecordsPUT{app.AppId, recordsUpdate} 60 | putRecords := &BulkRequestItem{"PUT", "/k/v1/records.json", dataPUT} 61 | 62 | bulkReq.Requests = append(bulkReq.Requests, putRecords) 63 | 64 | rs, err := bulkReq.Request(app) 65 | 66 | if err != nil { 67 | t.Error(" failed", err) 68 | } else { 69 | t.Log(rs) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /export_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "fmt" 7 | "io" 8 | "testing" 9 | 10 | "github.com/kintone-labs/go-kintone" 11 | ) 12 | 13 | func makeTestData(app *kintone.App) error { 14 | err := deleteRecords(app, "") 15 | if err != nil { 16 | return err 17 | } 18 | records := make([]*kintone.Record, 0) 19 | 20 | record := make(map[string]interface{}) 21 | record["single_line_text"] = kintone.SingleLineTextField("single line1") 22 | record["multi_line_text"] = kintone.SingleLineTextField("multi line1\nmulti line") 23 | record["number"] = kintone.DecimalField("12345") 24 | table := make([]*kintone.Record, 0) 25 | sub := make(map[string]interface{}) 26 | sub["table_single_line_text"] = kintone.SingleLineTextField("table single line1") 27 | sub["table_multi_line_text"] = kintone.SingleLineTextField("table multi line1\nmulti line") 28 | table = append(table, kintone.NewRecord(sub)) 29 | sub = make(map[string]interface{}) 30 | sub["table_single_line_text"] = kintone.SingleLineTextField("table single line2") 31 | sub["table_multi_line_text"] = kintone.SingleLineTextField("table multi line2\nmulti line") 32 | table = append(table, kintone.NewRecord(sub)) 33 | record["table"] = kintone.SubTableField(table) 34 | 35 | records = append(records, kintone.NewRecord(record)) 36 | 37 | record = make(map[string]interface{}) 38 | record["single_line_text"] = kintone.SingleLineTextField("single line2") 39 | record["multi_line_text"] = kintone.SingleLineTextField("multi line2\nmulti line") 40 | record["number"] = kintone.DecimalField("12345") 41 | records = append(records, kintone.NewRecord(record)) 42 | 43 | _, err = app.AddRecords(records) 44 | 45 | return err 46 | } 47 | 48 | func TestSeekMethod(t *testing.T) { 49 | app := newApp() 50 | config.Query = "" 51 | _, err := getRecordsForSeekMethod(app, 0, nil, true) 52 | if err != nil { 53 | t.Error("TestSeekMethod is failed:", err) 54 | } 55 | 56 | } 57 | 58 | func TestGetRecordsHaveLimitOffset(t *testing.T) { 59 | app := newApp() 60 | buf := &bytes.Buffer{} 61 | config.Query = "limit 100 offset 0" 62 | err := exportRecords(app, nil, buf) 63 | if err != nil { 64 | t.Error("TestGetRecordsHaveLimitOffset is failed:", err) 65 | } 66 | 67 | } 68 | 69 | func TestGetRecordsHaveQuery(t *testing.T) { 70 | app := newApp() 71 | buf := &bytes.Buffer{} 72 | config.Query = "order by $id desc" 73 | err := exportRecordsByCursor(app, nil, buf) 74 | if err != nil { 75 | t.Error("TestGetRecordsHaveQuery is failed:", err) 76 | } 77 | } 78 | 79 | func TestExport1(t *testing.T) { 80 | buf := &bytes.Buffer{} 81 | 82 | app := newApp() 83 | makeTestData(app) 84 | 85 | fields := []string{"single_line_text", "multi_line_text", "number"} 86 | config.Query = "order by record_number asc" 87 | 88 | err := exportRecords(app, fields, buf) 89 | if err != nil { 90 | t.Error(err) 91 | } 92 | 93 | //output := buf.String() 94 | //fmt.Printf(output) 95 | fmt.Printf("\n") 96 | 97 | reader := csv.NewReader(buf) 98 | 99 | row, err := reader.Read() 100 | if err != nil { 101 | t.Error(err) 102 | } 103 | //fmt.Printf(row[0]) 104 | if row[0] != "single_line_text" { 105 | t.Error("Invalid field code") 106 | } 107 | if row[1] != "multi_line_text" { 108 | t.Error("Invalid field code") 109 | } 110 | if row[2] != "number" { 111 | t.Error("Invalid field code") 112 | } 113 | 114 | row, err = reader.Read() 115 | if err != nil { 116 | t.Error(err) 117 | } 118 | if row[0] != "single line1" { 119 | t.Error("Invalid 1st field value of row 1") 120 | } 121 | if row[1] != "multi line1\nmulti line" { 122 | t.Error("Invalid 2nd field value of row 1") 123 | } 124 | if row[2] != "12345" { 125 | t.Error("Invalid 3rd field value of row 1") 126 | } 127 | 128 | row, err = reader.Read() 129 | if err != nil { 130 | t.Error(err) 131 | } 132 | if row[0] != "single line2" { 133 | t.Error("Invalid 1st field value of row 2") 134 | } 135 | if row[1] != "multi line2\nmulti line" { 136 | t.Error("Invalid 2nd field value of row 2") 137 | } 138 | if row[2] != "12345" { 139 | t.Error("Invalid 3rd field value of row 2") 140 | } 141 | 142 | row, err = reader.Read() 143 | if err != io.EOF { 144 | t.Error("Invalid record count") 145 | } 146 | } 147 | 148 | func TestExport2(t *testing.T) { 149 | buf := &bytes.Buffer{} 150 | 151 | app := newApp() 152 | makeTestData(app) 153 | 154 | config.Query = "order by record_number asc" 155 | err := exportRecords(app, nil, buf) 156 | if err != nil { 157 | t.Error(err) 158 | } 159 | 160 | //output := buf.String() 161 | //fmt.Printf(output) 162 | 163 | reader := csv.NewReader(buf) 164 | 165 | row, err := reader.Read() 166 | if err != nil { 167 | t.Error(err) 168 | } 169 | //fmt.Printf(row[0]) 170 | if row[0] != "*" { 171 | t.Error("Invalid field code") 172 | } 173 | if row[1] != "single_line_text" { 174 | t.Error("Invalid field code") 175 | } 176 | if row[2] != "multi_line_text" { 177 | t.Error("Invalid field code") 178 | } 179 | if row[3] != "number" { 180 | t.Error("Invalid field code") 181 | } 182 | if row[4] != "table" { 183 | t.Error("Invalid field code") 184 | } 185 | if row[5] != "table_single_line_text" { 186 | t.Error("Invalid field code") 187 | } 188 | if row[6] != "table_multi_line_text" { 189 | t.Error("Invalid field code") 190 | } 191 | 192 | row, err = reader.Read() 193 | if err != nil { 194 | t.Error(err) 195 | } 196 | if row[0] != "*" { 197 | t.Error("Invalid 1st field value of row 1") 198 | } 199 | if row[1] != "single line1" { 200 | t.Error("Invalid 2nd field value of row 1") 201 | } 202 | if row[2] != "multi line1\nmulti line" { 203 | t.Error("Invalid 3rd field value of row 1") 204 | } 205 | if row[3] != "12345" { 206 | t.Error("Invalid 4th field value of row 1") 207 | } 208 | if row[5] != "table single line1" { 209 | t.Error("Invalid 5th field value of row 1") 210 | } 211 | if row[6] != "table multi line1\nmulti line" { 212 | t.Error("Invalid 6th field value of row 1") 213 | } 214 | 215 | row, err = reader.Read() 216 | if err != nil { 217 | t.Error(err) 218 | } 219 | if row[0] != "" { 220 | t.Error("Invalid 1st field value of row 2") 221 | } 222 | if row[1] != "single line1" { 223 | t.Error("Invalid 2nd field value of row 2") 224 | } 225 | if row[2] != "multi line1\nmulti line" { 226 | t.Error("Invalid 3rd field value of row 2") 227 | } 228 | if row[3] != "12345" { 229 | t.Error("Invalid 4th field value of row 2") 230 | } 231 | if row[5] != "table single line2" { 232 | t.Error("Invalid 5th field value of row 2") 233 | } 234 | if row[6] != "table multi line2\nmulti line" { 235 | t.Error("Invalid 6th field value of row 2") 236 | } 237 | 238 | row, err = reader.Read() 239 | if err != nil { 240 | t.Error(err) 241 | } 242 | if row[0] != "*" { 243 | t.Error("Invalid 1st field value of row 3") 244 | } 245 | if row[1] != "single line2" { 246 | t.Error("Invalid 2nd field value of row 3") 247 | } 248 | if row[2] != "multi line2\nmulti line" { 249 | t.Error("Invalid 3rd field value of row 3") 250 | } 251 | if row[3] != "12345" { 252 | t.Error("Invalid 4th field value of row 3") 253 | } 254 | if row[5] != "" { 255 | t.Error("Invalid 5th field value of row 3") 256 | } 257 | if row[6] != "" { 258 | t.Error("Invalid 6th field value of row 3") 259 | } 260 | 261 | row, err = reader.Read() 262 | if err != io.EOF { 263 | t.Error("Invalid record count") 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cli-kintone 2 | ========== 3 | cli-kintone is a command line utility for exporting and importing kintone App data. 4 | 5 | ## :warning::warning: This package has been deprecated. Please use [@kintone/cli-kintone](https://github.com/kintone/cli-kintone) instead. :warning::warning: 6 | 7 | ## Version 8 | 9 | 0.14.1 10 | 11 | ## Downloads 12 | 13 | These binaries are available for download. 14 | 15 | - Windows 16 | - Linux 17 | - Mac OS X 18 | 19 | https://github.com/kintone-labs/cli-kintone/releases 20 | 21 | ## Usage 22 | ```text 23 | Usage: 24 | cli-kintone [OPTIONS] 25 | 26 | Application Options: 27 | --import Import data from stdin. If "-f" is also specified, data is imported from the file instead 28 | --export Export kintone data to stdout 29 | -d= Domain name (specify the FQDN) 30 | -a= App ID (default: 0) 31 | -u= User's log in name 32 | -p= User's password 33 | -t= API token 34 | -g= Guest Space ID (default: 0) 35 | -o= Output format. Specify either 'json' or 'csv' (default: csv) 36 | -e= Character encoding (default: utf-8). 37 | Only support the encoding below both field code and data itself: 38 | 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or'euc-jp', 'gbk' or 'big5' 39 | -U= Basic authentication user name 40 | -P= Basic authentication password 41 | -q= Query string 42 | -c= Fields to export (comma separated). Specify the field code name 43 | -f= Input file path 44 | -b= Attachment file directory 45 | -D Delete records before insert. You can specify the deleting record condition by option "-q" 46 | -l= Position index of data in the input file (default: 1) 47 | -v, --version Version of cli-kintone 48 | 49 | Help Options: 50 | -h, --help Show this help message 51 | ``` 52 | ## Examples 53 | Note: 54 | * If you use Windows device, please specify cli-kintone.exe 55 | * Please set the PATH to cli-kintone to match your local environment beforehand. 56 | 57 | ### Export all columns from an app 58 | ``` 59 | cli-kintone --export -a -d -t 60 | ``` 61 | ### Export the specified columns to csv file as Shift-JIS encoding 62 | ``` 63 | cli-kintone --export -a -d -e sjis -c "$id, name1, name2" -t > 64 | ``` 65 | ### Import specified file into an App 66 | ``` 67 | cli-kintone --import -a -d -e sjis -t -f 68 | ``` 69 | Records are updated and/or added if the import file contains either an $id column (that represents the Record Number field), or a column representing a key field (denoted with a * symbol before the field code name, such as "\*mykeyfield"). 70 | 71 | If the value in the $id (or key field) column matches a record number value, that record will be updated. 72 | If the value in the $id (or key field) column is empty, a new record will be added. 73 | If the value in the $id (or key field) column does not match with any record number values, the import process will stop, and an error will occur. 74 | If an $id (or key field) column does not exist in the file, new records will be added, and no records will be updated. 75 | 76 | ### Export and download attachment files to ./mydownloads directory 77 | ``` 78 | cli-kintone --export -a -d -t -b mydownloads 79 | ``` 80 | ### Import and upload attachment files from ./myuploads directory 81 | > :warning: WARNING 82 | >- If the flag "-b" has NOT been specified, even though value of attachment fields in csv is empty or not, attachment fields will be skipped and not updated to kintone. 83 | > 84 | >- If the flag "-b" has been specified and value of attachment fields in csv is empty (empty is mean leave blank or ""), the data of attachment fields after importing to kintone will be removed. 85 | > - The conditions to remove all of attachment files 86 | > - The flag "-b" has been specified with directory path is required. 87 | > - Attachment columns are required for csv file but directory path is empty in csv. 88 | > - Attachment files are optional. 89 | > - The conditions to remove part of attachment files and update part of them 90 | > - The flag "-b" has been specified with directory path is required. 91 | > - Attachment columns are required for csv file. 92 | > - Attachment files are required if there's only part of them will be updated. 93 | > 94 | >Ex: CSV file to removed files in attachment fields 95 | >``` 96 | >"$id","Name","Department","File" 97 | >"1","Adam Clark","Planning","" 98 | >"2","Sarah Jones","HR","" 99 | >``` 100 | >  101 | 102 | ``` 103 | cli-kintone --import -a -d -t -b myuploads -f 104 | ``` 105 | ### Import and update by selecting a key to bulk update 106 | > :warning: WARNING 107 | > 108 | >The error message `The "$id" field and update key fields cannot be specified together in CSV import file.` will be displayed when both "$id" and key fields are specified in CSV import file. 109 | 110 | The key to bulk update must be specified within the INPUT_FILE by placing an * in front of the field code name, 111 | e.g. “update_date",“*id",“status". 112 | 113 | ``` 114 | cli-kintone --import -a -d -e sjis -t -f 115 | ``` 116 | ### Import CSV from line 25 of the input file 117 | ``` 118 | cli-kintone --import -a -d -t -f -l 25 119 | ``` 120 | ### Import from standard input (stdin) 121 | ``` 122 | printf "name,age\nJohn,37\nJane,29" | cli-kintone --import -a -d -t 123 | ``` 124 | 125 | ## Restrictions 126 | * The limit of each file size for uploading to attachments field is 10MB. 127 | * Client certificates cannot be used with cli-kintone. 128 | * The following record data cannot be retrieved: Field group, Blank space, Label, Border, Related records, Status, Assignee, Category 129 | 130 | ## Restriction of Encode/Decode 131 | * Windows command prompt may not display characters correctly like "譁�蟄怜喧縺�". 132 | This is due to compatibility issues between Chinese & Japanese characters and the Windows command prompt. 133 | * Chinese (Traditional/Simplified): Display wrong even if exporting with gbk or big5 encoding. 134 | * Japanese: Display wrong even if exporting with sjis or euc-jp encoding. 135 | 136 | In this case, display the data by specifying utf-8 encoding like below: 137 | ``` 138 | cli-kintone --export -a -d -e utf-8 139 | ``` 140 | *This issue only occurs when displaying data on Windows command prompt. Data import/export with other means work fine with gbk, big5, sjis and euc-jp encoding. 141 | 142 | ## Documents for Basic Usage 143 | - English: https://kintone.dev/en/tutorials/tool-guides/features-of-the-command-line-tool/ 144 | - Japanese: https://developer.cybozu.io/hc/ja/articles/202957070 145 | 146 | ## How to Build 147 | 148 | Requirement 149 | 150 | - Go 1.15.15 151 | - Git and Mercurial to be able to clone the packages 152 | 153 | [Mac OS X/Linux](./docs/BuildForMacLinux.md) 154 | 155 | [Windows](./docs/BuildForWindows.md) 156 | 157 | ## License 158 | 159 | GPL v2 160 | 161 | ## Copyright 162 | 163 | Copyright(c) Cybozu, Inc. 164 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | "strings" 9 | 10 | "github.com/howeyc/gopass" 11 | "github.com/kintone-labs/go-kintone" 12 | "golang.org/x/text/encoding" 13 | "golang.org/x/text/encoding/japanese" 14 | "golang.org/x/text/encoding/simplifiedchinese" 15 | "golang.org/x/text/encoding/traditionalchinese" 16 | "golang.org/x/text/encoding/unicode" 17 | 18 | flags "github.com/jessevdk/go-flags" 19 | ) 20 | 21 | // NAME of this package 22 | const NAME = "cli-kintone" 23 | 24 | // VERSION of this package 25 | const VERSION = "0.14.1" 26 | 27 | // IMPORT_ROW_LIMIT The maximum row will be import 28 | const IMPORT_ROW_LIMIT = 100 29 | 30 | // EXPORT_ROW_LIMIT The maximum row will be export 31 | const EXPORT_ROW_LIMIT = 500 32 | 33 | // Configure of this package 34 | type Configure struct { 35 | IsImport bool `long:"import" description:"Import data from stdin. If \"-f\" is also specified, data is imported from the file instead"` 36 | IsExport bool `long:"export" description:"Export kintone data to stdout"` 37 | Domain string `short:"d" default:"" description:"Domain name (specify the FQDN)"` 38 | AppID uint64 `short:"a" default:"0" description:"App ID"` 39 | Login string `short:"u" default:"" description:"User's log in name"` 40 | Password string `short:"p" default:"" description:"User's password"` 41 | APIToken string `short:"t" default:"" description:"API token"` 42 | GuestSpaceID uint64 `short:"g" default:"0" description:"Guest Space ID"` 43 | Format string `short:"o" default:"csv" description:"Output format. Specify either 'json' or 'csv'"` 44 | Encoding string `short:"e" default:"utf-8" description:"Character encoding (default: utf-8).\n Only support the encoding below both field code and data itself: \n 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or 'euc-jp', 'gbk' or 'big5'"` 45 | BasicAuthUser string `short:"U" default:"" description:"Basic authentication user name"` 46 | BasicAuthPassword string `short:"P" default:"" description:"Basic authentication password"` 47 | Query string `short:"q" default:"" description:"Query string"` 48 | Fields []string `short:"c" description:"Fields to export (comma separated). Specify the field code name"` 49 | FilePath string `short:"f" default:"" description:"Input file path"` 50 | FileDir string `short:"b" default:"" description:"Attachment file directory"` 51 | DeleteAll bool `short:"D" description:"Delete records before insert. You can specify the deleting record condition by option \"-q\""` 52 | Line uint64 `short:"l" default:"1" description:"Position index of data in the input file"` 53 | Version bool `short:"v" long:"version" description:"Version of cli-kintone"` 54 | } 55 | 56 | var config Configure 57 | 58 | // Column config 59 | // Column config is deprecated, replace using Cell config 60 | type Column struct { 61 | Code string 62 | Type string 63 | IsSubField bool 64 | Table string 65 | } 66 | 67 | // Columns config 68 | // Columns config is deprecated, replace using Row config 69 | type Columns []*Column 70 | 71 | // Cell config 72 | type Cell struct { 73 | Code string 74 | Type string 75 | IsSubField bool 76 | Table string 77 | Index int 78 | } 79 | 80 | // Row config 81 | type Row []*Cell 82 | 83 | func getFields(app *kintone.App) (map[string]*kintone.FieldInfo, error) { 84 | fields, err := app.Fields() 85 | if err != nil { 86 | return nil, err 87 | } 88 | return fields, nil 89 | } 90 | 91 | func getSupportedFields(app *kintone.App) (map[string]*kintone.FieldInfo, error) { 92 | fields, err := getFields(app) 93 | if err != nil { 94 | return nil, err 95 | } 96 | for key, field := range fields { 97 | switch field.Type { 98 | case "STATUS_ASSIGNEE", "CATEGORY", "STATUS": 99 | delete(fields, key) 100 | default: 101 | continue 102 | } 103 | } 104 | return fields, nil 105 | } 106 | 107 | // set column information from fieldinfo 108 | // This function is deprecated, replace using function getCell 109 | func getColumn(code string, fields map[string]*kintone.FieldInfo) *Column { 110 | // initialize values 111 | column := Column{Code: code, IsSubField: false, Table: ""} 112 | 113 | if code == "$id" { 114 | column.Type = kintone.FT_ID 115 | return &column 116 | } else if code == "$revision" { 117 | column.Type = kintone.FT_REVISION 118 | return &column 119 | } else { 120 | // is this code the one of sub field? 121 | for _, val := range fields { 122 | if val.Code == code { 123 | column.Type = val.Type 124 | return &column 125 | } 126 | if val.Type == kintone.FT_SUBTABLE { 127 | for _, subField := range val.Fields { 128 | if subField.Code == code { 129 | column.IsSubField = true 130 | column.Type = subField.Type 131 | column.Table = val.Code 132 | return &column 133 | } 134 | } 135 | } 136 | } 137 | } 138 | 139 | // the code is not found 140 | column.Type = "UNKNOWN" 141 | return &column 142 | } 143 | 144 | func containtString(arr []string, str string) bool { 145 | for _, a := range arr { 146 | if a == str { 147 | return true 148 | } 149 | } 150 | return false 151 | } 152 | 153 | // set Cell information from fieldinfo 154 | // function replace getColumn so getColumn is invalid name 155 | func getCell(code string, fields map[string]*kintone.FieldInfo) *Cell { 156 | // initialize values 157 | cell := Cell{Code: code, IsSubField: false, Table: ""} 158 | 159 | if code == "$id" { 160 | cell.Type = kintone.FT_ID 161 | return &cell 162 | } else if code == "$revision" { 163 | cell.Type = kintone.FT_REVISION 164 | return &cell 165 | } else { 166 | // is this code the one of sub field? 167 | for _, val := range fields { 168 | if val.Code == code { 169 | cell.Type = val.Type 170 | return &cell 171 | } 172 | if val.Type == kintone.FT_SUBTABLE { 173 | for _, subField := range val.Fields { 174 | if subField.Code == code { 175 | cell.IsSubField = true 176 | cell.Type = subField.Type 177 | cell.Table = val.Code 178 | return &cell 179 | } 180 | } 181 | } 182 | } 183 | } 184 | 185 | // the code is not found 186 | cell.Type = "UNKNOWN" 187 | return &cell 188 | } 189 | 190 | func getEncoding() encoding.Encoding { 191 | switch config.Encoding { 192 | case "utf-16": 193 | return unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) 194 | case "utf-16be-with-signature": 195 | return unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM) 196 | case "utf-16le-with-signature": 197 | return unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM) 198 | case "euc-jp": 199 | return japanese.EUCJP 200 | case "sjis": 201 | return japanese.ShiftJIS 202 | case "gbk": 203 | return simplifiedchinese.GBK 204 | case "big5": 205 | return traditionalchinese.Big5 206 | default: 207 | return nil 208 | } 209 | } 210 | 211 | func main() { 212 | var err error 213 | 214 | _, err = flags.ParseArgs(&config, os.Args[1:]) 215 | if err != nil { 216 | if os.Args[1] != "-h" && os.Args[1] != "--help" { 217 | fileExecute := os.Args[0] 218 | fmt.Printf("\nTry '%s --help' for more information.\n", fileExecute) 219 | } 220 | os.Exit(1) 221 | } 222 | 223 | if len(os.Args) > 0 && config.Version { 224 | fmt.Println(VERSION) 225 | os.Exit(0) 226 | } 227 | 228 | if len(os.Args) == 0 || config.AppID == 0 || (config.APIToken == "" && (config.Domain == "" || config.Login == "")) { 229 | helpArg := []string{"-h"} 230 | flags.ParseArgs(&config, helpArg) 231 | os.Exit(1) 232 | } 233 | 234 | if !strings.Contains(config.Domain, ".") { 235 | config.Domain += ".cybozu.com" 236 | } 237 | 238 | // Support set columm with comma separated (",") in arg 239 | var cols []string 240 | if len(config.Fields) > 0 { 241 | for _, field := range config.Fields { 242 | curField := strings.Split(field, ",") 243 | cols = append(cols, curField...) 244 | } 245 | config.Fields = nil 246 | for _, col := range cols { 247 | curFieldString := strings.TrimSpace(col) 248 | if curFieldString != "" { 249 | config.Fields = append(config.Fields, curFieldString) 250 | } 251 | } 252 | } 253 | 254 | var app *kintone.App 255 | if config.BasicAuthUser != "" && config.BasicAuthPassword == "" { 256 | fmt.Printf("Basic authentication password: ") 257 | pass, _ := gopass.GetPasswd() 258 | config.BasicAuthPassword = string(pass) 259 | } 260 | 261 | if config.APIToken == "" { 262 | if config.Password == "" { 263 | fmt.Printf("Password: ") 264 | pass, _ := gopass.GetPasswd() 265 | config.Password = string(pass) 266 | } 267 | 268 | app = &kintone.App{ 269 | Domain: config.Domain, 270 | User: config.Login, 271 | Password: config.Password, 272 | AppId: config.AppID, 273 | GuestSpaceId: config.GuestSpaceID, 274 | } 275 | } else { 276 | app = &kintone.App{ 277 | Domain: config.Domain, 278 | ApiToken: config.APIToken, 279 | AppId: config.AppID, 280 | GuestSpaceId: config.GuestSpaceID, 281 | } 282 | } 283 | 284 | if config.BasicAuthUser != "" { 285 | app.SetBasicAuth(config.BasicAuthUser, config.BasicAuthPassword) 286 | } 287 | 288 | app.SetUserAgentHeader(NAME + "/" + VERSION + " (" + runtime.GOOS + " " + runtime.GOARCH + ")") 289 | 290 | // Old logic without force import/export 291 | if config.IsImport == false && config.IsExport == false { 292 | if config.FilePath == "" { 293 | writer := getWriter(os.Stdout) 294 | if config.Query != "" { 295 | err = exportRecordsWithQuery(app, config.Fields, writer) 296 | } else { 297 | fields := config.Fields 298 | isAppendIdCustome := false 299 | if len(config.Fields) > 0 && !containtString(config.Fields, "$id") { 300 | fields = append(fields, "$id") 301 | isAppendIdCustome = true 302 | } 303 | 304 | err = exportRecordsBySeekMethod(app, writer, fields, isAppendIdCustome) 305 | } 306 | } else { 307 | err = importDataFromFile(app) 308 | } 309 | } 310 | if config.IsImport && config.IsExport { 311 | log.Fatal("The options --import and --export cannot be specified together!") 312 | } 313 | 314 | if config.IsImport { 315 | if config.FilePath == "" { 316 | err = importFromCSV(app, os.Stdin) 317 | } else { 318 | 319 | err = importDataFromFile(app) 320 | } 321 | } 322 | 323 | if config.IsExport { 324 | if config.FilePath != "" { 325 | log.Fatal("The -f option is not supported with the --export option.") 326 | } 327 | writer := getWriter(os.Stdout) 328 | if config.Query != "" { 329 | err = exportRecordsWithQuery(app, config.Fields, writer) 330 | } else { 331 | fields := config.Fields 332 | isAppendIdCustome := false 333 | if len(config.Fields) > 0 && !containtString(config.Fields, "$id") { 334 | fields = append(fields, "$id") 335 | isAppendIdCustome = true 336 | } 337 | err = exportRecordsBySeekMethod(app, writer, fields, isAppendIdCustome) 338 | } 339 | } 340 | if err != nil { 341 | log.Fatal(err) 342 | } 343 | } 344 | 345 | func importDataFromFile(app *kintone.App) error { 346 | var file *os.File 347 | var err error 348 | file, err = os.Open(config.FilePath) 349 | if err == nil { 350 | defer file.Close() 351 | err = importFromCSV(app, file) 352 | } 353 | return err 354 | } 355 | -------------------------------------------------------------------------------- /import-with-bulkRequest.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "regexp" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/kintone-labs/go-kintone" 17 | "golang.org/x/text/transform" 18 | ) 19 | 20 | // SubRecord structure 21 | type SubRecord struct { 22 | Id uint64 23 | Fields map[string]interface{} 24 | } 25 | 26 | func getReader(reader io.Reader) io.Reader { 27 | readerWithoutBOM := removeBOMCharacter(reader) 28 | 29 | encoding := getEncoding() 30 | if encoding == nil { 31 | return readerWithoutBOM 32 | } 33 | return transform.NewReader(readerWithoutBOM, encoding.NewDecoder()) 34 | } 35 | 36 | // delete specific records 37 | func deleteRecords(app *kintone.App, query string) error { 38 | var lastID uint64 39 | for { 40 | ids := make([]uint64, 0, IMPORT_ROW_LIMIT) 41 | 42 | r := regexp.MustCompile(`limit\s+\d+`) 43 | var _query string 44 | if r.MatchString(query) { 45 | _query = query 46 | } else { 47 | _query = query + fmt.Sprintf(" limit %v", IMPORT_ROW_LIMIT) 48 | } 49 | records, err := app.GetRecords([]string{"$id"}, _query) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | if len(records) == 0 { 55 | break 56 | } 57 | 58 | for _, record := range records { 59 | id := record.Id() 60 | ids = append(ids, id) 61 | } 62 | 63 | err = app.DeleteRecords(ids) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | if len(records) < IMPORT_ROW_LIMIT { 69 | break 70 | } 71 | if lastID == ids[0] { 72 | // prevent an inifinite loop 73 | return fmt.Errorf("Unexpected error occured during deleting") 74 | } 75 | lastID = ids[0] 76 | } 77 | 78 | return nil 79 | } 80 | func getSubRecord(tableName string, tables map[string]*SubRecord) *SubRecord { 81 | table := tables[tableName] 82 | if table == nil { 83 | fields := make(map[string]interface{}) 84 | table = &SubRecord{0, fields} 85 | tables[tableName] = table 86 | } 87 | 88 | return table 89 | } 90 | func addSubField(app *kintone.App, column *Column, col string, table *SubRecord) error { 91 | if len(col) == 0 { 92 | return nil 93 | } 94 | 95 | if column.Type == kintone.FT_FILE { 96 | field, err := uploadFiles(app, col) 97 | if err != nil { 98 | return err 99 | } 100 | if field != nil { 101 | table.Fields[column.Code] = field 102 | } 103 | } else { 104 | field := getField(column.Type, col) 105 | if field != nil { 106 | table.Fields[column.Code] = field 107 | } 108 | } 109 | return nil 110 | } 111 | func importFromCSV(app *kintone.App, _reader io.Reader) error { 112 | 113 | reader := csv.NewReader(getReader(_reader)) 114 | 115 | head := true 116 | var columns Columns 117 | 118 | var nextRowImport uint64 119 | nextRowImport = config.Line 120 | bulkRequests := &BulkRequests{} 121 | // retrieve field list 122 | fields, err := getFields(app) 123 | if err != nil { 124 | return err 125 | } 126 | 127 | if config.DeleteAll { 128 | err = deleteRecords(app, config.Query) 129 | if err != nil { 130 | return err 131 | } 132 | config.DeleteAll = false 133 | } 134 | 135 | keyField := "" 136 | hasTable := false 137 | var peeked *[]string 138 | var rowNumber uint64 139 | for rowNumber = 1; ; rowNumber++ { 140 | var err error 141 | var row []string 142 | if peeked == nil { 143 | row, err = reader.Read() 144 | if err == io.EOF { 145 | rowNumber-- 146 | break 147 | } else if err != nil { 148 | return err 149 | } 150 | } else { 151 | row = *peeked 152 | peeked = nil 153 | } 154 | if head && columns == nil { 155 | columns = make([]*Column, 0) 156 | for _, col := range row { 157 | re := regexp.MustCompile("^(.*)\\[(.*)\\]$") 158 | match := re.FindStringSubmatch(col) 159 | if match != nil { 160 | // for backward compatible 161 | column := &Column{Code: match[1], Type: match[2]} 162 | columns = append(columns, column) 163 | col = column.Code 164 | } else { 165 | if len(col) > 0 && col[0] == '*' { 166 | col = col[1:] 167 | keyField = col 168 | } 169 | column := getColumn(col, fields) 170 | if column.IsSubField { 171 | if row[0] == "" || row[0] == "*" { 172 | hasTable = true 173 | } 174 | } 175 | columns = append(columns, column) 176 | } 177 | } 178 | head = false 179 | } else { 180 | if rowNumber < config.Line { 181 | continue 182 | } 183 | var id uint64 184 | var err error 185 | record := make(map[string]interface{}) 186 | hasId := false 187 | 188 | for { 189 | tables := make(map[string]*SubRecord) 190 | for i, col := range row { 191 | column := columns[i] 192 | if column.IsSubField { 193 | table := getSubRecord(column.Table, tables) 194 | err := addSubField(app, column, col, table) 195 | if err != nil { 196 | return err 197 | } 198 | } else if column.Type == kintone.FT_SUBTABLE { 199 | if col != "" { 200 | subID, _ := strconv.ParseUint(col, 10, 64) 201 | table := getSubRecord(column.Code, tables) 202 | table.Id = subID 203 | } 204 | } else { 205 | if hasTable && row[0] != "*" { 206 | continue 207 | } 208 | if column.Code == "$id" { 209 | hasId = true 210 | if col != "" { 211 | id, _ = strconv.ParseUint(col, 10, 64) 212 | } 213 | } else if column.Code == "$revision" { 214 | 215 | } else if column.Type == kintone.FT_FILE { 216 | 217 | field, err := uploadFiles(app, col) 218 | if err != nil { 219 | return fmt.Errorf("\ncolumn[" + strconv.Itoa(i) +"]"+ " - row[" + strconv.FormatUint(rowNumber, 10)+"]: "+ err.Error()) 220 | } 221 | if field != nil { 222 | record[column.Code] = field 223 | } 224 | } else { 225 | if column.Code == keyField && col == "" { 226 | } else { 227 | field := getField(column.Type, col) 228 | if field != nil { 229 | record[column.Code] = field 230 | } 231 | } 232 | } 233 | } 234 | } 235 | for key, table := range tables { 236 | if len(table.Fields) == 0 { 237 | continue 238 | } 239 | if record[key] == nil { 240 | record[key] = getField(kintone.FT_SUBTABLE, "") 241 | } 242 | 243 | stf := record[key].(kintone.SubTableField) 244 | stf = append(stf, kintone.NewRecordWithId(table.Id, table.Fields)) 245 | record[key] = stf 246 | } 247 | 248 | if !hasTable { 249 | break 250 | } 251 | row, err = reader.Read() 252 | if err == io.EOF { 253 | break 254 | } else if err != nil { 255 | return err 256 | } 257 | if len(row) > 0 && row[0] == "*" { 258 | peeked = &row 259 | break 260 | } 261 | } 262 | 263 | if hasId && keyField != "" { 264 | log.Fatalln("The \"$id\" field and update key fields cannot be specified together in CSV import file."); 265 | } 266 | 267 | _, hasKeyField := record[keyField] 268 | if id != 0 || (keyField != "" && hasKeyField) { 269 | setRecordUpdatable(record, columns) 270 | err = bulkRequests.ImportDataUpdate(app, kintone.NewRecordWithId(id, record), keyField) 271 | if err != nil { 272 | log.Fatalln(err) 273 | } 274 | } else { 275 | err = bulkRequests.ImportDataInsert(app, kintone.NewRecord(record)) 276 | if err != nil { 277 | log.Fatalln(err) 278 | } 279 | } 280 | if (rowNumber-nextRowImport+1)%(ConstBulkRequestLimitRecordOption) == 0 { 281 | showTimeLog() 282 | fmt.Printf("Start from lines: %d - %d", nextRowImport, rowNumber) 283 | 284 | resp, err := bulkRequests.Request(app) 285 | bulkRequests.HandelResponse(resp, err, nextRowImport, rowNumber) 286 | 287 | bulkRequests.Requests = bulkRequests.Requests[:0] 288 | nextRowImport = rowNumber + 1 289 | 290 | } 291 | } 292 | } 293 | if len(bulkRequests.Requests) > 0 { 294 | showTimeLog() 295 | fmt.Printf("Start from lines: %d - %d", nextRowImport, rowNumber) 296 | resp, err := bulkRequests.Request(app) 297 | bulkRequests.HandelResponse(resp, err, nextRowImport, rowNumber) 298 | } 299 | showTimeLog() 300 | fmt.Printf("DONE\n") 301 | 302 | return nil 303 | } 304 | func setRecordUpdatable(record map[string]interface{}, columns Columns) { 305 | for _, col := range columns { 306 | switch col.Type { 307 | case 308 | kintone.FT_CREATOR, 309 | kintone.FT_MODIFIER, 310 | kintone.FT_CTIME, 311 | kintone.FT_MTIME: 312 | delete(record, col.Code) 313 | } 314 | } 315 | } 316 | func uploadFiles(app *kintone.App, value string) (kintone.FileField, error) { 317 | if config.FileDir == "" { 318 | return nil, nil 319 | } 320 | 321 | var ret kintone.FileField = []kintone.File{} 322 | value = strings.TrimSpace(value) 323 | if value == "" { 324 | return ret, nil 325 | } 326 | 327 | files := strings.Split(value, "\n") 328 | for _, file := range files { 329 | var path string 330 | if filepath.IsAbs(file) { 331 | path = file 332 | } else { 333 | path = filepath.Join(config.FileDir, file) 334 | } 335 | fileKey, err := uploadFile(app, path) 336 | if err != nil { 337 | return nil, err 338 | } 339 | ret = append(ret, kintone.File{FileKey: fileKey}) 340 | } 341 | return ret, nil 342 | } 343 | 344 | func uploadFile(app *kintone.App, filePath string) (string, error) { 345 | fi, err := os.Open(filePath) 346 | if err != nil { 347 | return "", err 348 | } 349 | defer fi.Close() 350 | 351 | fileinfo, err := fi.Stat() 352 | 353 | if err != nil { 354 | return "", err 355 | } 356 | 357 | if fileinfo.Size() > 10*1024*1024 { 358 | return "", fmt.Errorf("%s file must be less than 10 MB", filePath) 359 | } 360 | 361 | fileKey, err := app.Upload(path.Base(filePath), "application/octet-stream", fi) 362 | return fileKey, err 363 | } 364 | 365 | func getField(fieldType string, value string) interface{} { 366 | switch fieldType { 367 | case kintone.FT_SINGLE_LINE_TEXT: 368 | return kintone.SingleLineTextField(value) 369 | case kintone.FT_MULTI_LINE_TEXT: 370 | return kintone.MultiLineTextField(value) 371 | case kintone.FT_RICH_TEXT: 372 | return kintone.RichTextField(value) 373 | case kintone.FT_DECIMAL: 374 | return kintone.DecimalField(value) 375 | case kintone.FT_CALC: 376 | return nil 377 | case kintone.FT_CHECK_BOX: 378 | if len(value) == 0 { 379 | return kintone.CheckBoxField([]string{}) 380 | } 381 | return kintone.CheckBoxField(strings.Split(value, "\n")) 382 | case kintone.FT_RADIO: 383 | return kintone.RadioButtonField(value) 384 | case kintone.FT_SINGLE_SELECT: 385 | if len(value) == 0 { 386 | return kintone.SingleSelectField{Valid: false} 387 | } 388 | return kintone.SingleSelectField{String: value, Valid: true} 389 | 390 | case kintone.FT_MULTI_SELECT: 391 | if len(value) == 0 { 392 | return kintone.MultiSelectField([]string{}) 393 | } 394 | return kintone.MultiSelectField(strings.Split(value, "\n")) 395 | 396 | case kintone.FT_FILE: 397 | return nil 398 | case kintone.FT_LINK: 399 | return kintone.LinkField(value) 400 | case kintone.FT_DATE: 401 | if value == "" { 402 | return kintone.DateField{Valid: false} 403 | } 404 | dt, err := time.Parse("2006-01-02", value) 405 | if err == nil { 406 | return kintone.DateField{Date: dt, Valid: true} 407 | } 408 | dt, err = time.Parse("2006/1/2", value) 409 | if err == nil { 410 | return kintone.DateField{Date: dt, Valid: true} 411 | } 412 | 413 | case kintone.FT_TIME: 414 | if value == "" { 415 | return kintone.TimeField{Valid: false} 416 | } 417 | dt, err := time.Parse("15:04:05", value) 418 | if err == nil { 419 | return kintone.TimeField{Time: dt, Valid: true} 420 | } 421 | 422 | case kintone.FT_DATETIME: 423 | if value == "" { 424 | return kintone.DateTimeField{Valid: false} 425 | } 426 | dt, err := time.Parse(time.RFC3339, value) 427 | if err == nil { 428 | return kintone.DateTimeField{Time: dt, Valid: true} 429 | } 430 | case kintone.FT_USER: 431 | users := strings.Split(value, "\n") 432 | var ret kintone.UserField = []kintone.User{} 433 | for _, user := range users { 434 | if len(strings.TrimSpace(user)) > 0 { 435 | ret = append(ret, kintone.User{Code: user}) 436 | } 437 | } 438 | return ret 439 | case kintone.FT_ORGANIZATION: 440 | organizations := strings.Split(value, "\n") 441 | var ret kintone.OrganizationField = []kintone.Organization{} 442 | for _, organization := range organizations { 443 | if len(strings.TrimSpace(organization)) > 0 { 444 | ret = append(ret, kintone.Organization{Code: organization}) 445 | } 446 | } 447 | return ret 448 | case kintone.FT_GROUP: 449 | groups := strings.Split(value, "\n") 450 | var ret kintone.GroupField = []kintone.Group{} 451 | for _, group := range groups { 452 | if len(strings.TrimSpace(group)) > 0 { 453 | ret = append(ret, kintone.Group{Code: group}) 454 | } 455 | } 456 | return ret 457 | case kintone.FT_CATEGORY: 458 | return nil 459 | case kintone.FT_STATUS: 460 | return nil 461 | case kintone.FT_RECNUM: 462 | return nil 463 | case kintone.FT_ASSIGNEE: 464 | return nil 465 | case kintone.FT_CREATOR: 466 | return kintone.CreatorField{Code: value} 467 | case kintone.FT_MODIFIER: 468 | return kintone.ModifierField{Code: value} 469 | case kintone.FT_CTIME: 470 | dt, err := time.Parse(time.RFC3339, value) 471 | if err == nil { 472 | return kintone.CreationTimeField(dt) 473 | } 474 | case kintone.FT_MTIME: 475 | dt, err := time.Parse(time.RFC3339, value) 476 | if err == nil { 477 | return kintone.ModificationTimeField(dt) 478 | } 479 | case kintone.FT_SUBTABLE: 480 | sr := make([]*kintone.Record, 0) 481 | return kintone.SubTableField(sr) 482 | case "UNKNOWN": 483 | return nil 484 | } 485 | 486 | return kintone.SingleLineTextField(value) 487 | 488 | } 489 | -------------------------------------------------------------------------------- /bulkRequest.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "mime" 12 | "net/http" 13 | "net/http/cookiejar" 14 | "net/url" 15 | "os" 16 | "reflect" 17 | "time" 18 | 19 | "github.com/kintone-labs/go-kintone" 20 | ) 21 | 22 | const ( 23 | // ConstBulkRequestLimitRecordOption set option: record per bulkRequest 24 | ConstBulkRequestLimitRecordOption = 100 25 | 26 | // ConstBulkRequestLimitRequest kintone limited: The request count per bulkRequest 27 | ConstBulkRequestLimitRequest = 20 28 | 29 | // ConstRecordsLimitPerRequest kintone limited: The records count per request 30 | ConstRecordsLimitPerRequest = 100 31 | ) 32 | 33 | // BulkRequestItem item in the bulkRequest array 34 | type BulkRequestItem struct { 35 | Method string `json:"method"` 36 | API string `json:"api"` 37 | Payload interface{} `json:"payload,string"` 38 | } 39 | 40 | // BulkRequests BulkRequests structure 41 | type BulkRequests struct { 42 | Requests []*BulkRequestItem `json:"requests,string"` 43 | } 44 | 45 | // BulkRequestsError structure 46 | type BulkRequestsError struct { 47 | HTTPStatus string `json:"-"` 48 | HTTPStatusCode int `json:"-"` 49 | Message string `json:"message"` // Human readable message. 50 | ID string `json:"id"` // A unique error ID. 51 | Code string `json:"code"` // For machines. 52 | Errors interface{} `json:"errors"` 53 | } 54 | 55 | // BulkRequestsErrors structure 56 | type BulkRequestsErrors struct { 57 | HTTPStatus string `json:"-"` 58 | HTTPStatusCode int `json:"-"` 59 | Results []*BulkRequestsError `json:"results"` 60 | } 61 | 62 | // DataResponseBulkPOST structure 63 | type DataResponseBulkPOST struct { 64 | Results []interface{} `json:"results,string"` 65 | } 66 | 67 | // DataRequestRecordsPOST structure 68 | type DataRequestRecordsPOST struct { 69 | App uint64 `json:"app,string"` 70 | Records []*kintone.Record `json:"records"` 71 | } 72 | 73 | //DataRequestRecordPUT structure 74 | type DataRequestRecordPUT struct { 75 | ID uint64 `json:"id,string"` 76 | Record *kintone.Record `json:"record,string"` 77 | } 78 | 79 | // DataRequestRecordPUTByKey structure 80 | type DataRequestRecordPUTByKey struct { 81 | UpdateKey *kintone.UpdateKey `json:"updateKey,string"` 82 | Record *kintone.Record `json:"record,string"` 83 | } 84 | 85 | // DataRequestRecordsPUT - Data which will be update in the kintone app 86 | // DataRequestRecordsPUT.Records - array include DataRequestRecordPUTByKey and DataRequestRecordPUT 87 | type DataRequestRecordsPUT struct { 88 | App uint64 `json:"app,string"` 89 | Records []interface{} `json:"records"` 90 | } 91 | 92 | // SetRecord set data record for PUT method 93 | func (recordsPut *DataRequestRecordsPUT) SetRecord(record *kintone.Record) { 94 | recordPut := &DataRequestRecordPUT{ID: record.Id(), Record: record} 95 | recordsPut.Records = append(recordsPut.Records, recordPut) 96 | 97 | } 98 | 99 | // SetRecordWithKey set data record for PUT method 100 | func (recordsPut *DataRequestRecordsPUT) SetRecordWithKey(record *kintone.Record, keyCode string) { 101 | updateKey := &kintone.UpdateKey{FieldCode: keyCode, Field: record.Fields[keyCode].(kintone.UpdateKeyField)} 102 | delete(record.Fields, keyCode) 103 | recordPut := &DataRequestRecordPUTByKey{UpdateKey: updateKey, Record: record} 104 | recordsPut.Records = append(recordsPut.Records, recordPut) 105 | 106 | } 107 | 108 | // Request bulkRequest with multi method which included only one request 109 | func (bulk *BulkRequests) Request(app *kintone.App) (*DataResponseBulkPOST, interface{}) { 110 | 111 | data, _ := json.Marshal(bulk) 112 | req, err := newRequest(app, "POST", "bulkRequest", bytes.NewReader(data)) 113 | 114 | if err != nil { 115 | return nil, err 116 | } 117 | resp, err := Do(app, req) 118 | if err != nil { 119 | return nil, err 120 | } 121 | body, errParse := parseResponse(resp) 122 | if errParse != nil { 123 | return nil, errParse 124 | } 125 | resp1, err := bulk.Decode(body) 126 | if err != nil { 127 | return nil, kintone.ErrInvalidResponse 128 | } 129 | return resp1, nil 130 | } 131 | 132 | // Decode for BulkRequests 133 | func (bulk *BulkRequests) Decode(b []byte) (*DataResponseBulkPOST, error) { 134 | var rsp *DataResponseBulkPOST 135 | err := json.Unmarshal(b, &rsp) 136 | if err != nil { 137 | return nil, errors.New("Invalid JSON format: " + err.Error()) 138 | } 139 | return rsp, nil 140 | } 141 | 142 | // ImportDataUpdate import data with update 143 | func (bulk *BulkRequests) ImportDataUpdate(app *kintone.App, recordData *kintone.Record, keyField string) error { 144 | bulkReqLength := len(bulk.Requests) 145 | 146 | if bulkReqLength > ConstBulkRequestLimitRequest { 147 | return errors.New("The length of bulk request was too large, maximun is " + string(rune(ConstBulkRequestLimitRequest)) + " per request") 148 | } 149 | var dataPUT *DataRequestRecordsPUT 150 | if bulkReqLength > 0 { 151 | for i, bulkReqItem := range bulk.Requests { 152 | if bulkReqItem.Method != "PUT" { 153 | continue 154 | } 155 | // TODO: Check limit 100 record - kintone limit 156 | dataPUT = bulkReqItem.Payload.(*DataRequestRecordsPUT) 157 | if len(dataPUT.Records) == ConstRecordsLimitPerRequest { 158 | continue 159 | } 160 | if keyField != "" { 161 | dataPUT.SetRecordWithKey(recordData, keyField) 162 | } else { 163 | dataPUT.SetRecord(recordData) 164 | } 165 | bulk.Requests[i].Payload = dataPUT 166 | return nil 167 | } 168 | } 169 | 170 | recordsUpdate := make([]interface{}, 0) 171 | var recordPUT interface{} 172 | if keyField != "" { 173 | updateKey := &kintone.UpdateKey{FieldCode: keyField, Field: recordData.Fields[keyField].(kintone.UpdateKeyField)} 174 | delete(recordData.Fields, keyField) 175 | recordPUT = &DataRequestRecordPUTByKey{UpdateKey: updateKey, Record: recordData} 176 | } else { 177 | recordPUT = &DataRequestRecordPUT{ID: recordData.Id(), Record: recordData} 178 | } 179 | recordsUpdate = append(recordsUpdate, recordPUT) 180 | dataPUT = &DataRequestRecordsPUT{App: app.AppId, Records: recordsUpdate} 181 | requestPUTRecords := &BulkRequestItem{"PUT", kintoneURLPath("records", app.GuestSpaceId), dataPUT} 182 | bulk.Requests = append(bulk.Requests, requestPUTRecords) 183 | 184 | return nil 185 | 186 | } 187 | 188 | // ImportDataInsert import data with insert only 189 | func (bulk *BulkRequests) ImportDataInsert(app *kintone.App, recordData *kintone.Record) error { 190 | bulkReqLength := len(bulk.Requests) 191 | 192 | if bulkReqLength > ConstBulkRequestLimitRequest { 193 | return errors.New("The length of bulk request was too large, maximun is " + string(rune(ConstBulkRequestLimitRequest)) + " per request") 194 | } 195 | var dataPOST *DataRequestRecordsPOST 196 | if bulkReqLength > 0 { 197 | for i, bulkReqItem := range bulk.Requests { 198 | if bulkReqItem.Method != "POST" { 199 | continue 200 | } 201 | dataPOST = bulkReqItem.Payload.(*DataRequestRecordsPOST) 202 | if len(dataPOST.Records) == ConstRecordsLimitPerRequest { 203 | continue 204 | } 205 | // TODO: Check limit 100 record - kintone limit 206 | dataPOST.Records = append(dataPOST.Records, recordData) 207 | bulk.Requests[i].Payload = dataPOST 208 | return nil 209 | } 210 | } 211 | recordsInsert := make([]*kintone.Record, 0) 212 | recordsInsert = append(recordsInsert, recordData) 213 | dataPOST = &DataRequestRecordsPOST{app.AppId, recordsInsert} 214 | requestPostRecords := &BulkRequestItem{"POST", kintoneURLPath("records", app.GuestSpaceId), dataPOST} 215 | bulk.Requests = append(bulk.Requests, requestPostRecords) 216 | 217 | return nil 218 | 219 | } 220 | 221 | // kintoneURLPath get path URL of kintone api 222 | func kintoneURLPath(apiName string, GuestSpaceID uint64) string { 223 | var path string 224 | if GuestSpaceID == 0 { 225 | path = fmt.Sprintf("/k/v1/%s.json", apiName) 226 | } else { 227 | path = fmt.Sprintf("/k/guest/%d/v1/%s.json", GuestSpaceID, apiName) 228 | } 229 | return path 230 | } 231 | 232 | func newRequest(app *kintone.App, method, api string, body io.Reader) (*http.Request, error) { 233 | path := kintoneURLPath(api, app.GuestSpaceId) 234 | u := url.URL{ 235 | Scheme: "https", 236 | Host: app.Domain, 237 | Path: path, 238 | } 239 | req, err := http.NewRequest(method, u.String(), body) 240 | if err != nil { 241 | return nil, err 242 | } 243 | if app.HasBasicAuth() { 244 | req.SetBasicAuth(app.GetBasicAuthUser(), app.GetBasicAuthPassword()) 245 | } 246 | if len(app.ApiToken) == 0 { 247 | req.Header.Set("X-Cybozu-Authorization", base64.StdEncoding.EncodeToString([]byte(app.User+":"+app.Password))) 248 | } else { 249 | req.Header.Set("X-Cybozu-API-Token", app.ApiToken) 250 | } 251 | 252 | if len(app.GetUserAgentHeader()) != 0 { 253 | req.Header.Set("User-Agent", app.GetUserAgentHeader()) 254 | } 255 | 256 | req.Header.Set("Content-Type", "application/json") 257 | return req, nil 258 | } 259 | 260 | // Do Request to webservice 261 | func Do(app *kintone.App, req *http.Request) (*http.Response, error) { 262 | if app.Client == nil { 263 | jar, err := cookiejar.New(nil) 264 | if err != nil { 265 | return nil, err 266 | } 267 | app.Client = &http.Client{Jar: jar} 268 | } 269 | if app.Timeout == time.Duration(0) { 270 | app.Timeout = kintone.DEFAULT_TIMEOUT 271 | } 272 | 273 | type result struct { 274 | resp *http.Response 275 | err error 276 | } 277 | done := make(chan result, 1) 278 | go func() { 279 | resp, err := app.Client.Do(req) 280 | done <- result{resp, err} 281 | }() 282 | 283 | type requestCanceler interface { 284 | CancelRequest(*http.Request) 285 | } 286 | 287 | select { 288 | case r := <-done: 289 | return r.resp, r.err 290 | case <-time.After(app.Timeout): 291 | if canceller, ok := app.Client.Transport.(requestCanceler); ok { 292 | canceller.CancelRequest(req) 293 | } else { 294 | go func() { 295 | r := <-done 296 | if r.err == nil { 297 | r.resp.Body.Close() 298 | } 299 | }() 300 | } 301 | return nil, kintone.ErrTimeout 302 | } 303 | } 304 | func isJSON(contentType string) bool { 305 | mediatype, _, err := mime.ParseMediaType(contentType) 306 | if err != nil { 307 | return false 308 | } 309 | return mediatype == "application/json" 310 | } 311 | 312 | func parseResponse(resp *http.Response) ([]byte, interface{}) { 313 | body, err := ioutil.ReadAll(resp.Body) 314 | resp.Body.Close() 315 | if err != nil { 316 | return nil, err 317 | } 318 | if resp.StatusCode != http.StatusOK { 319 | if !isJSON(resp.Header.Get("Content-Type")) { 320 | return nil, &kintone.AppError{ 321 | HttpStatus: resp.Status, 322 | HttpStatusCode: resp.StatusCode, 323 | } 324 | } 325 | 326 | var appErrorBulkRequest BulkRequestsErrors 327 | appErrorBulkRequest.HTTPStatus = resp.Status 328 | appErrorBulkRequest.HTTPStatusCode = resp.StatusCode 329 | json.Unmarshal(body, &appErrorBulkRequest) 330 | 331 | if len(appErrorBulkRequest.Results) == 0 { 332 | var appErrorRequest BulkRequestsError 333 | appErrorRequest.HTTPStatus = resp.Status 334 | appErrorRequest.HTTPStatusCode = resp.StatusCode 335 | json.Unmarshal(body, &appErrorRequest) 336 | 337 | return nil, &appErrorRequest 338 | } 339 | return nil, &appErrorBulkRequest 340 | } 341 | return body, nil 342 | } 343 | 344 | // ErrorResponse show error detail when the bulkRequest has error(s) 345 | type ErrorResponse struct { 346 | ID string 347 | Code string 348 | Status string 349 | Message string 350 | Errors interface{} 351 | } 352 | 353 | func (err *ErrorResponse) show(prefix string) { 354 | fmt.Println("ID: ", err.ID) 355 | fmt.Println("Code: ", err.Code) 356 | if err.Status != "" { 357 | fmt.Println("Status: ", err.Status) 358 | } 359 | fmt.Println("Message: ", err.Message) 360 | fmt.Printf(prefix + "Errors detail: ") 361 | if err.Errors != nil { 362 | fmt.Printf("\n") 363 | for indx, val := range err.Errors.(map[string]interface{}) { 364 | fieldMessage := val.(map[string]interface{}) 365 | detailMessage := fieldMessage["messages"].([]interface{}) 366 | fmt.Printf("%v '%v': ", prefix, indx) 367 | for i, mess := range detailMessage { 368 | if i > 0 { 369 | fmt.Printf(", ") 370 | } 371 | fmt.Printf(mess.(string)) 372 | } 373 | fmt.Printf("\n") 374 | } 375 | fmt.Printf("\n") 376 | } else { 377 | fmt.Printf("(none)\n\n") 378 | } 379 | 380 | } 381 | 382 | // HandelResponse for bulkRequest 383 | func (bulk *BulkRequests) HandelResponse(rep *DataResponseBulkPOST, err interface{}, lastRowImport, rowNumber uint64) { 384 | 385 | if err != nil { 386 | fmt.Printf(" => ERROR OCCURRED\n") 387 | CLIMessage := fmt.Sprintf("ERROR.\nFor error details, please read the details above.\n") 388 | CLIMessage += fmt.Sprintf("Lines %d to %d of the imported file contain errors. Please fix the errors on the file, and re-import it with the flag \"-l %d\"\n", lastRowImport, rowNumber, lastRowImport) 389 | 390 | method := map[string]string{"POST": "INSERT", "PUT": "UPDATE"} 391 | methodOccuredError := "" 392 | if reflect.TypeOf(err).String() != "*main.BulkRequestsErrors" { 393 | if reflect.TypeOf(err).String() != "*main.BulkRequestsError" { 394 | fmt.Printf("\n") 395 | fmt.Println(err) 396 | fmt.Printf("\n") 397 | // Reset CLI Message 398 | CLIMessage = "" 399 | } else { 400 | errorResp := &ErrorResponse{} 401 | errorResp.Status = err.(*BulkRequestsError).HTTPStatus 402 | errorResp.Message = err.(*BulkRequestsError).Message 403 | errorResp.Errors = err.(*BulkRequestsError).Errors 404 | errorResp.ID = err.(*BulkRequestsError).ID 405 | errorResp.Code = err.(*BulkRequestsError).Code 406 | errorResp.show("") 407 | } 408 | } else { 409 | errorsResp := err.(*BulkRequestsErrors) 410 | for idx, errorItem := range errorsResp.Results { 411 | if errorItem.Code == "" { 412 | continue 413 | } 414 | errorResp := &ErrorResponse{} 415 | errorResp.ID = errorItem.ID 416 | errorResp.Code = errorItem.Code 417 | errorResp.Status = errorsResp.HTTPStatus 418 | errorResp.Message = errorItem.Message 419 | errorResp.Errors = errorItem.Errors 420 | 421 | errorResp.show("") 422 | methodOccuredError = method[bulk.Requests[idx].Method] 423 | } 424 | } 425 | showTimeLog() 426 | fmt.Printf("PROCESS STOPPED!\n\n") 427 | if CLIMessage != "" { 428 | fmt.Println(methodOccuredError, CLIMessage) 429 | } 430 | os.Exit(1) 431 | } 432 | fmt.Println(" => SUCCESS") 433 | } 434 | func showTimeLog() { 435 | fmt.Printf("%v: ", time.Now().Format("[2006-01-02 15:04:05]")) 436 | } 437 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | {description} 294 | Copyright (C) {year} {fullname} 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /export.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "reflect" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/kintone-labs/go-kintone" 14 | "golang.org/x/text/transform" 15 | ) 16 | 17 | const ( 18 | SUBTABLE_ROW_PREFIX = "*" 19 | RECORD_NOT_FOUND = "No record found. \nPlease check your query or permission settings." 20 | ) 21 | 22 | func checkNoRecord(records []*kintone.Record) { 23 | if len(records) < 1 { 24 | fmt.Println(RECORD_NOT_FOUND) 25 | os.Exit(1) 26 | } 27 | } 28 | 29 | func getRecordsForSeekMethod(app *kintone.App, id uint64, fields []string, isRecordFound bool) ([]*kintone.Record, error) { 30 | query := fmt.Sprintf(" order by $id desc limit %v", EXPORT_ROW_LIMIT) 31 | if id > 0 { 32 | query = "$id < " + fmt.Sprintf("%v", id) + query 33 | } 34 | records, err := app.GetRecords(fields, query) 35 | if err != nil { 36 | return nil, err 37 | } 38 | if isRecordFound { 39 | checkNoRecord(records) 40 | } 41 | return records, nil 42 | } 43 | 44 | func getRow(app *kintone.App) (Row, error) { 45 | var row Row 46 | // retrieve field list 47 | fields, err := getSupportedFields(app) 48 | if err != nil { 49 | return row, err 50 | } 51 | 52 | if config.Fields == nil { 53 | row = makeRow(fields) 54 | } else { 55 | row = makePartialRow(fields, config.Fields) 56 | } 57 | fixOrderCell(row) 58 | return row, err 59 | } 60 | 61 | func fixOrderCell(row Row) { 62 | for x := range row { 63 | hasIdOrRevision := x == 0 || x == 1 64 | if hasIdOrRevision { 65 | continue 66 | } 67 | y := x + 1 68 | for y = range row { 69 | if row[x].Index < row[y].Index { 70 | temp := row[x] 71 | row[x] = row[y] 72 | row[y] = temp 73 | } 74 | } 75 | } 76 | } 77 | 78 | func downloadFile(app *kintone.App, field interface{}, dir string) error { 79 | if config.FileDir == "" { 80 | return nil 81 | } 82 | 83 | v, ok := field.(kintone.FileField) 84 | if !ok { 85 | return nil 86 | } 87 | 88 | if len(v) == 0 { 89 | return nil 90 | } 91 | 92 | fileDir := fmt.Sprintf("%s%c%s", config.FileDir, os.PathSeparator, dir) 93 | if err := os.MkdirAll(fileDir, 0777); err != nil { 94 | return err 95 | } 96 | for idx, file := range v { 97 | fileName := getUniqueFileName(file.Name, fileDir) 98 | path := fmt.Sprintf("%s%c%s", fileDir, os.PathSeparator, fileName) 99 | data, err := app.Download(file.FileKey) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | fo, err := os.Create(path) 105 | if err != nil { 106 | return err 107 | } 108 | defer fo.Close() 109 | 110 | // make a buffer to keep chunks that are read 111 | buf := make([]byte, 256*1024) 112 | for { 113 | // read a chunk 114 | n, err := data.Reader.Read(buf) 115 | if err != nil && err != io.EOF { 116 | return err 117 | } 118 | if n == 0 { 119 | break 120 | } 121 | 122 | // write a chunk 123 | if _, err := fo.Write(buf[:n]); err != nil { 124 | return err 125 | } 126 | } 127 | 128 | v[idx].Name = fmt.Sprintf("%s%c%s", dir, os.PathSeparator, fileName) 129 | } 130 | 131 | return nil 132 | } 133 | 134 | func escapeCol(s string) string { 135 | return strings.Replace(s, "\"", "\"\"", -1) 136 | } 137 | 138 | func exportRecordsBySeekMethod(app *kintone.App, writer io.Writer, fields []string, isAppendIdCustome bool) error { 139 | row, err := getRow(app) 140 | hasTable := hasSubTable(row) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | if config.Format == "json" { 146 | err := writeRecordsBySeekMethodForJson(app, 0, writer, 0, fields, true, isAppendIdCustome) 147 | return err 148 | } 149 | return writeRecordsBySeekMethodForCsv(app, 0, writer, row, hasTable, 0, fields, true, isAppendIdCustome) 150 | } 151 | 152 | func exportRecordsWithQuery(app *kintone.App, fields []string, writer io.Writer) error { 153 | containLimit := regexp.MustCompile(`limit\s+\d+`) 154 | containOffset := regexp.MustCompile(`offset\s+\d+`) 155 | 156 | hasLimit := containLimit.MatchString(config.Query) 157 | hasOffset := containOffset.MatchString(config.Query) 158 | 159 | if hasLimit || hasOffset { 160 | return exportRecords(app, fields, writer) 161 | } 162 | return exportRecordsByCursor(app, fields, writer) 163 | } 164 | 165 | func exportRecords(app *kintone.App, fields []string, writer io.Writer) error { 166 | records, err := app.GetRecords(fields, config.Query) 167 | if err != nil { 168 | return err 169 | } 170 | checkNoRecord(records) 171 | if config.Format == "json" { 172 | fmt.Fprint(writer, "{\"records\": [\n") 173 | _, err = writeRecordsJSON(app, writer, records, 0, false) 174 | if err != nil { 175 | return err 176 | } 177 | fmt.Fprint(writer, "\n]}") 178 | } else { 179 | row, err := getRow(app) 180 | hasTable := hasSubTable(row) 181 | 182 | if err != nil { 183 | return err 184 | } 185 | _, err = writeRecordsCsv(app, writer, records, row, hasTable, 0, false) 186 | if err != nil { 187 | return err 188 | } 189 | } 190 | 191 | if err != nil { 192 | return err 193 | } 194 | 195 | return nil 196 | } 197 | 198 | func exportRecordsByCursor(app *kintone.App, fields []string, writer io.Writer) error { 199 | if config.Format == "json" { 200 | return exportRecordsByCursorForJSON(app, fields, writer) 201 | } 202 | return exportRecordsByCursorForCsv(app, fields, writer) 203 | } 204 | 205 | func exportRecordsByCursorForJSON(app *kintone.App, fields []string, writer io.Writer) error { 206 | cursor, err := app.CreateCursor(fields, config.Query, EXPORT_ROW_LIMIT) 207 | if err != nil { 208 | return err 209 | } 210 | index := uint64(0) 211 | for { 212 | recordsCursor, err := getAllRecordsByCursor(app, cursor.Id) 213 | if err != nil { 214 | return err 215 | } 216 | if index == 0 { 217 | fmt.Fprint(writer, "{\"records\": [\n") 218 | } 219 | index, err = writeRecordsJSON(app, writer, recordsCursor.Records, index, false) 220 | if err != nil { 221 | return err 222 | } 223 | 224 | if !recordsCursor.Next { 225 | fmt.Fprint(writer, "\n]}") 226 | break 227 | } 228 | } 229 | 230 | return nil 231 | } 232 | 233 | func exportRecordsByCursorForCsv(app *kintone.App, fields []string, writer io.Writer) error { 234 | cursor, err := app.CreateCursor(fields, config.Query, EXPORT_ROW_LIMIT) 235 | if err != nil { 236 | return err 237 | } 238 | 239 | row, err := getRow(app) 240 | if err != nil { 241 | return err 242 | } 243 | hasTable := hasSubTable(row) 244 | index := uint64(0) 245 | for { 246 | recordsCursor, err := getAllRecordsByCursor(app, cursor.Id) 247 | if err != nil { 248 | return err 249 | } 250 | index, err = writeRecordsCsv(app, writer, recordsCursor.Records, row, hasTable, index, false) 251 | if err != nil { 252 | return err 253 | } 254 | 255 | if !recordsCursor.Next { 256 | break 257 | } 258 | } 259 | return nil 260 | } 261 | 262 | func getAllRecordsByCursor(app *kintone.App, id string) (*kintone.GetRecordsCursorResponse, error) { 263 | recordsCursor, err := app.GetRecordsByCursor(id) 264 | if err != nil { 265 | return nil, err 266 | } 267 | checkNoRecord(recordsCursor.Records) 268 | return recordsCursor, nil 269 | } 270 | 271 | func getType(f interface{}) string { 272 | switch f.(type) { 273 | case kintone.SingleLineTextField: 274 | return kintone.FT_SINGLE_LINE_TEXT 275 | case kintone.MultiLineTextField: 276 | return kintone.FT_MULTI_LINE_TEXT 277 | case kintone.RichTextField: 278 | return kintone.FT_RICH_TEXT 279 | case kintone.DecimalField: 280 | return kintone.FT_DECIMAL 281 | case kintone.CalcField: 282 | return kintone.FT_CALC 283 | case kintone.CheckBoxField: 284 | return kintone.FT_CHECK_BOX 285 | case kintone.RadioButtonField: 286 | return kintone.FT_RADIO 287 | case kintone.SingleSelectField: 288 | return kintone.FT_SINGLE_SELECT 289 | case kintone.MultiSelectField: 290 | return kintone.FT_MULTI_SELECT 291 | case kintone.FileField: 292 | return kintone.FT_FILE 293 | case kintone.LinkField: 294 | return kintone.FT_LINK 295 | case kintone.DateField: 296 | return kintone.FT_DATE 297 | case kintone.TimeField: 298 | return kintone.FT_TIME 299 | case kintone.DateTimeField: 300 | return kintone.FT_DATETIME 301 | case kintone.UserField: 302 | return kintone.FT_USER 303 | case kintone.OrganizationField: 304 | return kintone.FT_ORGANIZATION 305 | case kintone.GroupField: 306 | return kintone.FT_GROUP 307 | case kintone.CategoryField: 308 | return kintone.FT_CATEGORY 309 | case kintone.StatusField: 310 | return kintone.FT_STATUS 311 | case kintone.RecordNumberField: 312 | return kintone.FT_RECNUM 313 | case kintone.AssigneeField: 314 | return kintone.FT_ASSIGNEE 315 | case kintone.CreatorField: 316 | return kintone.FT_CREATOR 317 | case kintone.ModifierField: 318 | return kintone.FT_MODIFIER 319 | case kintone.CreationTimeField: 320 | return kintone.FT_CTIME 321 | case kintone.ModificationTimeField: 322 | return kintone.FT_MTIME 323 | case kintone.SubTableField: 324 | return kintone.FT_SUBTABLE 325 | } 326 | return "" 327 | } 328 | 329 | func getUniqueFileName(filename, dir string) string { 330 | filenameOuput := filename 331 | fileExt := filepath.Ext(filename) 332 | fileBaseName := filename[0 : len(filename)-len(fileExt)] 333 | index := 0 334 | parentDir := fmt.Sprintf("%s%c", dir, os.PathSeparator) 335 | if dir == "" { 336 | parentDir = "" 337 | } 338 | for { 339 | fileFullPath := fmt.Sprintf("%s%s", parentDir, filenameOuput) 340 | if !isExistFile(fileFullPath) { 341 | break 342 | } 343 | index++ 344 | filenameOuput = fmt.Sprintf("%s (%d)%s", fileBaseName, index, fileExt) 345 | } 346 | return filenameOuput 347 | } 348 | 349 | func getSubTableRowCount(record *kintone.Record, row []*Cell) int { 350 | var ret = 1 351 | for _, cell := range row { 352 | if cell.IsSubField { 353 | subTable := record.Fields[cell.Table].(kintone.SubTableField) 354 | 355 | count := len(subTable) 356 | if count > ret { 357 | ret = count 358 | } 359 | } 360 | } 361 | 362 | return ret 363 | } 364 | 365 | func getWriter(writer io.Writer) io.Writer { 366 | encoding := getEncoding() 367 | if encoding == nil { 368 | return writer 369 | } 370 | return transform.NewWriter(writer, encoding.NewEncoder()) 371 | } 372 | 373 | func hasSubTable(row []*Cell) bool { 374 | for _, cell := range row { 375 | if cell.IsSubField { 376 | return true 377 | } 378 | } 379 | return false 380 | } 381 | 382 | func isExistFile(fileFullPath string) bool { 383 | _, fileNotExist := os.Stat(fileFullPath) 384 | return !os.IsNotExist(fileNotExist) 385 | } 386 | 387 | func makeRow(fields map[string]*kintone.FieldInfo) Row { 388 | row := make([]*Cell, 0) 389 | 390 | var cell *Cell 391 | 392 | cell = &Cell{Code: "$id", Type: kintone.FT_ID} 393 | row = append(row, cell) 394 | cell = &Cell{Code: "$revision", Type: kintone.FT_REVISION} 395 | row = append(row, cell) 396 | 397 | for _, val := range fields { 398 | if val.Code == "" { 399 | continue 400 | } 401 | if val.Type == kintone.FT_SUBTABLE { 402 | // record id for subtable 403 | cell := &Cell{Code: val.Code, Type: val.Type, Index: val.Index} 404 | row = append(row, cell) 405 | 406 | for _, subField := range val.Fields { 407 | cell := &Cell{Code: subField.Code, Type: subField.Type, IsSubField: true, Table: val.Code, Index: subField.Index} 408 | row = append(row, cell) 409 | } 410 | } else { 411 | cell := &Cell{Code: val.Code, Type: val.Type, Index: val.Index} 412 | row = append(row, cell) 413 | } 414 | } 415 | 416 | return row 417 | } 418 | 419 | func makePartialRow(fields map[string]*kintone.FieldInfo, partialFields []string) Row { 420 | row := make([]*Cell, 0) 421 | 422 | maxFieldIdx := 0 423 | for index, val := range partialFields { 424 | cell := getCell(val, fields) 425 | if cell.Type == "UNKNOWN" || cell.IsSubField { 426 | continue 427 | } 428 | currentFieldIdx := index + maxFieldIdx 429 | if cell.Type == kintone.FT_SUBTABLE { 430 | // record id for subtable 431 | cell := &Cell{Code: cell.Code, Type: cell.Type, Index: currentFieldIdx} 432 | row = append(row, cell) 433 | 434 | // append all sub fields 435 | field := fields[val] 436 | maxSubFieldIdx := 0 437 | for _, subField := range field.Fields { 438 | currentSubFieldIdx := subField.Index + maxFieldIdx 439 | cell := &Cell{Code: subField.Code, Type: subField.Type, IsSubField: true, Table: val, Index: currentSubFieldIdx} 440 | row = append(row, cell) 441 | if currentSubFieldIdx > maxSubFieldIdx { 442 | maxSubFieldIdx = currentSubFieldIdx 443 | } 444 | } 445 | if maxSubFieldIdx > maxFieldIdx { 446 | maxFieldIdx = maxSubFieldIdx 447 | } 448 | } else { 449 | cell := &Cell{Code: cell.Code, Type: cell.Type, Index: currentFieldIdx} 450 | maxFieldIdx = currentFieldIdx 451 | row = append(row, cell) 452 | } 453 | } 454 | return row 455 | } 456 | 457 | func writeHeaderCsv(writer io.Writer, hasTable bool, row Row) { 458 | i := 0 459 | if hasTable { 460 | fmt.Fprint(writer, SUBTABLE_ROW_PREFIX) 461 | i++ 462 | } 463 | for _, cell := range row { 464 | if i > 0 { 465 | fmt.Fprint(writer, ",") 466 | } 467 | fmt.Fprint(writer, "\""+cell.Code+"\"") 468 | i++ 469 | } 470 | fmt.Fprint(writer, "\r\n") 471 | } 472 | 473 | func writeRecordsJSON(app *kintone.App, writer io.Writer, records []*kintone.Record, i uint64, isAppendIdCustome bool) (uint64, error) { 474 | for _, record := range records { 475 | if i > 0 { 476 | fmt.Fprint(writer, ",\n") 477 | } 478 | rowID := record.Id() 479 | if rowID == 0 || isAppendIdCustome { 480 | rowID = i 481 | } 482 | // Download file to local folder that is the value of param -b 483 | for fieldCode, fieldInfo := range record.Fields { 484 | fieldType := reflect.TypeOf(fieldInfo).String() 485 | if fieldType == "kintone.FileField" { 486 | dir := fmt.Sprintf("%s-%d", fieldCode, rowID) 487 | err := downloadFile(app, fieldInfo, dir) 488 | if err != nil { 489 | return 0, err 490 | 491 | } 492 | } else if fieldType == "kintone.SubTableField" { 493 | subTable := fieldInfo.(kintone.SubTableField) 494 | for subTableIndex, subTableValue := range subTable { 495 | for fieldCodeInSubTable, fieldValueInSubTable := range subTableValue.Fields { 496 | if reflect.TypeOf(fieldValueInSubTable).String() == "kintone.FileField" { 497 | dir := fmt.Sprintf("%s-%d-%d", fieldCodeInSubTable, rowID, subTableIndex) 498 | err := downloadFile(app, fieldValueInSubTable, dir) 499 | if err != nil { 500 | return 0, err 501 | 502 | } 503 | } 504 | } 505 | } 506 | } 507 | } 508 | jsonArray, _ := record.MarshalJSON() 509 | json := string(jsonArray) 510 | _, err := fmt.Fprint(writer, json) 511 | if err != nil { 512 | return 0, err 513 | } 514 | i++ 515 | } 516 | return i, nil 517 | } 518 | 519 | func writeRecordsCsv(app *kintone.App, writer io.Writer, records []*kintone.Record, row Row, hasTable bool, i uint64, isAppendIdCustome bool) (uint64, error) { 520 | if i == 0 { 521 | writeHeaderCsv(writer, hasTable, row) 522 | } 523 | for _, record := range records { 524 | rowID := record.Id() 525 | if rowID == 0 || isAppendIdCustome { 526 | rowID = i 527 | } 528 | 529 | // determine subtable's row count 530 | rowNum := getSubTableRowCount(record, row) 531 | for j := 0; j < rowNum; j++ { 532 | k := 0 533 | if hasTable { 534 | if j == 0 { 535 | fmt.Fprint(writer, "*") 536 | } 537 | k++ 538 | } 539 | 540 | for _, f := range row { 541 | if k > 0 { 542 | fmt.Fprint(writer, ",") 543 | } 544 | 545 | if f.Code == "$id" { 546 | fmt.Fprintf(writer, "\"%d\"", record.Id()) 547 | } else if f.Code == "$revision" { 548 | fmt.Fprintf(writer, "\"%d\"", record.Revision()) 549 | } else if f.Type == kintone.FT_SUBTABLE { 550 | table := record.Fields[f.Code].(kintone.SubTableField) 551 | if j < len(table) { 552 | fmt.Fprintf(writer, "\"%d\"", table[j].Id()) 553 | } 554 | } else if f.IsSubField { 555 | table := record.Fields[f.Table].(kintone.SubTableField) 556 | if j < len(table) { 557 | subField := table[j].Fields[f.Code] 558 | if f.Type == kintone.FT_FILE { 559 | dir := fmt.Sprintf("%s-%d-%d", f.Code, rowID, j) 560 | err := downloadFile(app, subField, dir) 561 | if err != nil { 562 | return 0, err 563 | } 564 | } 565 | fmt.Fprint(writer, "\"") 566 | _, err := fmt.Fprint(writer, escapeCol(toString(subField, "\n"))) 567 | if err != nil { 568 | return 0, err 569 | } 570 | fmt.Fprint(writer, "\"") 571 | } 572 | } else { 573 | field := record.Fields[f.Code] 574 | if field != nil { 575 | if j == 0 && f.Type == kintone.FT_FILE { 576 | dir := fmt.Sprintf("%s-%d", f.Code, rowID) 577 | err := downloadFile(app, field, dir) 578 | if err != nil { 579 | return 0, err 580 | } 581 | } 582 | fmt.Fprint(writer, "\"") 583 | _, err := fmt.Fprint(writer, escapeCol(toString(field, "\n"))) 584 | if err != nil { 585 | return 0, err 586 | } 587 | fmt.Fprint(writer, "\"") 588 | } 589 | } 590 | k++ 591 | } 592 | fmt.Fprint(writer, "\r\n") 593 | } 594 | i++ 595 | 596 | } 597 | 598 | return i, nil 599 | } 600 | 601 | func writeRecordsBySeekMethodForCsv(app *kintone.App, id uint64, writer io.Writer, row Row, hasTable bool, index uint64, fields []string, isRecordFound bool, isAppendIdCustome bool) error { 602 | records, err := getRecordsForSeekMethod(app, id, fields, isRecordFound) 603 | if err != nil { 604 | return err 605 | } 606 | index, err = writeRecordsCsv(app, writer, records, row, hasTable, index, isAppendIdCustome) 607 | if err != nil { 608 | return err 609 | } 610 | if len(records) == EXPORT_ROW_LIMIT { 611 | isRecordFound = false 612 | return writeRecordsBySeekMethodForCsv(app, records[len(records)-1].Id(), writer, row, hasTable, index, fields, isRecordFound, isAppendIdCustome) 613 | } 614 | return nil 615 | } 616 | 617 | func writeRecordsBySeekMethodForJson(app *kintone.App, id uint64, writer io.Writer, index uint64, fields []string, isRecordsNotFound bool, isAppendIdCustome bool) error { 618 | records, err := getRecordsForSeekMethod(app, id, fields, isRecordsNotFound) 619 | if err != nil { 620 | return err 621 | } 622 | if index == 0 { 623 | fmt.Fprint(writer, "{\"records\": [\n") 624 | } 625 | index, err = writeRecordsJSON(app, writer, records, index, isAppendIdCustome) 626 | if err != nil { 627 | return err 628 | } 629 | if len(records) == EXPORT_ROW_LIMIT { 630 | isRecordsNotFound = false 631 | return writeRecordsBySeekMethodForJson(app, records[len(records)-1].Id(), writer, index, fields, isRecordsNotFound, isAppendIdCustome) 632 | } 633 | fmt.Fprint(writer, "\n]}") 634 | return nil 635 | } 636 | 637 | func toString(f interface{}, delimiter string) string { 638 | 639 | if delimiter == "" { 640 | delimiter = "," 641 | } 642 | switch f.(type) { 643 | case kintone.SingleLineTextField: 644 | singleLineTextField := f.(kintone.SingleLineTextField) 645 | return string(singleLineTextField) 646 | case kintone.MultiLineTextField: 647 | multiLineTextField := f.(kintone.MultiLineTextField) 648 | return string(multiLineTextField) 649 | case kintone.RichTextField: 650 | richTextField := f.(kintone.RichTextField) 651 | return string(richTextField) 652 | case kintone.DecimalField: 653 | decimalField := f.(kintone.DecimalField) 654 | return string(decimalField) 655 | case kintone.CalcField: 656 | calcField := f.(kintone.CalcField) 657 | return string(calcField) 658 | case kintone.RadioButtonField: 659 | radioButtonField := f.(kintone.RadioButtonField) 660 | return string(radioButtonField) 661 | case kintone.LinkField: 662 | linkField := f.(kintone.LinkField) 663 | return string(linkField) 664 | case kintone.StatusField: 665 | statusField := f.(kintone.StatusField) 666 | return string(statusField) 667 | case kintone.RecordNumberField: 668 | recordNumberField := f.(kintone.RecordNumberField) 669 | return string(recordNumberField) 670 | case kintone.CheckBoxField: 671 | checkBoxField := f.(kintone.CheckBoxField) 672 | return strings.Join(checkBoxField, delimiter) 673 | case kintone.MultiSelectField: 674 | multiSelectField := f.(kintone.MultiSelectField) 675 | return strings.Join(multiSelectField, delimiter) 676 | case kintone.CategoryField: 677 | categoryField := f.(kintone.CategoryField) 678 | return strings.Join(categoryField, delimiter) 679 | case kintone.SingleSelectField: 680 | singleSelect := f.(kintone.SingleSelectField) 681 | return singleSelect.String 682 | case kintone.FileField: 683 | fileField := f.(kintone.FileField) 684 | files := make([]string, 0, len(fileField)) 685 | for _, file := range fileField { 686 | files = append(files, file.Name) 687 | } 688 | return strings.Join(files, delimiter) 689 | case kintone.DateField: 690 | dateField := f.(kintone.DateField) 691 | if dateField.Valid { 692 | return dateField.Date.Format("2006-01-02") 693 | } 694 | return "" 695 | case kintone.TimeField: 696 | timeField := f.(kintone.TimeField) 697 | if timeField.Valid { 698 | return timeField.Time.Format("15:04:05") 699 | } 700 | return "" 701 | case kintone.DateTimeField: 702 | dateTimeField := f.(kintone.DateTimeField) 703 | if dateTimeField.Valid { 704 | return dateTimeField.Time.Format(time.RFC3339) 705 | } 706 | return "" 707 | case kintone.UserField: 708 | userField := f.(kintone.UserField) 709 | users := make([]string, 0, len(userField)) 710 | for _, user := range userField { 711 | users = append(users, user.Code) 712 | } 713 | return strings.Join(users, delimiter) 714 | case kintone.OrganizationField: 715 | organizationField := f.(kintone.OrganizationField) 716 | organizations := make([]string, 0, len(organizationField)) 717 | for _, organization := range organizationField { 718 | organizations = append(organizations, organization.Code) 719 | } 720 | return strings.Join(organizations, delimiter) 721 | case kintone.GroupField: 722 | groupField := f.(kintone.GroupField) 723 | groups := make([]string, 0, len(groupField)) 724 | for _, group := range groupField { 725 | groups = append(groups, group.Code) 726 | } 727 | return strings.Join(groups, delimiter) 728 | case kintone.AssigneeField: 729 | assigneeField := f.(kintone.AssigneeField) 730 | users := make([]string, 0, len(assigneeField)) 731 | for _, user := range assigneeField { 732 | users = append(users, user.Code) 733 | } 734 | return strings.Join(users, delimiter) 735 | case kintone.CreatorField: 736 | creatorField := f.(kintone.CreatorField) 737 | return creatorField.Code 738 | case kintone.ModifierField: 739 | modifierField := f.(kintone.ModifierField) 740 | return modifierField.Code 741 | case kintone.CreationTimeField: 742 | creationTimeField := f.(kintone.CreationTimeField) 743 | return time.Time(creationTimeField).Format(time.RFC3339) 744 | case kintone.ModificationTimeField: 745 | modificationTimeField := f.(kintone.ModificationTimeField) 746 | return time.Time(modificationTimeField).Format(time.RFC3339) 747 | case kintone.SubTableField: 748 | return "" // unsupported 749 | } 750 | return "" 751 | } 752 | --------------------------------------------------------------------------------