├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── dependabot.yml └── workflows │ └── linters.yml ├── .gitignore ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── README.md ├── app.go ├── config.go ├── embed ├── config.yml └── data.csv ├── filtered.go ├── go.mod ├── go.sum ├── helpers.go ├── inputs.go ├── main.go ├── models.go ├── outputs.go ├── save_filtered.go ├── update_fields.go └── wire.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug, help wanted 6 | assignees: koddr 7 | --- 8 | 9 | 10 | 11 | **Required check list:** 12 | 13 | - [ ] I didn't find in the repository's issues section a similar bug. 14 | - [ ] I understand, this is an Open Source and not-for-profit product. 15 | - [ ] This is not about a third-party project, framework, or technology. 16 | 17 | **My environment:** 18 | 19 | - OS (`uname -a`): 20 | - Go (`go version`): 21 | 22 | **Describe the bug:** 23 | _A clear and concise description of what the bug is._ 24 | 25 | ... 26 | 27 | **Steps to reproduce the behavior:** 28 | 29 | 1. ... 30 | 2. ... 31 | 3. ... 32 | 33 | **Expected behavior:** 34 | _A clear and concise description of what you expected to happen._ 35 | 36 | ... 37 | 38 | **Screenshots:** 39 | _If applicable, add screenshots to help explain your problem._ 40 | 41 | ... 42 | 43 | **Additional context:** 44 | _Any other context about the issue here._ 45 | 46 | ... 47 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: GitHub Community Support 4 | url: https://github.com/orgs/community/discussions 5 | about: Please ask and answer questions here. 6 | - name: GitHub Security Bug Bounty 7 | url: https://bounty.github.com/ 8 | about: Please report security vulnerabilities here. 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: koddr 7 | --- 8 | 9 | 10 | 11 | **Required check list:** 12 | 13 | - [ ] I didn't find in the repository's issues section a similar bug. 14 | - [ ] I understand, this is an Open Source and not-for-profit product. 15 | - [ ] This is not about a third-party project, framework, or technology. 16 | 17 | **Is your feature request related to a problem? Please describe.** 18 | _A clear and concise description of what the issue is._ 19 | 20 | ... 21 | 22 | **Describe the solution you'd like:** 23 | _A clear and concise description of what you want to happen._ 24 | 25 | ... 26 | 27 | **Describe alternatives you've considered:** 28 | _A clear and concise description of any alternative solutions._ 29 | 30 | ... 31 | 32 | **Screenshots:** 33 | _If applicable, add screenshots to help explain your feature._ 34 | 35 | ... 36 | 37 | **Additional context:** 38 | _Any other context or screenshots about the feature request here._ 39 | 40 | ... 41 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: gomod 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | time: "06:00" 9 | open-pull-requests-limit: 10 10 | -------------------------------------------------------------------------------- /.github/workflows/linters.yml: -------------------------------------------------------------------------------- 1 | name: Linter & Scanner 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "**.go" 7 | 8 | jobs: 9 | build: 10 | strategy: 11 | matrix: 12 | go: [ 'stable' ] 13 | os: [ 'ubuntu-latest' ] 14 | runs-on: ${{ matrix.os }} 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-go@v4 18 | with: 19 | go-version: ${{ matrix.go }} 20 | - name: Run "google/wire" for DI 21 | run: go run github.com/google/wire/cmd/wire@latest 22 | - name: Run "govulncheck" scanner 23 | run: go run golang.org/x/vuln/cmd/govulncheck@latest -test ./... 24 | - name: Run "securego/gosec" scanner 25 | run: go run github.com/securego/gosec/v2/cmd/gosec@latest -quiet ./... 26 | - name: Run "go-critic/go-critic" linter 27 | run: go run github.com/go-critic/go-critic/cmd/gocritic@latest check -enableAll ./... 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test folder and binary, built with `go test -c` 9 | tests/ 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Go workspace file 16 | go.work 17 | 18 | # macOS files 19 | .DS_Store 20 | 21 | # IDE config directories (remove the comment below to include it) 22 | .idea/ 23 | .vscode/ 24 | 25 | # Dependency directories (remove the comment below to include it) 26 | vendor/ 27 | 28 | # Binary directories 29 | bin/ 30 | 31 | # Goreleaser directories 32 | dist/ 33 | 34 | # Generate files 35 | *_gen.go 36 | filtered-*.csv 37 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: csv2api 2 | report_sizes: true 3 | 4 | env_files: 5 | github_token: ~/.github_token 6 | 7 | before: 8 | hooks: 9 | - go mod tidy 10 | - go mod download 11 | - go run github.com/google/wire/cmd/wire@latest 12 | - go run golang.org/x/vuln/cmd/govulncheck@latest -test ./... 13 | - go run github.com/securego/gosec/v2/cmd/gosec@latest -quiet ./... 14 | - go run github.com/go-critic/go-critic/cmd/gocritic@latest check -enableAll ./... 15 | 16 | builds: 17 | - 18 | id: default 19 | env: [ CGO_ENABLED=0 ] 20 | goos: [ linux, windows ] 21 | goarch: [ amd64, arm64 ] 22 | # skip: true # useful for library projects 23 | 24 | - # HACK for macOS Ventura (13.x), which not supported UPX 25 | id: macOS_only 26 | env: [ CGO_ENABLED=0 ] 27 | goos: [ darwin ] 28 | goarch: [ amd64, arm64 ] 29 | 30 | upx: 31 | - 32 | ids: [ default ] 33 | enabled: true 34 | compress: best 35 | lzma: true 36 | brute: true 37 | # goos: [ darwin ] # wait for v1.19 38 | 39 | release: 40 | ids: [ default, macOS_only ] 41 | draft: true 42 | replace_existing_draft: true 43 | target_commitish: "{{ .Commit }}" 44 | # discussion_category_name: General 45 | prerelease: auto 46 | mode: replace 47 | header: | 48 | ## ⚙️ The `{{ .Tag }}` release 49 | footer: | 50 | ## Install or update 51 | 52 | For native Go installation (any platforms): 53 | 54 | ```console 55 | go install github.com/koddr/csv2api@latest 56 | ``` 57 | 58 | For [Homebrew][brew_url] users (GNU/Linux, macOS): 59 | 60 | ```console 61 | brew upgrade koddr/tap/csv2api 62 | ``` 63 | 64 | > 💡 Note: See the [`Wiki page`][wiki_url] to understand structures of JSON files and get general recommendations. 65 | 66 | ## Your help to improve project 67 | 68 | I'd be truly grateful for help with: 69 | 70 | - Creating tests (and/or benchmarks) for code 71 | - Improving existing functions, structs, or tests 72 | - Feature requests with interesting functions that would be good to add 73 | 74 | Your PRs & issues are welcome! Thanks 😉 75 | 76 | [brew_url]: https://brew.sh 77 | [wiki_url]: https://github.com/koddr/csv2api/wiki 78 | disable: false 79 | skip_upload: false 80 | 81 | brews: 82 | - 83 | tap: 84 | owner: koddr 85 | name: homebrew-tap 86 | branch: main 87 | token: "{{ .Env.GITHUB_TOKEN }}" 88 | pull_request: 89 | enabled: true 90 | # draft: true # wait for v1.19 91 | git: 92 | url: "git@github.com:koddr/homebrew-tap.git" 93 | private_key: "{{ .Env.PRIVATE_KEY_PATH }}" 94 | commit_author: 95 | name: Vic Shóstak 96 | email: koddr.me@gmail.com 97 | commit_msg_template: "Brew formula update for {{ .ProjectName }} version {{ .Tag }}" 98 | folder: Formula 99 | caveats: | 100 | After install csv2api, run it with '-i' option to generate initial config.yml and data.csv files in the current dir: 101 | 102 | $ csv2api -i 103 | 104 | Prepare your config and input data files: 105 | 106 | In your config.yml: 107 | 108 | - Make sure that the first column name in the columns_order section is a primary key (PK) for your process. 109 | - Set up your API (base URL, token, endpoints, etc) in the api section. 110 | - Set up the filter for your fields in the filter_columns section. 111 | - Set up fields to be updated in the update_fields section. 112 | 113 | In your input data.csv: 114 | 115 | - Make sure that the first line of your CSV file contains the correct field names. 116 | 117 | 💡 Note: See the repository's Wiki page (https://github.com/koddr/csv2api/wiki) to understand the structure of the config and input data files. 118 | 119 | And now, run csv2api with options: 120 | 121 | $ csv2api -c /path/to/config.yml -d /path/to/data.csv -e CONFIG 122 | 123 | Done! Your transactions have been performed. 124 | homepage: "https://github.com/koddr/{{ .ProjectName }}" 125 | description: | 126 | The csv2api parser reads the CSV file with the raw data, filters the records, identifies fields to be changed, and sends a request to update the data to the specified endpoint of your REST API. 127 | All actions take place according to the settings in the configuration file. 128 | license: Apache 2.0 129 | skip_upload: false 130 | # dependencies: 131 | # - name: git 132 | 133 | dockers: 134 | - 135 | id: "{{ .ProjectName }}" 136 | ids: [ default ] 137 | image_templates: 138 | - "koddr/{{ .ProjectName }}:latest" 139 | - "koddr/{{ .ProjectName }}:{{ .Tag }}" 140 | - "koddr/{{ .ProjectName }}:v{{ .Major }}" 141 | - "koddr/{{ .ProjectName }}:v{{ .Major }}.{{ .Minor }}" 142 | build_flag_templates: 143 | - "--pull" 144 | - "--label=org.opencontainers.image.created={{ .Date }}" 145 | - "--label=org.opencontainers.image.title={{ .ProjectName }}" 146 | - "--label=org.opencontainers.image.revision={{ .FullCommit }}" 147 | - "--label=org.opencontainers.image.version={{ .Version }}" 148 | - "--platform=linux/amd64" 149 | skip_push: false 150 | push_flags: 151 | - --tls-verify=false 152 | 153 | nfpms: 154 | - 155 | maintainer: Vic Shóstak 156 | description: | 157 | The csv2api parser reads the CSV file with the raw data, filters the records, identifies fields to be changed, and sends a request to update the data to the specified endpoint of your REST API. 158 | All actions take place according to the settings in the configuration file. 159 | homepage: "https://github.com/koddr/{{ .ProjectName }}" 160 | license: Apache 2.0 161 | formats: [ deb, rpm, apk, archlinux ] 162 | # dependencies: [ git ] 163 | 164 | archives: 165 | - 166 | format_overrides: 167 | - goos: windows 168 | format: zip 169 | files: [ LICENSE, README.md ] 170 | rlcp: true 171 | 172 | checksum: 173 | name_template: "checksums.txt" 174 | 175 | changelog: 176 | sort: asc 177 | abbrev: -1 178 | filters: 179 | exclude: [ "^*.md:", "^*.yml:" ] 180 | groups: 181 | - title: Features 182 | regexp: ^.*?(F|f)eature.*?$ 183 | order: 0 184 | - title: Bug fixes 185 | regexp: ^.*?((B|b)ug)|((F|f)ix).*?$ 186 | order: 1 187 | - title: Improvements 188 | regexp: ^.*?(I|i)mprove.*?$ 189 | order: 2 190 | - title: Updates 191 | regexp: ^.*?(U|u)pdate.*?$ 192 | order: 3 193 | - title: Security issues 194 | regexp: ^.*?(S|s)ecurity.*?$ 195 | order: 4 196 | - title: Others 197 | order: 999 198 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | ENTRYPOINT ["/csv2api"] 3 | COPY csv2api / 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 Vic Shóstak 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # csv2api – Parse CSV with filtering and sending to API 2 | 3 | [![Go version][go_version_img]][go_dev_url] 4 | [![Go report][go_report_img]][go_report_url] 5 | [![Wiki][wiki_img]][wiki_url] 6 | [![License][license_img]][license_url] 7 | 8 | The **csv2api** parser reads the CSV file with the raw data, filters the 9 | records, identifies fields to be changed, and sends a request to update the 10 | data to the specified endpoint of your REST API. 11 | 12 | All actions take place according to the settings in the configuration file. 13 | 14 | Features: 15 | 16 | - 100% **free** and **open source**. 17 | - Works with any **size** of input CSV file. 18 | - Use **any** configuration file format: JSON, YAML, TOML, or HCL (Terraform). 19 | - Ability to keep a configuration file (_and, in the future, input data 20 | file_) on a **remote server** with HTTP access, it will be read as if it 21 | was in a folder on your local machine. 22 | - Configure **any** request body for the REST API endpoint directly in the 23 | configuration file (in a clear declarative style). 24 | - Extensive options for **filtering** incoming data from a CSV file. 25 | - Provides extensive capabilities for constructing **multiple filters** to 26 | accurately perform actions on selected fields. 27 | 28 | ## ⚡️ Quick start 29 | 30 | First, [download][go_download] and install **Go**. Version `1.20` (or higher) 31 | is required. 32 | 33 | Installation is done by using the [`go install`][go_install] command: 34 | 35 | ```console 36 | go install github.com/koddr/csv2api@latest 37 | ``` 38 | 39 | > 💡 Note: See the repository's [Release page][repo_releases_url], if you want 40 | > to download a ready-made `deb`, `rpm`, `apk` or `Arch Linux` package. 41 | 42 | GNU/Linux and macOS users available way to install via [Homebrew][brew_url]: 43 | 44 | ```console 45 | # Tap a new formula: 46 | brew tap koddr/tap 47 | 48 | # Installation: 49 | brew install koddr/tap/csv2api 50 | ``` 51 | 52 | Next, run `csv2api` with `-i` option to generate initial `config.yml` and 53 | `data.csv` files in the current dir: 54 | 55 | ```console 56 | csv2api -i 57 | ``` 58 | 59 | Prepare your config and input data files: 60 | 61 | - In your `config.yml`: 62 | - Make sure that the first column name in the `columns_order` section is a 63 | primary key (PK) for your process. 64 | - Set up your API (base URL, token, endpoints, etc) in the `api` section. 65 | - Set up the filter for your fields in the `filter_columns` section. 66 | - Set up fields to be updated in the `update_fields` section. 67 | - In your input `data.csv`: 68 | - Make sure that the first line of your CSV file contains the correct field names. 69 | 70 | > 💡 Note: See the repository's [Wiki][wiki_url] page to understand the 71 | > structure of the config and input data files. 72 | 73 | And now, run `csv2api` with options: 74 | 75 | ```console 76 | csv2api \ 77 | -c /path/to/config.yml \ 78 | -d /path/to/data.csv \ 79 | -e CONFIG 80 | ``` 81 | 82 | Done! 🎉 Your transactions have been performed: 83 | 84 | ``` console 85 | Hello and welcome to csv2api! 👋 86 | 87 | – According to the settings in './config.yml', 5 transactions were filtered out of 10 to start the process. 88 | – Only 3 transactions got into the final set of actions to be taken... Please wait! 89 | 90 | ✓ Field 'tags' with values '[paid]' in the transaction '2' has been successfully updated (HTTP 200) 91 | ✓ Field 'tags' with values '[paid]' in the transaction '8' has been successfully updated (HTTP 200) 92 | ✓ Field 'tags' with values '[paid]' in the transaction '10' has been successfully updated (HTTP 200) 93 | 94 | – Saving filtered transactions to CSV file './filtered-1686993960.csv' in the current dir... OK! 95 | 96 | All done! 🎉 Time elapsed: 0.11s 97 | ``` 98 | 99 | ### 🐳 Docker-way to quick start 100 | 101 | If you don't want to physically install `csv2api` to your system, you feel 102 | free to using our [official Docker image][docker_image_url] and run it from 103 | isolated container: 104 | 105 | ```console 106 | docker run --rm -it -v ${PWD}:${PWD} -w ${PWD} koddr/csv2api:latest [OPTIONS] 107 | ``` 108 | 109 | ## 🧩 Options 110 | 111 | | Option | Description | Is required? | Default value | 112 | |--------|------------------------------------------------------------------------------------|--------------|---------------| 113 | | `-i` | set to generate initial config (`config.yaml`) and example data (`data.csv`) files | no | `false` | 114 | | `-c` | set path to your config file | yes | `""` | 115 | | `-d` | set path to your CSV file with input data | yes | `""` | 116 | | `-e` | set prefix used in your environment variables | no | `CONFIG` | 117 | 118 | ## ✨ Solving case 119 | 120 | In my work, I often have to work with **large amounts** of raw data in CSV format. 121 | 122 | Usually it goes like this: 123 | 124 | 1. Unload a file with data from one system. 125 | 2. Clean up this file from duplicates and unnecessary columns. 126 | 3. Make some changes in some columns of some rows. 127 | 4. Mapping the processed lines from CSV file to the database structure fields. 128 | 5. Write a function to bypass the CSV file and form the request body. 129 | 6. Write an HTTP client that will send requests to the REST API endpoint. 130 | 7. Send prepared request body to the REST API endpoint in other system 131 | for specified DB records. 132 | 133 | > And I'm not talking about the fact that the final REST API (where to send a 134 | request with the processed data) **do not** always have the same parameters for 135 | the request body. 136 | 137 | To ease this whole process, I created this parser that takes absolutely any 138 | data file as input, does the conversions and filtering, and is set up in one 139 | single configuration file. 140 | 141 | Just prepare the data, set the configuration to your liking, run `csv2api` 142 | and wait a bit! Yes, it's that simple. 143 | 144 | ## 🏆 A win-win cooperation 145 | 146 | And now, I invite you to participate in this project! Let's work **together** to 147 | create the **most useful** tool for developers on the web today. 148 | 149 | - [Issues][repo_issues_url]: ask questions and submit your features. 150 | - [Pull requests][repo_pull_request_url]: send your improvements to the current. 151 | 152 | Your PRs & issues are welcome! Thank you 😘 153 | 154 | ## ⚠️ License 155 | 156 | [`csv2api`][repo_url] is free and open-source software licensed 157 | under the [Apache 2.0 License][license_url], created and supported with 🩵 158 | for people and robots by [Vic Shóstak][author]. 159 | 160 | [go_download]: https://golang.org/dl/ 161 | [go_install]: https://golang.org/cmd/go/#hdr-Compile_and_install_packages_and_dependencies 162 | [go_version_img]: https://img.shields.io/badge/Go-1.20+-00ADD8?style=for-the-badge&logo=go 163 | [go_report_img]: https://img.shields.io/badge/Go_report-A+-success?style=for-the-badge&logo=none 164 | [go_report_url]: https://goreportcard.com/report/github.com/koddr/csv2api 165 | [go_code_coverage_img]: https://img.shields.io/badge/code_coverage-0%25-success?style=for-the-badge&logo=none 166 | [go_dev_url]: https://pkg.go.dev/github.com/koddr/csv2api 167 | [docker_image_url]: https://hub.docker.com/repository/docker/koddr/csv2api 168 | [brew_url]: https://brew.sh 169 | [wiki_img]: https://img.shields.io/badge/docs-wiki_page-blue?style=for-the-badge&logo=none 170 | [wiki_url]: https://github.com/koddr/csv2api/wiki 171 | [license_img]: https://img.shields.io/badge/license-Apache_2.0-red?style=for-the-badge&logo=none 172 | [license_url]: https://github.com/koddr/csv2api/blob/main/LICENSE 173 | [repo_url]: https://github.com/koddr/csv2api 174 | [repo_releases_url]: https://github.com/koddr/csv2api/releases 175 | [repo_issues_url]: https://github.com/koddr/csv2api/issues 176 | [repo_pull_request_url]: https://github.com/koddr/csv2api/pulls 177 | [author]: https://github.com/koddr 178 | -------------------------------------------------------------------------------- /app.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // newApp provides a new application instance. 9 | func newApp(config *Config, inputs *Inputs, filtered *Filtered, outputs *Outputs) *App { 10 | return &App{ 11 | Config: config, 12 | Inputs: inputs, 13 | Filtered: filtered, 14 | Outputs: outputs, 15 | } 16 | } 17 | 18 | // start starts an application with a beauty output to console. 19 | func (app *App) start() error { 20 | // Start timer. 21 | start := time.Now() 22 | 23 | // Check, if input data is not empty. 24 | if app.Inputs.Data != nil { 25 | if len(app.Inputs.Data) > 0 { 26 | printStyled( 27 | "Hello and welcome to csv2api! 👋", 28 | "margin-top-bottom", 29 | ) 30 | printStyled( 31 | fmt.Sprintf( 32 | "– According to the settings in '%s', %d transactions were filtered out of %d to start the process.", 33 | configFilePath, len(app.Filtered.Data), len(app.Inputs.Data), 34 | ), 35 | "", 36 | ) 37 | 38 | // Check, if output data is not empty. 39 | if len(app.Outputs.Data) > 0 { 40 | printStyled( 41 | fmt.Sprintf( 42 | "– Only %d transactions got into the final set of actions to be taken... Please wait!", 43 | len(app.Outputs.Data), 44 | ), 45 | "margin-bottom", 46 | ) 47 | 48 | // Loop for all output data. 49 | for _, data := range app.Outputs.Data { 50 | // Start updating fields. 51 | if err := app.updateField(data.ID, data.FieldName, data.Values); err != nil { 52 | printStyled( 53 | fmt.Sprintf("✕ There was an error with collect output data: %v", err), 54 | "margin-left", 55 | ) 56 | } 57 | } 58 | 59 | // Check, if you need to save CSV file with filtered PK. 60 | if app.Config.SaveFilteredPKToCSV { 61 | // Save filtered PK to CSV. 62 | if err := app.saveFilteredPKToCSV(); err != nil { 63 | printStyled( 64 | fmt.Sprintf("✕ There was an error with save filtered PK to CSV: %v", err), 65 | "margin-top", 66 | ) 67 | } 68 | } 69 | } 70 | } 71 | 72 | printStyled( 73 | fmt.Sprintf( 74 | "All done! 🎉 Time elapsed: %.2fs", 75 | time.Since(start).Seconds(), 76 | ), 77 | "margin-top-bottom", 78 | ) 79 | } 80 | 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/koddr/gosl" 7 | ) 8 | 9 | // newConfig provides a new config instance. 10 | func newConfig() (*Config, error) { 11 | // Check, if the file name is too short. 12 | if configFilePath == "" { 13 | return nil, fmt.Errorf("invalid format of config file, see: %s", WikiPageURL) 14 | } 15 | 16 | // Check, if the input data file is not empty. 17 | if inputDataFilePath == "" { 18 | return nil, fmt.Errorf("path to the input data file is empty, see: %s", WikiPageURL) 19 | } 20 | 21 | // Create a new config instance. 22 | config := &Config{} 23 | 24 | // Load config from path or HTTP by the given file format. 25 | _, err := gosl.ParseFileWithEnvToStruct(configFilePath, envPrefix, config) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | return config, nil 31 | } 32 | -------------------------------------------------------------------------------- /embed/config.yml: -------------------------------------------------------------------------------- 1 | # save_filtered_pk_to_csv allows to save all filtered data 2 | # to the CSV file (for report or else) 3 | save_filtered_pk_to_csv: true 4 | 5 | # columns_order allows to set order of the columns in the input data file 6 | # 7 | # WARNING: this order will be saved in all scripts when processing 8 | # the CSV file! The names of the columns must match the header 9 | # line of the file completely. 10 | columns_order: 11 | - id # first column is the primary key (PK) 12 | - order_id 13 | - is_paid 14 | 15 | # csv_column_separator allows to set a column separator 16 | # for the input data file (CSV) 17 | csv_column_separator: ',' 18 | 19 | # api allows to set params for API requests 20 | api: 21 | 22 | # base_url allows to set a base URL for API 23 | base_url: my-api-server.com 24 | 25 | # base_url_schema allows to set an HTTP schema for API 26 | base_url_schema: https 27 | 28 | # auth_method allows to set an authorization method to make API request 29 | auth_method: Bearer 30 | 31 | # token allows to set an authorization token to make API requests 32 | # 33 | # WARNING: do not forget to add this file to .gitignore, if you place 34 | # token to this file! Please use environment variables for security. 35 | token: '{{ CONFIG_API_TOKEN }}' 36 | 37 | # request_timeout allows to set a timeout in seconds 38 | # after each request to API (prevent throttling) 39 | request_timeout: 0 40 | 41 | # update_endpoint allows to set endpoint for update fields 42 | update_endpoint: 43 | endpoint_name: /api/v1/order/%s 44 | content_type: application/json 45 | add_pk_to_endpoint_name: true 46 | http_method: PATCH 47 | endpoint_body: 48 | data: 49 | id: id 50 | attributes: 51 | tags: tags 52 | 53 | # filter_columns allows to set columns, that will be filtered 54 | filter_columns: 55 | 56 | - column_name: order_id 57 | condition: NEQ 58 | value: '' 59 | 60 | # update_fields allows to set fields and its values, that will be updated, 61 | # if conditions are successes 62 | update_fields: 63 | 64 | - field_name: tags 65 | values: 66 | - paid 67 | conditions: 68 | - column_name: is_paid 69 | condition: EQ 70 | value: '1' 71 | -------------------------------------------------------------------------------- /embed/data.csv: -------------------------------------------------------------------------------- 1 | id,order_id,is_paid 2 | 1,000789,0 3 | 2,000123,1 4 | 3,,0 5 | 4,,0 6 | 5,000321,0 7 | 6,,0 8 | 7,,0 9 | 8,000322,1 10 | 9,,0 11 | 10,000654,1 12 | -------------------------------------------------------------------------------- /filtered.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/koddr/gosl" 8 | ) 9 | 10 | // newFiltered provides a new filter instance. 11 | func newFiltered(config *Config, inputs *Inputs) (*Filtered, error) { 12 | // Check, if the input data is not nil or empty. 13 | if inputs.Data == nil || len(inputs.Data) == 0 { 14 | return nil, errors.New("input data is empty or not parsed") 15 | } 16 | 17 | // Create a new Filtered instance. 18 | filtered := &Filtered{} 19 | 20 | // Check, if the filter has columns. 21 | if config.FilterColumns != nil && len(config.FilterColumns) > 0 { 22 | // Loop for all inputs data. 23 | for _, data := range inputs.Data { 24 | // Create a temp slice. 25 | m := make([]bool, 0) 26 | 27 | // Loop for all filtered columns. 28 | for index, column := range config.FilterColumns { 29 | // Matching values with condition between input (in data file) and column (in config file). 30 | match, err := matchValues(data[inputs.Mapping[column.ColumnName]], column.Value, column.Condition) 31 | if err != nil { 32 | return nil, fmt.Errorf( 33 | "%w, column %d ('%v') in row %v", 34 | err, index, column.ColumnName, data, 35 | ) 36 | } 37 | 38 | // Collect matching result to the temp slice. 39 | m = append(m, match) 40 | } 41 | 42 | // Check, if there is no false in the conditions, it means 43 | // that the filter settings have been respected. 44 | if !gosl.ContainsInSlice(m, false) { 45 | // Collect the current data to the filtered result. 46 | filtered.Data = append(filtered.Data, data) 47 | } 48 | } 49 | } 50 | 51 | return filtered, nil 52 | } 53 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/koddr/csv2api 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/charmbracelet/lipgloss v1.1.0 7 | github.com/google/wire v0.6.0 8 | github.com/koddr/gosl v1.6.0 9 | ) 10 | 11 | require ( 12 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 13 | github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect 14 | github.com/charmbracelet/x/ansi v0.8.0 // indirect 15 | github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect 16 | github.com/charmbracelet/x/term v0.2.1 // indirect 17 | github.com/fsnotify/fsnotify v1.6.0 // indirect 18 | github.com/hashicorp/hcl v1.0.0 // indirect 19 | github.com/json-iterator/go v1.1.12 // indirect 20 | github.com/knadh/koanf/maps v0.1.1 // indirect 21 | github.com/knadh/koanf/parsers/hcl v0.1.0 // indirect 22 | github.com/knadh/koanf/parsers/json v0.1.0 // indirect 23 | github.com/knadh/koanf/parsers/toml v0.1.0 // indirect 24 | github.com/knadh/koanf/parsers/yaml v0.1.0 // indirect 25 | github.com/knadh/koanf/providers/env v0.1.0 // indirect 26 | github.com/knadh/koanf/providers/file v0.1.0 // indirect 27 | github.com/knadh/koanf/providers/rawbytes v0.1.0 // indirect 28 | github.com/knadh/koanf/v2 v2.0.1 // indirect 29 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 30 | github.com/mattn/go-isatty v0.0.20 // indirect 31 | github.com/mattn/go-runewidth v0.0.16 // indirect 32 | github.com/mitchellh/copystructure v1.2.0 // indirect 33 | github.com/mitchellh/mapstructure v1.5.0 // indirect 34 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 35 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 36 | github.com/modern-go/reflect2 v1.0.2 // indirect 37 | github.com/muesli/termenv v0.16.0 // indirect 38 | github.com/pelletier/go-toml v1.9.5 // indirect 39 | github.com/rivo/uniseg v0.4.7 // indirect 40 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect 41 | golang.org/x/sys v0.30.0 // indirect 42 | gopkg.in/yaml.v3 v3.0.1 // indirect 43 | ) 44 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 2 | github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 3 | github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= 4 | github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= 5 | github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= 6 | github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= 7 | github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= 8 | github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= 9 | github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= 10 | github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= 11 | github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= 12 | github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= 13 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 15 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= 17 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 18 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 19 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 20 | github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= 21 | github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= 22 | github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= 23 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 24 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 25 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 26 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 27 | github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= 28 | github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= 29 | github.com/knadh/koanf/parsers/hcl v0.1.0 h1:PuAAdRMXbxmhwzZftiQBEtWIKc3EbRHk/Fi+olo02z4= 30 | github.com/knadh/koanf/parsers/hcl v0.1.0/go.mod h1:7ClRvH1oP5ne8SfaDZZBK28/o9r4rek0PC4Vrc8qdvE= 31 | github.com/knadh/koanf/parsers/json v0.1.0 h1:dzSZl5pf5bBcW0Acnu20Djleto19T0CfHcvZ14NJ6fU= 32 | github.com/knadh/koanf/parsers/json v0.1.0/go.mod h1:ll2/MlXcZ2BfXD6YJcjVFzhG9P0TdJ207aIBKQhV2hY= 33 | github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6ODgOub/6LCI= 34 | github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18= 35 | github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= 36 | github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY= 37 | github.com/knadh/koanf/providers/env v0.1.0 h1:LqKteXqfOWyx5Ab9VfGHmjY9BvRXi+clwyZozgVRiKg= 38 | github.com/knadh/koanf/providers/env v0.1.0/go.mod h1:RE8K9GbACJkeEnkl8L/Qcj8p4ZyPXZIQ191HJi44ZaQ= 39 | github.com/knadh/koanf/providers/file v0.1.0 h1:fs6U7nrV58d3CFAFh8VTde8TM262ObYf3ODrc//Lp+c= 40 | github.com/knadh/koanf/providers/file v0.1.0/go.mod h1:rjJ/nHQl64iYCtAW2QQnF0eSmDEX/YZ/eNFj5yR6BvA= 41 | github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI51k5pe8RYKQ0qME= 42 | github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c= 43 | github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= 44 | github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= 45 | github.com/koddr/gosl v1.6.0 h1:v2uBwrJ1WEAoyCiCZ2e7Eqq9IvxLrA9/I0cdwTp5hRY= 46 | github.com/koddr/gosl v1.6.0/go.mod h1:nuoJ5M/GiQrml4K75zkGEsWLHYnBt3ta/0/1Unw7rEM= 47 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 48 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 49 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 50 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 51 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 52 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 53 | github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= 54 | github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= 55 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 56 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 57 | github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= 58 | github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 59 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 60 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 61 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 62 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 63 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 64 | github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= 65 | github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= 66 | github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= 67 | github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= 68 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 69 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 70 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 71 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 72 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 73 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 74 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 75 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 76 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= 77 | github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= 78 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 79 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 80 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 81 | golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= 82 | golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= 83 | golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= 84 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 85 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 86 | golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 87 | golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 88 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 89 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 90 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 91 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 92 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 93 | golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= 94 | golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= 95 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 96 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 97 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 98 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 99 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 100 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 101 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 102 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 103 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 104 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 105 | golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 106 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 107 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 108 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 109 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 110 | golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 111 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 112 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 113 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 114 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 115 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 116 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 117 | golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= 118 | golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= 119 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 120 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 121 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 122 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 123 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 124 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 125 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 126 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 127 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 128 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 129 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 130 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 131 | golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= 132 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 133 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 134 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 135 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 136 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 137 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "embed" 5 | "fmt" 6 | "strconv" 7 | 8 | "github.com/charmbracelet/lipgloss" 9 | "github.com/koddr/gosl" 10 | ) 11 | 12 | const ( 13 | // WikiPageURL link to project's Wiki page. 14 | WikiPageURL string = "https://github.com/koddr/csv2api/wiki" 15 | ) 16 | 17 | var ( 18 | //go:embed embed/config.yml 19 | embedConfigYAMLFile []byte 20 | 21 | //go:embed embed/data.csv 22 | embedDataCSVFile []byte 23 | 24 | // Flag for generate init config. 25 | initConfig bool 26 | 27 | // Flags for config, data file path, env prefix. 28 | configFilePath, inputDataFilePath, envPrefix string 29 | ) 30 | 31 | // removeDuplicates provides process to remove all duplicates in the given slice 32 | // of slices, and return cleared slice ot slices. 33 | func removeDuplicates(source [][]string) [][]string { 34 | // Create a temp map and slice. 35 | seen := make(map[string]bool, 0) 36 | result := make([][]string, 0) 37 | 38 | // Run loop for all slices in the given source. 39 | for _, item := range source { 40 | // Check, if current key is not in source slice. 41 | if _, ok := seen[fmt.Sprint(item)]; !ok { 42 | // Collect element into temp slice. 43 | result = append(result, item) 44 | 45 | // Set marker. 46 | seen[fmt.Sprint(item)] = true 47 | } 48 | } 49 | 50 | return result 51 | } 52 | 53 | // matchIndexes provides process to matching indexes for mapping columns. 54 | func matchIndexes(wanted, source []string) map[string]int { 55 | // Create a temp map. 56 | indexes := make(map[string]int, 0) 57 | 58 | // Loop for all strings in the given wanted slice. 59 | for _, columnName := range wanted { 60 | // Loop for all strings in the given source slice. 61 | for sourceIndex, sourceName := range source { 62 | // Check, if names of the source and wanted slices were match. 63 | if sourceName == columnName { 64 | // Collect index of the source slice to the temp map. 65 | indexes[columnName] = sourceIndex 66 | break 67 | } 68 | } 69 | } 70 | 71 | return indexes 72 | } 73 | 74 | // matchValues provides process to matching two values with a condition. 75 | func matchValues(value1, value2, condition string) (bool, error) { 76 | // Check condition. 77 | switch condition { 78 | case "EQ": 79 | // Equals to the given value (==). 80 | return value1 == value2, nil 81 | case "NEQ": 82 | // Not equal to the given value (!=). 83 | return value1 != value2, nil 84 | case "GT", "LT", "GTE", "LTE": 85 | // Convert value1 from string to int, or error. 86 | val1, err := strconv.Atoi(value1) 87 | if err != nil { 88 | return false, fmt.Errorf( 89 | "wrong value '%s' to convert input value to type 'int' for this condition '%s'", 90 | value1, condition, 91 | ) 92 | } 93 | 94 | // Convert value2 from string to int, or error. 95 | val2, err := strconv.Atoi(value2) 96 | if err != nil { 97 | return false, fmt.Errorf( 98 | "wrong value '%s' to convert filter value to type 'int' for this condition '%s'", 99 | value2, condition, 100 | ) 101 | } 102 | 103 | // Check condition. 104 | switch condition { 105 | case "GT": 106 | // Greater than the given value (>). 107 | return val1 > val2, nil 108 | case "LT": 109 | // Less than the given value (<). 110 | return val1 < val2, nil 111 | case "GTE": 112 | // Greater than or equal to the given value (>=). 113 | return val1 >= val2, nil 114 | case "LTE": 115 | // Less than or equal to the given value (<=). 116 | return val1 <= val2, nil 117 | } 118 | } 119 | 120 | return false, fmt.Errorf("unknown condition: %s", condition) 121 | } 122 | 123 | // printStyled provides a beauty output for console. 124 | func printStyled(s, style string) { 125 | // Create a new blank style or the lipgloss. 126 | lp := lipgloss.NewStyle() 127 | 128 | // Switch between styles. 129 | switch style { 130 | case "margin-top-bottom": 131 | fmt.Println(gosl.RenderStyled(s, lp.MarginTop(1).MarginBottom(1))) 132 | case "margin-top": 133 | fmt.Println(gosl.RenderStyled(s, lp.MarginTop(1))) 134 | case "margin-bottom": 135 | fmt.Println(gosl.RenderStyled(s, lp.MarginBottom(1))) 136 | case "margin-left": 137 | fmt.Println(gosl.RenderStyled(s, lp.MarginLeft(1))) 138 | default: 139 | fmt.Println(s) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /inputs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/csv" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "unicode/utf8" 10 | ) 11 | 12 | // newInputs provides a new data input instance. 13 | func newInputs(config *Config) (*Inputs, error) { 14 | // Open the given file with input data. 15 | file, err := os.Open(filepath.Clean(inputDataFilePath)) 16 | if err != nil { 17 | return nil, fmt.Errorf("can't open file with input data: %v", err) 18 | } 19 | defer file.Close() 20 | 21 | // Set up CSV reader. 22 | csvReader := csv.NewReader(bufio.NewReader(file)) 23 | csvReader.LazyQuotes = true 24 | 25 | // Check, if the CSV column separator is not empty in config. 26 | if config.CSVColumnSeparator != "" { 27 | // Decode rune from the string. 28 | separator, _ := utf8.DecodeRuneInString(config.CSVColumnSeparator) 29 | 30 | // Set separator to the CSV reader. 31 | csvReader.Comma = separator 32 | } 33 | 34 | // Reading CSV. 35 | records, err := csvReader.ReadAll() 36 | if err != nil { 37 | return nil, fmt.Errorf("can't read CSV file with input data: %v", err) 38 | } 39 | 40 | // Create a new Inputs struct. 41 | inputs := &Inputs{} 42 | 43 | // Loop for get mapping for fields of the CSV file. 44 | for index, data := range records { 45 | // Header is the first row. 46 | if index == 0 { 47 | // Collect header to mapping list. 48 | inputs.Mapping = matchIndexes(config.ColumnsOrder, data) 49 | continue 50 | } 51 | 52 | // Collect all other rows to data list. 53 | inputs.Data = append(inputs.Data, data) 54 | } 55 | 56 | // Remove duplicates. 57 | inputs.Data = removeDuplicates(inputs.Data) 58 | 59 | return inputs, nil 60 | } 61 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | func main() { 11 | // Listen to flags. 12 | flag.BoolVar(&initConfig, "i", false, "set to generate initial config and example data files") 13 | flag.StringVar(&configFilePath, "c", "", "set path to your config file") 14 | flag.StringVar(&inputDataFilePath, "d", "", "set path to your CSV file with input data") 15 | flag.StringVar(&envPrefix, "e", "CONFIG", "set prefix used in your environment variables") 16 | 17 | // Parse all given flags. 18 | flag.Parse() 19 | 20 | // If '-i' flag is true, generates config and example data files. 21 | if initConfig { 22 | // Create the config file in the current dir. 23 | if err := os.WriteFile(filepath.Clean("./config.yml"), embedConfigYAMLFile, 0o600); err != nil { 24 | printStyled( 25 | fmt.Sprintf("✕ There was an error with generate config.yml file: %v", err), 26 | "", 27 | ) 28 | } 29 | 30 | // Create the example data file in the current dir. 31 | if err := os.WriteFile(filepath.Clean("./data.csv"), embedDataCSVFile, 0o600); err != nil { 32 | printStyled( 33 | fmt.Sprintf("✕ There was an error with generate data.csv file: %v", err), 34 | "", 35 | ) 36 | } 37 | 38 | // Show success message. 39 | printStyled( 40 | "The configuration and example data files was successfully generated in the current dir!", 41 | "margin-top-bottom", 42 | ) 43 | } else { 44 | // App initialization. 45 | app, err := initialize() 46 | if err != nil { 47 | printStyled( 48 | fmt.Sprintf("✕ There was an error with initialize app: %v", err), 49 | "", 50 | ) 51 | } 52 | 53 | // App start. 54 | if err = app.start(); err != nil { 55 | printStyled( 56 | fmt.Sprintf("✕ There was an error with starting app: %v", err), 57 | "", 58 | ) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // App represents struct for the application instance. 4 | type App struct { 5 | Config *Config 6 | Inputs *Inputs 7 | Filtered *Filtered 8 | Outputs *Outputs 9 | } 10 | 11 | // Config represents struct for the configuration instance. 12 | type Config struct { 13 | SaveFilteredPKToCSV bool `koanf:"save_filtered_pk_to_csv"` 14 | ColumnsOrder []string `koanf:"columns_order"` 15 | CSVColumnSeparator string `koanf:"csv_column_separator"` 16 | API *API `koanf:"api"` 17 | FilterColumns []*Column `koanf:"filter_columns"` 18 | UpdateFields []*Field `koanf:"update_fields"` 19 | } 20 | 21 | // API represents struct for the api instance. 22 | type API struct { 23 | RequestTimeout int `koanf:"request_timeout"` 24 | BaseURL string `koanf:"base_url"` 25 | BaseURLSchema string `koanf:"base_url_schema"` 26 | AuthMethod string `koanf:"auth_method"` 27 | Token string `koanf:"token"` 28 | UpdateEndpoint *Endpoint `koanf:"update_endpoint"` 29 | } 30 | 31 | // Column represents struct for the one column instance.. 32 | type Column struct { 33 | ColumnName string `koanf:"column_name"` 34 | Condition string `koanf:"condition"` 35 | Value string `koanf:"value"` 36 | } 37 | 38 | // Field represents struct for the one field instance. 39 | type Field struct { 40 | FieldName string `koanf:"field_name"` 41 | Values []string `koanf:"values"` 42 | Conditions []*Column `koanf:"conditions"` 43 | } 44 | 45 | // Endpoint represents struct for the one endpoint instance. 46 | type Endpoint struct { 47 | AddPKToEndpointName bool `koanf:"add_pk_to_endpoint_name"` 48 | EndpointName string `koanf:"endpoint_name"` 49 | ContentType string `koanf:"content_type"` 50 | HTTPMethod string `koanf:"http_method"` 51 | EndpointBody map[string]any `koanf:"endpoint_body"` 52 | } 53 | 54 | // Inputs represents struct for the inputs instance. 55 | type Inputs struct { 56 | Mapping map[string]int 57 | Data [][]string 58 | } 59 | 60 | // Filtered represents struct for the filtered instance. 61 | type Filtered struct { 62 | Data [][]string 63 | } 64 | 65 | // Outputs represents struct for the outputs instance. 66 | type Outputs struct { 67 | Data []*Output 68 | } 69 | 70 | // Output represents struct for the one output instance. 71 | type Output struct { 72 | ID string 73 | FieldName string 74 | Values []string 75 | } 76 | -------------------------------------------------------------------------------- /outputs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/koddr/gosl" 7 | ) 8 | 9 | // newOutputs provides a new output instance. 10 | func newOutputs(config *Config, inputs *Inputs, filtered *Filtered) (*Outputs, error) { 11 | // Create a temp slice of slices for data. 12 | dataToOutput := inputs.Data 13 | 14 | // Check, if filtered data is not nil or empty. 15 | if filtered.Data != nil && len(filtered.Data) > 0 { 16 | // Set filtered data to the temp slice of slices. 17 | dataToOutput = filtered.Data 18 | } 19 | 20 | // Create a new Outputs instance. 21 | outputs := &Outputs{} 22 | 23 | // Loop for all data (into filtered or inputs slices). 24 | for _, data := range dataToOutput { 25 | // Loop for all update fields. 26 | for _, field := range config.UpdateFields { 27 | // Create a temp slice. 28 | m := make([]bool, 0) 29 | 30 | // Loop for checking condition. 31 | for index, condition := range field.Conditions { 32 | // Matching values with condition between input (in filtered or inputs) data 33 | // and column (in config file). 34 | match, err := matchValues(data[inputs.Mapping[condition.ColumnName]], condition.Value, condition.Condition) 35 | if err != nil { 36 | return nil, fmt.Errorf( 37 | "%w, column %d ('%v') in row %v", 38 | err, index, condition.ColumnName, data, 39 | ) 40 | } 41 | 42 | // Collect matching result to the temp slice. 43 | m = append(m, match) 44 | } 45 | 46 | // Check, if there is no false in the conditions, it means 47 | // that the update fields settings have been respected. 48 | if !gosl.ContainsInSlice(m, false) { 49 | // Collect the current data to the outputs result. 50 | outputs.Data = append(outputs.Data, &Output{ 51 | ID: data[inputs.Mapping[config.ColumnsOrder[0]]], 52 | FieldName: field.FieldName, 53 | Values: field.Values, 54 | }) 55 | } 56 | } 57 | } 58 | 59 | return outputs, nil 60 | } 61 | -------------------------------------------------------------------------------- /save_filtered.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | ) 10 | 11 | // saveFilteredPKToCSV provides save process to filtered PK. 12 | func (app *App) saveFilteredPKToCSV() error { 13 | // Create name for CSV file. 14 | fileName := fmt.Sprintf("./filtered-%d.csv", time.Now().Unix()) 15 | 16 | // Create a new CSV file in the current dir. 17 | file, err := os.Create(filepath.Clean(fileName)) 18 | if err != nil { 19 | return err 20 | } 21 | defer file.Close() 22 | 23 | // Create a new CSV writer instance. 24 | writer := csv.NewWriter(file) 25 | defer writer.Flush() 26 | 27 | // Write the header with PK column name of the CSV file. 28 | if errWriter := writer.Write([]string{app.Config.ColumnsOrder[0]}); errWriter != nil { 29 | return errWriter 30 | } 31 | 32 | // Write data body of the CSV file. 33 | for _, data := range app.Outputs.Data { 34 | // Take only PK column data. 35 | if errWriter := writer.Write([]string{data.ID}); errWriter != nil { 36 | return errWriter 37 | } 38 | } 39 | 40 | printStyled( 41 | fmt.Sprintf("– Saving filtered transactions to CSV file '%s' in the current dir... OK!", fileName), 42 | "margin-top", 43 | ) 44 | 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /update_fields.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "net/textproto" 8 | "net/url" 9 | "strings" 10 | "time" 11 | 12 | "github.com/koddr/gosl" 13 | ) 14 | 15 | // updateField provides update process for the given transaction ID. 16 | func (app *App) updateField(id, fieldName string, fieldValues []string) error { 17 | // Set primary key (PK) to the endpoint body. 18 | _, body := gosl.ModifyByValue(app.Config.API.UpdateEndpoint.EndpointBody, app.Config.ColumnsOrder[0], id) 19 | 20 | // Check, if field has a many values. 21 | if len(fieldValues) > 1 { 22 | // If true, set many values to the field. 23 | _, body = gosl.ModifyByValue(body, fieldName, fieldValues) 24 | } else { 25 | // If false, set the only one value to the field. 26 | _, body = gosl.ModifyByValue(body, fieldName, fieldValues[0]) 27 | } 28 | 29 | // Marshaling endpoint body. 30 | reqBody, err := gosl.Marshal(&body) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | // Set endpoint from the config. 36 | endpoint := app.Config.API.UpdateEndpoint.EndpointName 37 | 38 | // If endpoint want to set primary key (PK) to it. 39 | if app.Config.API.UpdateEndpoint.AddPKToEndpointName { 40 | // Set primary key (PK) to the endpoint from the config. 41 | endpoint = fmt.Sprintf(app.Config.API.UpdateEndpoint.EndpointName, id) 42 | } 43 | 44 | // Build API URL. 45 | apiUrl := url.URL{ 46 | Scheme: strings.ToLower(app.Config.API.BaseURLSchema), 47 | Host: strings.ToLower(app.Config.API.BaseURL), 48 | Path: strings.ToLower(endpoint), 49 | } 50 | 51 | // Create HTTP request to API URL with endpoint body. 52 | req, err := http.NewRequest(app.Config.API.UpdateEndpoint.HTTPMethod, apiUrl.String(), bytes.NewBuffer(reqBody)) 53 | if err != nil { 54 | return err 55 | } 56 | defer req.Body.Close() 57 | 58 | // Set request headers. 59 | req.Header.Set( 60 | textproto.CanonicalMIMEHeaderKey("authorization"), 61 | gosl.Concat(app.Config.API.AuthMethod, " ", app.Config.API.Token), 62 | ) 63 | req.Header.Set( 64 | textproto.CanonicalMIMEHeaderKey("content-type"), 65 | app.Config.API.UpdateEndpoint.ContentType, 66 | ) 67 | 68 | // Create a new HTTP client. 69 | client := &http.Client{ 70 | // Set timeout count from the config. 71 | Timeout: time.Second * time.Duration(app.Config.API.RequestTimeout), 72 | } 73 | 74 | // Make HTTP request by the client. 75 | resp, err := client.Do(req) 76 | if err != nil { 77 | return err 78 | } 79 | defer resp.Body.Close() 80 | 81 | // Check, if response HTTP status < 300. 82 | if resp.StatusCode < http.StatusMultipleChoices { 83 | printStyled( 84 | fmt.Sprintf( 85 | "✓ Field '%s' with values '%s' in the transaction '%s' has been successfully updated (HTTP %d)", 86 | fieldName, fieldValues, id, resp.StatusCode, 87 | ), 88 | "margin-left", 89 | ) 90 | } else { 91 | // Else, return error message with the raw response body. 92 | return fmt.Errorf("transaction '%s' returned HTTP %d", id, resp.StatusCode) 93 | } 94 | 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /wire.go: -------------------------------------------------------------------------------- 1 | //go:build wireinject 2 | 3 | package main 4 | 5 | import "github.com/google/wire" 6 | 7 | // initialize provides dependency injection process by the "google/wire" package. 8 | func initialize() (*App, error) { 9 | wire.Build(newConfig, newInputs, newFiltered, newOutputs, newApp) 10 | return &App{}, nil 11 | } 12 | --------------------------------------------------------------------------------