├── .github
└── workflows
│ ├── codeql-analysis.yml
│ ├── go.yml
│ └── release.yml
├── .gitignore
├── .vscode
└── extensions.json
├── LICENSE
├── README.md
├── docs
├── README.md
├── about.md
├── examples.md
├── img
│ ├── bafi.svg
│ ├── bafiIcon.svg
│ ├── calc.jpg
│ ├── favicon.ico
│ └── scheme.svg
└── js
│ ├── bafi.wasm
│ └── wasm_exec.js
├── filesTest.yaml
├── functions.go
├── functions_test.go
├── go.mod
├── go.sum
├── lua
├── functions.lua
└── json.lua
├── main.go
├── main_test.go
├── mkdocs.yml
├── template.tmpl
├── testdata.xml
└── workspace.code-workspace
/.github/workflows/codeql-analysis.yml:
--------------------------------------------------------------------------------
1 | # For most projects, this workflow file will not need changing; you simply need
2 | # to commit it to your repository.
3 | #
4 | # You may wish to alter this file to override the set of languages analyzed,
5 | # or to provide custom queries or build logic.
6 | #
7 | # ******** NOTE ********
8 | # We have attempted to detect the languages in your repository. Please check
9 | # the `language` matrix defined below to confirm you have the correct set of
10 | # supported CodeQL languages.
11 | #
12 | name: "CodeQL"
13 |
14 | on:
15 | push:
16 | branches: [ main ]
17 | pull_request:
18 | # The branches below must be a subset of the branches above
19 | branches: [ main ]
20 | schedule:
21 | - cron: '44 2 * * 4'
22 |
23 | jobs:
24 | analyze:
25 | name: Analyze
26 | runs-on: ubuntu-latest
27 | permissions:
28 | actions: read
29 | contents: read
30 | security-events: write
31 |
32 | strategy:
33 | fail-fast: false
34 | matrix:
35 | language: [ 'go' ]
36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
37 | # Learn more:
38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
39 |
40 | steps:
41 | - name: Checkout repository
42 | uses: actions/checkout@v2
43 |
44 | # Initializes the CodeQL tools for scanning.
45 | - name: Initialize CodeQL
46 | uses: github/codeql-action/init@v1
47 | with:
48 | languages: ${{ matrix.language }}
49 | # If you wish to specify custom queries, you can do so here or in a config file.
50 | # By default, queries listed here will override any specified in a config file.
51 | # Prefix the list here with "+" to use these queries and those in the config file.
52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main
53 |
54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
55 | # If this step fails, then you should remove it and run the build manually (see below)
56 | - name: Autobuild
57 | uses: github/codeql-action/autobuild@v1
58 |
59 | # ℹ️ Command-line programs to run using the OS shell.
60 | # 📚 https://git.io/JvXDl
61 |
62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
63 | # and modify them (or add more) to build your code if your project
64 | # uses a compiled language
65 |
66 | #- run: |
67 | # make bootstrap
68 | # make release
69 |
70 | - name: Perform CodeQL Analysis
71 | uses: github/codeql-action/analyze@v1
72 |
--------------------------------------------------------------------------------
/.github/workflows/go.yml:
--------------------------------------------------------------------------------
1 | name: Go
2 |
3 | on:
4 | push:
5 | branches: [main]
6 | pull_request:
7 | branches: [main]
8 |
9 | jobs:
10 | build:
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v2
14 |
15 | - name: Set up Go
16 | uses: actions/setup-go@v2
17 | with:
18 | go-version: 1.21
19 |
20 | - name: Build
21 | run: go build -v ./...
22 | - name: Test
23 | run: go test -v -cover ./...
24 | goreportcard:
25 | runs-on: ubuntu-latest
26 | steps:
27 | - name: refresh goreportcard
28 | uses: creekorful/goreportcard-action@v1.0
29 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: releaseBuild
2 |
3 | on: release
4 |
5 | jobs:
6 | update-mkdocs:
7 | name: Deploy mkdocs documentation
8 | runs-on: ubuntu-latest
9 | steps:
10 | - uses: actions/checkout@v2
11 | with:
12 | fetch-depth: 0
13 | - uses: actions/setup-python@v2
14 | with:
15 | python-version: 3.x
16 | - run: pip install mkdocs
17 | - run: mkdocs gh-deploy --force
18 | release-windows-amd64:
19 | name: release windows/amd64
20 | runs-on: ubuntu-latest
21 | steps:
22 | - name: checkout
23 | uses: actions/checkout@v2
24 | - name: compile and release
25 | uses: mmalcek/go-release.action@v1.0.6
26 | env:
27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
28 | PROJECT_NAME: bafi
29 | CGO_ENABLED: 0
30 | GOARCH: amd64
31 | GOOS: windows
32 | CMD_PATH: -buildvcs=false
33 | EXTRA_FILES: "docs/README.md docs/examples.md docs/about.md LICENSE testdata.xml template.tmpl lua/json.lua lua/functions.lua"
34 | release-linux-amd64:
35 | name: release linux/amd64
36 | runs-on: ubuntu-latest
37 | steps:
38 | - name: checkout
39 | uses: actions/checkout@v2
40 | - name: compile and release
41 | uses: mmalcek/go-release.action@v1.0.6
42 | env:
43 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44 | PROJECT_NAME: bafi
45 | CGO_ENABLED: 0
46 | GOARCH: amd64
47 | GOOS: linux
48 | CMD_PATH: -buildvcs=false
49 | EXTRA_FILES: "docs/README.md docs/examples.md docs/about.md LICENSE testdata.xml template.tmpl lua/json.lua lua/functions.lua"
50 | release-linux-386:
51 | name: release linux/386
52 | runs-on: ubuntu-latest
53 | steps:
54 | - name: checkout
55 | uses: actions/checkout@v2
56 | - name: compile and release
57 | uses: mmalcek/go-release.action@v1.0.6
58 | env:
59 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
60 | PROJECT_NAME: bafi
61 | CGO_ENABLED: 0
62 | GOARCH: "386"
63 | GOOS: linux
64 | CMD_PATH: -buildvcs=false
65 | EXTRA_FILES: "docs/README.md docs/examples.md docs/about.md LICENSE testdata.xml template.tmpl lua/json.lua lua/functions.lua"
66 | release-darwin-amd64:
67 | name: release darwin/amd64
68 | runs-on: ubuntu-latest
69 | steps:
70 | - name: checkout
71 | uses: actions/checkout@v2
72 | - name: compile and release
73 | uses: mmalcek/go-release.action@v1.0.6
74 | env:
75 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
76 | PROJECT_NAME: bafi
77 | CGO_ENABLED: 0
78 | GOARCH: amd64
79 | GOOS: darwin
80 | CMD_PATH: -buildvcs=false
81 | EXTRA_FILES: "docs/README.md docs/examples.md docs/about.md LICENSE testdata.xml template.tmpl lua/json.lua lua/functions.lua"
82 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | dist
2 | bafi
3 | site
4 | *.exe
5 | *.exe~
6 | *.csv
7 | output.*
8 | temp.txt
9 | cover.out
10 | oryxBuildBinary
11 |
12 | .DS_Store
13 |
--------------------------------------------------------------------------------
/.vscode/extensions.json:
--------------------------------------------------------------------------------
1 | {
2 | "recommendations": [
3 | "golang.go",
4 | "casualjim.gotemplate"
5 | ]
6 | }
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 mmalcek
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://github.com/mmalcek/bafi/actions/workflows/go.yml)
2 | [](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml)
3 | [](https://goreportcard.com/report/github.com/mmalcek/bafi)
4 | [](https://github.com/mmalcek/bafi/blob/main/LICENSE)
5 | [](https://github.com/avelino/awesome-go#text-processing)
6 | [](https://github.com/mmalcek/bafi/releases/latest)
7 |
8 | # Universal JSON, BSON, YAML, CSV, XML, mt940 translator to ANY format using templates
9 |
10 | 
11 |
12 | ## Key features
13 |
14 | - Various input formats **(json, bson, yaml, csv, xml, mt940)**
15 | - Flexible output formatting using text templates
16 | - Support for [Lua](https://www.lua.org/pil/contents.html) custom functions which allows very flexible data manipulation
17 | - stdin/stdout support which allows get data from source -> translate -> delivery to destination. This allows easily translate data between different web services like **REST to SOAP, SOAP to REST, REST to CSV, ...**
18 | - Merge multiple input files in various formats into single output file formated using template
19 | - Support chatGPT queries to analyze or format data (experimental)
20 |
21 | ## Documentation [https://mmalcek.github.io/bafi/](https://mmalcek.github.io/bafi/)
22 |
23 | ## Releases (Windows, MAC, Linux) [https://github.com/mmalcek/bafi/releases](https://github.com/mmalcek/bafi/releases)
24 |
25 | usage:
26 |
27 | ```
28 | bafi.exe -i testdata.xml -t template.tmpl -o output.txt
29 | ```
30 |
31 | or
32 |
33 | ```
34 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}"
35 | ```
36 |
37 | or
38 |
39 | ```
40 | curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "What's the current CZK rate?"
41 | ```
42 |
43 | More examples and description in [documentation](https://mmalcek.github.io/bafi/)
44 |
45 | **If you like this app you can buy me a coffe ;)**
46 |
47 |
48 | 
49 |
50 |
--------------------------------------------------------------------------------
/docs/README.md:
--------------------------------------------------------------------------------
1 | # BaFi
2 |
3 | **Universal JSON, BSON, YAML, CSV, XML, mt940 translator to ANY format using templates**
4 |
5 | **Github repository**
6 |
7 | - [https://github.com/mmalcek/bafi](https://github.com/mmalcek/bafi)
8 |
9 | **Releases (Windows, MAC, Linux)**
10 |
11 | - [https://github.com/mmalcek/bafi/releases](https://github.com/mmalcek/bafi/releases)
12 |
13 | ## Key features
14 |
15 | - Various input formats **(json, bson, yaml, csv, xml, mt940)**
16 | - Flexible output formatting using text templates
17 | - Output can be anything: HTML page, SQL Query, Shell script, CSV file, ...
18 | - Support for [Lua](https://www.lua.org/pil/contents.html) custom functions which allows very flexible data manipulation
19 | - stdin/stdout support which allows get data from source -> translate -> delivery to destination. This allows easily translate data between different web services like **REST to SOAP, SOAP to REST, REST to CSV, ...**
20 | - Merge multiple input files in various formats into single output file formated using template
21 | - Support chatGPT queries to analyze or format data (experimental)
22 |
23 | 
24 |
25 | [](https://github.com/mmalcek/bafi/actions/workflows/go.yml)
26 | [](https://github.com/mmalcek/bafi/actions/workflows/codeql-analysis.yml)
27 | [](https://goreportcard.com/report/github.com/mmalcek/bafi)
28 | [](https://github.com/mmalcek/bafi/blob/main/LICENSE)
29 | [](https://github.com/avelino/awesome-go#text-processing)
30 | [](https://github.com/mmalcek/bafi/releases/latest)
31 |
32 |
33 |
34 | **If you like this app you can buy me a coffe ;)**
35 |
36 |
37 |
38 |
39 |
40 | ## How does it work?
41 |
42 | Application automaticaly parse input data into object which can be simply accessed in tamplate using dot notation where first dot represent root of object **{{ . }}**.
43 |
44 | For example JSON document **myUser.json**
45 |
46 | ```json
47 | {
48 | "user": {
49 | "name": "John Doe",
50 | "age": 25,
51 | "address": {
52 | "street": "Main Street",
53 | "city": "New York",
54 | "state": "NY"
55 | },
56 | "favourite_colors": ["red", "green", "blue"]
57 | }
58 | }
59 | ```
60 |
61 | - Get user name:
62 |
63 | ```sh
64 | bafi.exe -i myUser.json -t '?{{.user.name}}'
65 | ```
66 |
67 | - Use function to change all letters to uppercase:
68 |
69 | ```sh
70 | bafi.exe -i myUser.json -t '?{{upper .user.name}}'
71 | ```
72 |
73 | - Use IF statement to compare user age to 20:
74 |
75 | ```sh
76 | bafi.exe -i myUser.json -t '?User is {{if gt (toInt .user.age) 20}}old{{else}}young{{end}}.'
77 | ```
78 |
79 | - List favourite colors:
80 |
81 | ```sh
82 | bafi.exe -i myUser.json -t '?{{range .user.favourite_colors}}{{.}},{{end}}'
83 | ```
84 |
85 | - Format data using template file **myTemplate.tmpl** and save output to **myUser.txt**:
86 |
87 | ```sh
88 | bafi.exe -i myUser.json -t myTemplate.tmpl -o myUser.txt
89 | ```
90 |
91 | ```
92 | {{- /* Content of myTemplate.tmpl file */ -}}
93 | User: {{.user.name}}
94 | Age: {{.user.age}}
95 | Address: {{.user.address.street}}, {{.user.address.city}} - {{.user.address.state}}
96 | {{- /* Create list of colors and remove comma at the end */ -}}
97 | {{- $colors := ""}}{{range .user.favourite_colors}}{{$colors = print $colors . ", "}}{{end}}
98 | {{- $colors = print (trimSuffix $colors ", " )}}
99 | Favourite colors: {{$colors}}
100 | ```
101 |
102 | note: in Powershell you must use .\\bafi.exe e.g.
103 |
104 | ```powershell
105 | .\bafi.exe -i input.csv -t "?{{toXML .}}"
106 | curl.exe -s someurl.com/api/xxx | .\bafi.exe -f json -t "?{{toXML .}}"
107 | ```
108 |
109 | More examples [here](examples/#template)
110 |
111 |
112 |
125 |
126 | ## Online demo (WASM)
127 |
128 | **Just try it here :)**
129 |
143 |
144 |
153 |
154 |
161 |
162 |
163 |
164 |
165 | ## Command line arguments
166 |
167 | - **-i input.xml** Input file name.
168 | - If not defined app tries read stdin
169 | - If prefixed with "?" (**-i ?files.yaml**) app will expect yaml file with multiple files description. See [example](examples/#multiple-input-files)
170 | - **-o output.txt** Output file name.
171 | - If not defined result is send to stdout
172 | - **-t template.tmpl** Template file. Alternatively you can use _inline_ template
173 | - inline template must start with **?** e.g. -t **"?{{.someValue}}"**
174 | - **-f json** Input format.
175 | - Supported formats: **json, bson, yaml, csv, xml, mt940**
176 | - If not defined (for file input) app tries detect input format automatically by file extension
177 | - **-d ','** Data delimiter
178 | - format CSV:
179 | - Can be defined as string e.g. -d ',' or as [hex](https://www.asciitable.com/asciifull.gif) value prefixed by **0x** e.g. 'TAB' can be defined as -f 0x09. Default delimiter is comma (**,**)
180 | - format mt940:
181 | - For Multiple messages in one file (e.g. Multicash). Can be defined as string e.g. -d "-\}\r\n" or "\r\n$" . If delimiter is set BaFi will return array of mt940 messages
182 | - **-v** Show current verion
183 | - **-h** list available command line arguments
184 | - **-gk myChatGPTToken** - ChatGPT token
185 | - **-gq "What's the current CZK rate?"** - ChatGPT query
186 | - **-gm gpt35** - ChatGPT model. Currently supportsed options "gpt35"(default), "gpt4", "gpt4o", "gpt4o-mini"
187 |
188 | ```sh
189 | bafi.exe -i testdata.xml -t template.tmpl -o output.txt
190 | ```
191 |
192 | More examples [here](examples/#command-line)
193 |
194 | ## Templates
195 |
196 | Bafi uses [text/template](https://pkg.go.dev/text/template). Here is a quick summary how to use. Examples are based on _testdata.xml_ included in project
197 |
198 | note: in **vscode** you can use [gotemplate-syntax](https://marketplace.visualstudio.com/items?itemName=casualjim.gotemplate) for syntax highlighting
199 |
200 | ### Comments
201 |
202 | ```
203 | {{/* a comment */}}
204 | {{- /* a comment with white space trimmed from preceding and following text */ -}}
205 | ```
206 |
207 | ### Trim new line
208 |
209 | New line before or after text can be trimmed by adding dash
210 |
211 | ```
212 | {{- .TOP_LEVEL}}, {{.TOP_LEVEL -}}
213 | ```
214 |
215 | ### Accessing data
216 |
217 | Data are accessible by _pipline_ which is represented by dot
218 |
219 | - Simplest template
220 |
221 | ```
222 | {{.}}
223 | ```
224 |
225 | - Get data form inner node
226 |
227 | ```
228 | {{.TOP_LEVEL}}
229 | ```
230 |
231 | - Get data from XML tag. XML tags are autoprefixed by dash and accessible as index
232 |
233 | ```
234 | {{index .TOP_LEVEL "-description"}}
235 | ```
236 |
237 | - Convert TOP_LEVEL node to JSON
238 |
239 | ```
240 | {{toJSON .TOP_LEVEL}}
241 | ```
242 |
243 | ### Variables
244 |
245 | You can store selected data to [template variable](https://pkg.go.dev/text/template#hdr-Variables)
246 |
247 | ```
248 | {{$myVar := .TOP_LEVEL}}
249 | ```
250 |
251 | ### Actions
252 |
253 | Template allows to use [actions](https://pkg.go.dev/text/template#hdr-Actions), for example
254 |
255 | Iterate over lines
256 |
257 | ```
258 | {{range .TOP_LEVEL.DATA_LINE}}{{.val1}}{{end}}
259 | ```
260 |
261 | If statement
262 |
263 | ```
264 | {{if gt (int $val1) (int $val2)}}Value1{{else}}Value2{{end}} is greater
265 | ```
266 |
267 | ### Functions
268 |
269 | In go templates all operations are done by functions where function name is followed by operands
270 |
271 | For example:
272 |
273 | count val1+val2
274 |
275 | ```
276 | {{add $val1 $val2}}
277 | ```
278 |
279 | count (val1+val2)/val3
280 |
281 | ```
282 | {{div (add $val1 $val2) $val3}}
283 | ```
284 |
285 | This is called [Polish notation](https://en.wikipedia.org/wiki/Polish_notation) or "Prefix notation" also used in another languages like [Lisp]()
286 |
287 | The key benefit of using this notation is that order of operations is clear. For example **6/2\*(1+2)** - even diferent calculators may have different opinion on order of operations in this case. With Polish notation order of operations is strictly defined (from inside to outside) **div 6 (mul 2 (add 1 2))** . This brings benefits with increasing number of operations especially in templates where math and non-math operations can be mixed together.
288 |
289 | 
290 |
291 | For example we have json array of items numbered from **0**
292 |
293 | ```json
294 | { "items": ["item-0", "item-1", "item-2", "item-3"] }
295 | ```
296 |
297 | We need change items numbering to start with **1**. To achieve this we have to do series of operations: 1. trim prefix "item-" -> 2. convert to int -> 3. add 1 -> 4. convert to string -> 5. append "item-" for all items in range. This can be done in one line
298 |
299 | ```
300 | {{ range .items }}{{ print "item-" (toString (add1 (toInt (trimPrefix . "item-")))) }} {{ end }}
301 | ```
302 |
303 | or alternatively (slightly shorter) print formatted string - examples [here](https://zetcode.com/golang/string-format/), documentation [here](https://golang.org/pkg/fmt/)
304 |
305 | ```
306 | {{ range .items }}{{ printf "item-%d " (add1 (toInt (trimPrefix . "item-"))) }}{{ end }}
307 | ```
308 |
309 | but BaFi also tries automaticaly cast variables so the shortest option is
310 |
311 | ```
312 | {{range .items}}{{print "item-" (add1 (trimPrefix . "item-"))}} {{end}}
313 | ```
314 |
315 | Expected result: **item-1 item-2 item-3 item-4**
316 |
317 | There are 3 categories of functions
318 |
319 | #### Native functions
320 |
321 | text/template integrates [native functions](https://pkg.go.dev/text/template#hdr-Functions) to work with data
322 |
323 | #### Additional functions
324 |
325 | Asside of integated functions bafi contains additional common functions
326 |
327 | ##### Math functions
328 |
329 | - **add** - {{add .Value1 .Value2}}
330 | - **add1** - {{add1 .Value1}} = Value1+1
331 | - **sub** - substract
332 | - **div** - divide
333 | - **mod** - modulo
334 | - **mul** - multiply
335 | - **randInt** - return random integer {{randInt .Min .Max}}
336 | - **add1f** - "...f" functions parse float but provide **decimal** operations using [shopspring decimal](https://github.com/shopspring/decimal)
337 | - **addf**
338 | - **subf**
339 | - **divf**
340 | - **mulf**
341 | - **round** - {{round .Value1 2}} - will round to 2 decimals
342 | - **max** - {{round .Value1 .Value2 .Value3 ...}} get Max value from range
343 | - **min** - get Min value from range
344 | - **maxf**
345 | - **minf**
346 |
347 | ##### Date functions
348 |
349 | - **dateFormat** - {{dateFormat .Value "oldFormat" "newFormat"}} - [GO time format](https://programming.guide/go/format-parse-string-time-date-example.html)
350 | - {{dateFormat "2021-08-26T22:14:00" "2006-01-02T15:04:05" "02.01.2006-15:04"}}
351 | - **dateFormatTZ** - {{dateFormatTZ .Value "oldFormat" "newFormat" "timeZone"}}
352 | - This fuction is similar to dateFormat but applies timezone offset - [Timezones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
353 | - {{dateFormatTZ "2021-08-26T03:35:00.000+04:00" "2006-01-02T15:04:05.000-07:00" "02.01.2006-15:04" "Europe/Prague"}}
354 | - **dateToInt** - {{dateToInt .Value "dateFormat"}} - convert date to integer (unixtime, int64), usefull for comparing dates
355 | - **intToDate** - {{intToDate .Value "dateFormat"}} - convert integer (unixtime, int64) to date, usefull for comparing dates
356 | - **now** - {{now "02.01.2006"}} - GO format date (see notes below)
357 |
358 | ##### String functions
359 |
360 | - **addSubstring** - {{addSubstring $myString, "XX", $position}} add substring to $position in string (if $position is 1,2,3 = Adding from right, if -1,-2,-3 = Adding from left)
361 | - **atoi** - {{atoi "042"}} - string to int. Result will be 42. atoi must be used especially for convert strings with leading zeroes
362 | - **b64enc** - encode to base64
363 | - **b64dec** - decode from base64
364 | - **b32enc** - oncode to base32
365 | - **b32dec** - decode from base32
366 | - **contains** - check if string contains substring e.g. {{contains "aaxbb" "xb"}}
367 | - **indexOf** - {{indexOf "aaxbb" "xb"}} - returns indexOf first char of substring in string
368 | - **isArray** - {{isArray .Value1}} - check if value is array
369 | - **isBool** - {{isBool .Value1}} - check if value is bool
370 | - **isInt** - {{isInt .Value1}} - check if value is int
371 | - **isFloat64** - {{isFloat64 .Value1}} - check if value is float64
372 | - **isString** - {{isString .Value1}} - check if value is string
373 | - **isMap** - {{isMap .Value1}} - check if value is map
374 | - **regexMatch** - {{regexMatch pattern .Value1}} more about go [regex](https://gobyexample.com/regular-expressions)
375 | - **replaceAll** - {{replaceAll "oldValue" "newValue" .Value}} - replace all occurences of "oldValue" with "newValue" e.g. {{replaceAll "x" "Z" "aaxbb"}} -> "aaZbb"
376 | - **replaceAllRegex** - {{replaceAllRegex "regex" "newValue" .Value}} - replace all occurences of "regex" with "newValue" e.g. {{replaceAllRegex "[a-d]", "Z" "aaxbb"}} -> "ZZxZZ"
377 | - **lower** - to lowercase
378 | - **trim** - remove leading and trailing whitespace
379 | - **trimPrefix** - {{trimPrefix "!Hello World!" "!"}} - returns "Hello World!"
380 | - **trimSuffix** - {{trimSuffix "!Hello World!" "!"}} - returns "!HelloWorld"
381 | - **mapJSON** - convert stringified JSON to map so it can be used as object or translated to other formats (e.g. "toXML"). Check template.tmpl for example
382 | - **mustArray** - {{mustArray .Value1}} - convert to array. Useful with XML where single node is not treated as array
383 | - **toBool** - {{toBool "true"}} - string to bool
384 | - **toDecimal** - {{toDecimal "3.14159"}} - cast to decimal (if error return 0)
385 | - **toDecimalString** - {{toDecimalString "3.14159"}} - cast to decimal string (if error return "error message")
386 | - **toFloat64** - {{float64 "3.14159"}} - cast to float64
387 | - **toInt** - {{int true}} - cast to int. Result will be 1. If you need convert string with leading zeroes use "atoi"
388 | - **toInt64** - {{int64 "42"}} - cast to int64. Result will be 42. If you need convert string with leading zeroes use "atoi"
389 | - **toString** - {{toString 42}} - int to string
390 | - **toJSON** - convert input object to JSON
391 | - **toBSON** - convert input object to BSON
392 | - **toYAML** - convert input object to YAML
393 | - **toXML** - convert input object to XML
394 | - **trimAll** - {{trimAll "!Hello World!" "!"}} - returns "Hello World"
395 | - **upper** - to uppercase
396 | - **uuid** - generate UUID
397 |
398 | #### Lua custom functions
399 |
400 | You can write your own custom lua functions defined in ./lua/functions.lua file
401 |
402 | Call Lua function in template ("sum" - Lua function name)
403 |
404 | ```
405 | {{lua "sum" .val1 .val2}}
406 | ```
407 |
408 | - Input is always passed as stringified JSON and should be decoded (json.decode(incomingData))
409 | - Output must be passed as string
410 | - lua table array starts with 1
411 | - Lua [documentation](http://www.lua.org/manual/5.1/)
412 |
413 | Minimal functions.lua example
414 |
415 | ```lua
416 | json = require './lua/json'
417 |
418 | function sum(incomingData)
419 | dataTable = json.decode(incomingData)
420 | return tostring(tonumber(dataTable[1]) + tonumber(dataTable[2]))
421 | end
422 | ```
423 |
424 | Check [examples](examples/) and **template.tmpl** and **testdata.xml** for advanced examples
425 |
--------------------------------------------------------------------------------
/docs/about.md:
--------------------------------------------------------------------------------
1 | # About BaFi
2 |
3 | ## What BaFi stands for
4 | BaFi is an acronym for [**Babel Fish**](https://hitchhikers.fandom.com/wiki/Babel_Fish) from [The Hitchhiker's Guide to the Galaxy](https://en.wikipedia.org/wiki/The_Hitchhiker's_Guide_to_the_Galaxy)
5 |
8 |
9 | ## Motivation
10 | This tool has been created for [systems@work](https://systemsatwork.com) internal purposes.
11 | It is and always will be Open-source/[MIT license](https://github.com/mmalcek/bafi/blob/main/LICENSE).
12 |
13 | ## Questions
14 | If you have any questions, please feel free to open discussion [here](https://github.com/mmalcek/bafi/discussions) or report an issue [here](https://github.com/mmalcek/bafi/issues).
15 |
--------------------------------------------------------------------------------
/docs/examples.md:
--------------------------------------------------------------------------------
1 | # Examples
2 |
3 | ## Command line
4 |
5 | note: in Powershell you must use .\\bafi.exe e.g.
6 |
7 | ```powershell
8 | .\bafi.exe -i input.csv -t "?{{toXML .}}"
9 | curl.exe -s someurl.com/api/xxx | .\bafi.exe -f json -t "?{{toXML .}}"
10 | ```
11 |
12 | ### Basic
13 |
14 | Get data from _testdata.xml_ -> process using _template.tmpl_ -> save output as _output.txt_
15 |
16 | ```sh
17 | bafi.exe -i testdata.xml -t template.tmpl -o output.txt
18 | ```
19 |
20 | ### Inline template
21 |
22 | Get data from _testdata.xml_ -> process using inline template -> save output as _output.json_
23 |
24 | ```sh
25 | bafi.exe -i testdata.xml -o output.json -t "?{{toJSON .}}"
26 | ```
27 |
28 | note: BaFi inline template must start with **?** e.g. **"?{{toJSON .}}"**
29 | [How to format inline templates](https://pkg.go.dev/text/template#hdr-Examples)
30 |
31 | ### Stdin/REST
32 |
33 | Get data from REST api -> convert to XML -> output to Stdout.
34 |
35 | ```sh
36 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}"
37 | ```
38 |
39 | More info about curl [here](https://curl.se/) but you can of course use any tool with stdout
40 |
41 | ### Append output file
42 |
43 | Redirect stdout to file and append ( > = replace, >> = apppend )
44 |
45 | ```sh
46 | bafi.exe -i testdata.xml -t template.tmpl >> output.txt
47 | ```
48 |
49 | ## Template
50 |
51 | Examples are based on testdata.tmpl included in project
52 |
53 | ### XML to CSV
54 |
55 | - command
56 |
57 | ```sh
58 | bafi.exe -i testdata.xml -t myTemplate.tmpl -o output.csv
59 | ```
60 |
61 | - myTemplate.tmpl
62 |
63 | ```
64 | Employee,Date,val1,val2,val3,SUM,LuaMultiply,linkedText
65 | {{- range .TOP_LEVEL.DATA_LINE}}
66 | {{index .Employee "-ID"}},
67 | {{- dateFormat .Trans_Date "2006-01-02" "02.01.2006"}},
68 | {{- .val1}},{{.val2}},{{.val3}},
69 | {{- add .val1 .val2}},
70 | {{- lua "mul" .val1 .val2}},"{{index .Linked_Text "-VALUE"}}"
71 | {{- end}}
72 | ```
73 |
74 | ### JSON to CSV
75 |
76 | - command
77 |
78 | ```sh
79 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.html
80 | ```
81 |
82 | - myTemplate.tmpl
83 |
84 | ```
85 | name,surname
86 | {{- range .customers}}
87 | "{{.firstname}}","{{.lastname}}"
88 | {{- end}}
89 | ```
90 |
91 | ### JSON to HTML
92 |
93 | - command
94 |
95 | ```sh
96 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.html
97 | ```
98 |
99 | - myTemplate.tmpl
100 |
101 | ```
102 |
103 |
104 |
105 | | Name | Surname |
|---|
106 | {{- range .customers}}
107 | | {{.firstname}} | {{.lastname}} |
108 | {{- end }}
109 |
110 |
111 |
112 |
113 |
116 | ```
117 |
118 | ### JSON to custom XML
119 |
120 | - command
121 |
122 | ```sh
123 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t myTemplate.tmpl -o output.xml
124 | ```
125 |
126 | - myTemplate.tmpl
127 |
128 | ```
129 |
130 |
131 | {{- range .customers}}
132 |
133 | {{.firstname}}
134 | {{.lastname}}
135 |
136 | {{- end }}
137 |
138 | ```
139 |
140 | ### XML to custom JSON
141 |
142 | - command
143 |
144 | ```sh
145 | bafi.exe -i testdata.xml -t myTemplate.tmpl -o output.json
146 | ```
147 |
148 | - myTemplate.tmpl
149 |
150 | ```
151 | {{- $new := "{\"employees\": [" }}
152 | {{- range .TOP_LEVEL.DATA_LINE}}
153 | {{- $new = print $new "{\"employeeID\":\"" (index .Employee "-ID") "\", \"val1\":" .val1 "}," }}
154 | {{- end}}
155 | {{- /* Trim trailing comma, alternatively you can remove last char by "(slice $new 0 (sub (len $new) 1))" */}}
156 | {{- $new = print (trimSuffix $new "," ) "]}"}}
157 | {{ $new}}
158 | ```
159 |
160 | JSON in $new variable can be mapped to struct and autoformatted to other formats like:
161 |
162 | - Transform $new to YAML
163 |
164 | ```
165 | {{toYAML (mapJSON $new) -}}
166 | ```
167 |
168 | - Transform $new to XML
169 |
170 | ```
171 | {{toXML (mapJSON $new) -}}
172 | ```
173 |
174 | ### CSV to text
175 |
176 | - command
177 |
178 | ```sh
179 | bafi.exe -i users.csv -t myTemplate.tmpl -o output.txt
180 | ```
181 |
182 | users.csv
183 |
184 | ```
185 | name,surname
186 | John,"Jack Doe"
187 | ```
188 |
189 | - myTemplate.tmpl
190 |
191 | ```
192 | Users:
193 | {{- range .}}
194 | Name: {{.name}}, Surname: {{.surname}}
195 | {{- end}}
196 | ```
197 |
198 | note: CSV file must be **[RFC4180](https://datatracker.ietf.org/doc/html/rfc4180)** compliant, file must have header line and separator must be **comma ( , )**. Or you can use command line argument -d ( e.g. **-d ';'** or **-d 0x09** ) to define separator(delimiter).
199 |
200 | ### mt940 to CSV
201 |
202 | - mt940 returns simple struct (Header,Fields,[]Transactions) of strings and additional parsing needs to be done in template. This allows full flexibility on data processing
203 | - Identifiers are prefixed by **"F\_"** (e.g. **:20:** = **.Fields.F_20**)
204 | - if parameter -d (delimiter e.g. -d "-\}\r\n" or "\r\n$") is defined for files with multiple messages (e.g. - Multicash), app returns array of mt940 messages.
205 | - Note: This is actually good place to use integrated [LUA interpreter](/bafi/#lua-custom-functions) where you can create your own set of custom functions to parse data and easily reuse them in templates.
206 |
207 | - command
208 |
209 | ```sh
210 | bafi.exe -i message.sta -t myTemplate.tmpl -o output.csv
211 | ```
212 |
213 | - myTemplate.tmpl
214 |
215 | ```
216 | Reference, balance, VS
217 | {{- $F20 := .Fields.F_20 }}{{ $F60F := .Fields.F_60F }}
218 | {{range .Transactions }}
219 | {{- $vsS := add (indexOf .F_86 "?21") 3 }} {{- $vsE := add $vsS 17 -}}
220 | {{- $F20}}, {{$F60F}}, {{slice .F_86 $vsS $vsE}}
221 | {{ end }}
222 | ```
223 |
224 | ### Any SQL to XML
225 |
226 | Bafi can be used in combination with very interesting tool **USQL** [https://github.com/xo/usql](https://github.com/xo/usql). USQL allows query almost any SQL like database (MSSQL,MySQL,postgres, ...) and get result in various formats. In this example we use -J for JSON. Output can be further processed by BaFi and templates
227 |
228 | ```sh
229 | usql.exe mssql://user:password@server/instance/database -c "SELECT * FROM USERS" -J -q | bafi.exe -f json -t "?{{toXML .}}"
230 | ```
231 |
232 | ### MongoDump to CSV
233 |
234 | - command
235 |
236 | ```sh
237 | bafi.exe -i users.bson -t myTemplate.tmpl -o output.html
238 | ```
239 |
240 | - myTemplate.tmpl
241 |
242 | ```
243 | name,surname
244 | {{- range .}}
245 | "{{.firstname}}","{{.lastname}}"
246 | {{- end}}
247 | ```
248 |
249 | ### Dashes in key names
250 |
251 | If key name contains dashes ( - ) bafi will fail with error "bad character U+002D '-'" for example:
252 |
253 | ```
254 | {{.my-key.subkey}}
255 | ```
256 |
257 | This is known limitation of go templates which can be solved by workaround
258 |
259 | ```
260 | {{index . "my-key" "subkey"}}
261 | ```
262 |
263 | ### Input autoformat to XXX
264 |
265 | Input data can be easily fomated to oher formats by functions **toXML,toJSON,toBSON,toYAML**. In this case its not necesarry add template file because it's as easy as
266 |
267 | ```sh
268 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toXML .}}" -o output.xml
269 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toJSON .}}" -o output.json
270 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toBSON .}}" -o output.bson
271 | curl.exe -s https://api.predic8.de/shop/customers/ | bafi.exe -f json -t "?{{toYAML .}}" -o output.yml
272 | ```
273 |
274 | ### ChatGPT query
275 |
276 | ```sh
277 | curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "What's the current CZK rate?"
278 | curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml | ./bafi -f xml -gk myChatGPTToken -gq "format rates to html" -gm gpt4
279 | curl -s "https://api.open-meteo.com/v1/forecast?latitude=52.52&longitude=13.41&hourly=temperature_2m&past_days=7&forecast_days=0" | ./bafi -f json -gk "myChatGPTToken" -gm gpt4o-mini -gq "Create forecast for next 2 days based on provided data"
280 | ./bafi -i invoice.json -gk myChatGPTToken -gq "create XML UBL format invoice" -o invoice.xml
281 |
282 | ```
283 |
284 | ### Multiple input files
285 |
286 | Bafi can read multiple input files and merge them into one output file. This will require aditional file with files description.
287 | Description file must be in YAML format as described below and prefixed by question mark **"?"** for examle **bafi.exe -i ?files.yaml**
288 |
289 | Example:
290 |
291 | - batch file which gets the data from multiple sources **myFiles.bat**
292 |
293 | ```sh
294 | curl -s https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml > ecbRates.xml
295 | curl -s https://goweather.herokuapp.com/weather/prague > pragueWeather.json
296 | ```
297 |
298 | - Files description **myFiles.yaml**
299 |
300 | ```yaml
301 | - file: ./ecbRates.xml # file path
302 | format: xml # File format
303 | label: RATES # Label which will be used in the template {{ .RATES }}
304 | - file: ./pragueWeather.json
305 | format: json
306 | label: WEATHER
307 | ```
308 |
309 | - Template file **myTemplate.tmpl** which will generate simple HTML page with data
310 |
311 | ```html
312 |
313 |
314 | Weather in Prague
315 | Temperatre: {{.WEATHER.temperature}}
316 | Wind: {{.WEATHER.wind}}
317 | ECB Exchange rates from: {{dateFormat (index .RATES.Envelope.Cube.Cube "-time") "2006-01-02" "02.01.2006" }}
318 |
319 | | currency | rate |
320 | {{- range .RATES.Envelope.Cube.Cube.Cube }}
321 |
|---|
| {{index . "-currency" }} | {{index . "-rate" }} |
322 | {{- end}}
323 |
324 |
325 |
326 |
327 |
330 | ```
331 |
332 | - Finally run bafi
333 |
334 | ```sh
335 | bafi.exe -t myTemplate.tmpl -i ?myFiles.yaml -o output.html
336 | ```
337 |
--------------------------------------------------------------------------------
/docs/img/bafi.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
61 |
--------------------------------------------------------------------------------
/docs/img/bafiIcon.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
75 |
--------------------------------------------------------------------------------
/docs/img/calc.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mmalcek/bafi/0aa8972647b5af9854f923097f8575f541a760f2/docs/img/calc.jpg
--------------------------------------------------------------------------------
/docs/img/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mmalcek/bafi/0aa8972647b5af9854f923097f8575f541a760f2/docs/img/favicon.ico
--------------------------------------------------------------------------------
/docs/img/scheme.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
209 |
--------------------------------------------------------------------------------
/docs/js/bafi.wasm:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mmalcek/bafi/0aa8972647b5af9854f923097f8575f541a760f2/docs/js/bafi.wasm
--------------------------------------------------------------------------------
/docs/js/wasm_exec.js:
--------------------------------------------------------------------------------
1 | // Copyright 2018 The Go Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | "use strict";
6 |
7 | (() => {
8 | const enosys = () => {
9 | const err = new Error("not implemented");
10 | err.code = "ENOSYS";
11 | return err;
12 | };
13 |
14 | if (!globalThis.fs) {
15 | let outputBuf = "";
16 | globalThis.fs = {
17 | constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused
18 | writeSync(fd, buf) {
19 | outputBuf += decoder.decode(buf);
20 | const nl = outputBuf.lastIndexOf("\n");
21 | if (nl != -1) {
22 | console.log(outputBuf.substring(0, nl));
23 | outputBuf = outputBuf.substring(nl + 1);
24 | }
25 | return buf.length;
26 | },
27 | write(fd, buf, offset, length, position, callback) {
28 | if (offset !== 0 || length !== buf.length || position !== null) {
29 | callback(enosys());
30 | return;
31 | }
32 | const n = this.writeSync(fd, buf);
33 | callback(null, n);
34 | },
35 | chmod(path, mode, callback) { callback(enosys()); },
36 | chown(path, uid, gid, callback) { callback(enosys()); },
37 | close(fd, callback) { callback(enosys()); },
38 | fchmod(fd, mode, callback) { callback(enosys()); },
39 | fchown(fd, uid, gid, callback) { callback(enosys()); },
40 | fstat(fd, callback) { callback(enosys()); },
41 | fsync(fd, callback) { callback(null); },
42 | ftruncate(fd, length, callback) { callback(enosys()); },
43 | lchown(path, uid, gid, callback) { callback(enosys()); },
44 | link(path, link, callback) { callback(enosys()); },
45 | lstat(path, callback) { callback(enosys()); },
46 | mkdir(path, perm, callback) { callback(enosys()); },
47 | open(path, flags, mode, callback) { callback(enosys()); },
48 | read(fd, buffer, offset, length, position, callback) { callback(enosys()); },
49 | readdir(path, callback) { callback(enosys()); },
50 | readlink(path, callback) { callback(enosys()); },
51 | rename(from, to, callback) { callback(enosys()); },
52 | rmdir(path, callback) { callback(enosys()); },
53 | stat(path, callback) { callback(enosys()); },
54 | symlink(path, link, callback) { callback(enosys()); },
55 | truncate(path, length, callback) { callback(enosys()); },
56 | unlink(path, callback) { callback(enosys()); },
57 | utimes(path, atime, mtime, callback) { callback(enosys()); },
58 | };
59 | }
60 |
61 | if (!globalThis.process) {
62 | globalThis.process = {
63 | getuid() { return -1; },
64 | getgid() { return -1; },
65 | geteuid() { return -1; },
66 | getegid() { return -1; },
67 | getgroups() { throw enosys(); },
68 | pid: -1,
69 | ppid: -1,
70 | umask() { throw enosys(); },
71 | cwd() { throw enosys(); },
72 | chdir() { throw enosys(); },
73 | }
74 | }
75 |
76 | if (!globalThis.crypto) {
77 | throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)");
78 | }
79 |
80 | if (!globalThis.performance) {
81 | throw new Error("globalThis.performance is not available, polyfill required (performance.now only)");
82 | }
83 |
84 | if (!globalThis.TextEncoder) {
85 | throw new Error("globalThis.TextEncoder is not available, polyfill required");
86 | }
87 |
88 | if (!globalThis.TextDecoder) {
89 | throw new Error("globalThis.TextDecoder is not available, polyfill required");
90 | }
91 |
92 | const encoder = new TextEncoder("utf-8");
93 | const decoder = new TextDecoder("utf-8");
94 |
95 | globalThis.Go = class {
96 | constructor() {
97 | this.argv = ["js"];
98 | this.env = {};
99 | this.exit = (code) => {
100 | if (code !== 0) {
101 | console.warn("exit code:", code);
102 | }
103 | };
104 | this._exitPromise = new Promise((resolve) => {
105 | this._resolveExitPromise = resolve;
106 | });
107 | this._pendingEvent = null;
108 | this._scheduledTimeouts = new Map();
109 | this._nextCallbackTimeoutID = 1;
110 |
111 | const setInt64 = (addr, v) => {
112 | this.mem.setUint32(addr + 0, v, true);
113 | this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true);
114 | }
115 |
116 | const setInt32 = (addr, v) => {
117 | this.mem.setUint32(addr + 0, v, true);
118 | }
119 |
120 | const getInt64 = (addr) => {
121 | const low = this.mem.getUint32(addr + 0, true);
122 | const high = this.mem.getInt32(addr + 4, true);
123 | return low + high * 4294967296;
124 | }
125 |
126 | const loadValue = (addr) => {
127 | const f = this.mem.getFloat64(addr, true);
128 | if (f === 0) {
129 | return undefined;
130 | }
131 | if (!isNaN(f)) {
132 | return f;
133 | }
134 |
135 | const id = this.mem.getUint32(addr, true);
136 | return this._values[id];
137 | }
138 |
139 | const storeValue = (addr, v) => {
140 | const nanHead = 0x7FF80000;
141 |
142 | if (typeof v === "number" && v !== 0) {
143 | if (isNaN(v)) {
144 | this.mem.setUint32(addr + 4, nanHead, true);
145 | this.mem.setUint32(addr, 0, true);
146 | return;
147 | }
148 | this.mem.setFloat64(addr, v, true);
149 | return;
150 | }
151 |
152 | if (v === undefined) {
153 | this.mem.setFloat64(addr, 0, true);
154 | return;
155 | }
156 |
157 | let id = this._ids.get(v);
158 | if (id === undefined) {
159 | id = this._idPool.pop();
160 | if (id === undefined) {
161 | id = this._values.length;
162 | }
163 | this._values[id] = v;
164 | this._goRefCounts[id] = 0;
165 | this._ids.set(v, id);
166 | }
167 | this._goRefCounts[id]++;
168 | let typeFlag = 0;
169 | switch (typeof v) {
170 | case "object":
171 | if (v !== null) {
172 | typeFlag = 1;
173 | }
174 | break;
175 | case "string":
176 | typeFlag = 2;
177 | break;
178 | case "symbol":
179 | typeFlag = 3;
180 | break;
181 | case "function":
182 | typeFlag = 4;
183 | break;
184 | }
185 | this.mem.setUint32(addr + 4, nanHead | typeFlag, true);
186 | this.mem.setUint32(addr, id, true);
187 | }
188 |
189 | const loadSlice = (addr) => {
190 | const array = getInt64(addr + 0);
191 | const len = getInt64(addr + 8);
192 | return new Uint8Array(this._inst.exports.mem.buffer, array, len);
193 | }
194 |
195 | const loadSliceOfValues = (addr) => {
196 | const array = getInt64(addr + 0);
197 | const len = getInt64(addr + 8);
198 | const a = new Array(len);
199 | for (let i = 0; i < len; i++) {
200 | a[i] = loadValue(array + i * 8);
201 | }
202 | return a;
203 | }
204 |
205 | const loadString = (addr) => {
206 | const saddr = getInt64(addr + 0);
207 | const len = getInt64(addr + 8);
208 | return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len));
209 | }
210 |
211 | const timeOrigin = Date.now() - performance.now();
212 | this.importObject = {
213 | _gotest: {
214 | add: (a, b) => a + b,
215 | },
216 | gojs: {
217 | // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters)
218 | // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported
219 | // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function).
220 | // This changes the SP, thus we have to update the SP used by the imported function.
221 |
222 | // func wasmExit(code int32)
223 | "runtime.wasmExit": (sp) => {
224 | sp >>>= 0;
225 | const code = this.mem.getInt32(sp + 8, true);
226 | this.exited = true;
227 | delete this._inst;
228 | delete this._values;
229 | delete this._goRefCounts;
230 | delete this._ids;
231 | delete this._idPool;
232 | this.exit(code);
233 | },
234 |
235 | // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32)
236 | "runtime.wasmWrite": (sp) => {
237 | sp >>>= 0;
238 | const fd = getInt64(sp + 8);
239 | const p = getInt64(sp + 16);
240 | const n = this.mem.getInt32(sp + 24, true);
241 | fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n));
242 | },
243 |
244 | // func resetMemoryDataView()
245 | "runtime.resetMemoryDataView": (sp) => {
246 | sp >>>= 0;
247 | this.mem = new DataView(this._inst.exports.mem.buffer);
248 | },
249 |
250 | // func nanotime1() int64
251 | "runtime.nanotime1": (sp) => {
252 | sp >>>= 0;
253 | setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
254 | },
255 |
256 | // func walltime() (sec int64, nsec int32)
257 | "runtime.walltime": (sp) => {
258 | sp >>>= 0;
259 | const msec = (new Date).getTime();
260 | setInt64(sp + 8, msec / 1000);
261 | this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true);
262 | },
263 |
264 | // func scheduleTimeoutEvent(delay int64) int32
265 | "runtime.scheduleTimeoutEvent": (sp) => {
266 | sp >>>= 0;
267 | const id = this._nextCallbackTimeoutID;
268 | this._nextCallbackTimeoutID++;
269 | this._scheduledTimeouts.set(id, setTimeout(
270 | () => {
271 | this._resume();
272 | while (this._scheduledTimeouts.has(id)) {
273 | // for some reason Go failed to register the timeout event, log and try again
274 | // (temporary workaround for https://github.com/golang/go/issues/28975)
275 | console.warn("scheduleTimeoutEvent: missed timeout event");
276 | this._resume();
277 | }
278 | },
279 | getInt64(sp + 8),
280 | ));
281 | this.mem.setInt32(sp + 16, id, true);
282 | },
283 |
284 | // func clearTimeoutEvent(id int32)
285 | "runtime.clearTimeoutEvent": (sp) => {
286 | sp >>>= 0;
287 | const id = this.mem.getInt32(sp + 8, true);
288 | clearTimeout(this._scheduledTimeouts.get(id));
289 | this._scheduledTimeouts.delete(id);
290 | },
291 |
292 | // func getRandomData(r []byte)
293 | "runtime.getRandomData": (sp) => {
294 | sp >>>= 0;
295 | crypto.getRandomValues(loadSlice(sp + 8));
296 | },
297 |
298 | // func finalizeRef(v ref)
299 | "syscall/js.finalizeRef": (sp) => {
300 | sp >>>= 0;
301 | const id = this.mem.getUint32(sp + 8, true);
302 | this._goRefCounts[id]--;
303 | if (this._goRefCounts[id] === 0) {
304 | const v = this._values[id];
305 | this._values[id] = null;
306 | this._ids.delete(v);
307 | this._idPool.push(id);
308 | }
309 | },
310 |
311 | // func stringVal(value string) ref
312 | "syscall/js.stringVal": (sp) => {
313 | sp >>>= 0;
314 | storeValue(sp + 24, loadString(sp + 8));
315 | },
316 |
317 | // func valueGet(v ref, p string) ref
318 | "syscall/js.valueGet": (sp) => {
319 | sp >>>= 0;
320 | const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16));
321 | sp = this._inst.exports.getsp() >>> 0; // see comment above
322 | storeValue(sp + 32, result);
323 | },
324 |
325 | // func valueSet(v ref, p string, x ref)
326 | "syscall/js.valueSet": (sp) => {
327 | sp >>>= 0;
328 | Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32));
329 | },
330 |
331 | // func valueDelete(v ref, p string)
332 | "syscall/js.valueDelete": (sp) => {
333 | sp >>>= 0;
334 | Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16));
335 | },
336 |
337 | // func valueIndex(v ref, i int) ref
338 | "syscall/js.valueIndex": (sp) => {
339 | sp >>>= 0;
340 | storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16)));
341 | },
342 |
343 | // valueSetIndex(v ref, i int, x ref)
344 | "syscall/js.valueSetIndex": (sp) => {
345 | sp >>>= 0;
346 | Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24));
347 | },
348 |
349 | // func valueCall(v ref, m string, args []ref) (ref, bool)
350 | "syscall/js.valueCall": (sp) => {
351 | sp >>>= 0;
352 | try {
353 | const v = loadValue(sp + 8);
354 | const m = Reflect.get(v, loadString(sp + 16));
355 | const args = loadSliceOfValues(sp + 32);
356 | const result = Reflect.apply(m, v, args);
357 | sp = this._inst.exports.getsp() >>> 0; // see comment above
358 | storeValue(sp + 56, result);
359 | this.mem.setUint8(sp + 64, 1);
360 | } catch (err) {
361 | sp = this._inst.exports.getsp() >>> 0; // see comment above
362 | storeValue(sp + 56, err);
363 | this.mem.setUint8(sp + 64, 0);
364 | }
365 | },
366 |
367 | // func valueInvoke(v ref, args []ref) (ref, bool)
368 | "syscall/js.valueInvoke": (sp) => {
369 | sp >>>= 0;
370 | try {
371 | const v = loadValue(sp + 8);
372 | const args = loadSliceOfValues(sp + 16);
373 | const result = Reflect.apply(v, undefined, args);
374 | sp = this._inst.exports.getsp() >>> 0; // see comment above
375 | storeValue(sp + 40, result);
376 | this.mem.setUint8(sp + 48, 1);
377 | } catch (err) {
378 | sp = this._inst.exports.getsp() >>> 0; // see comment above
379 | storeValue(sp + 40, err);
380 | this.mem.setUint8(sp + 48, 0);
381 | }
382 | },
383 |
384 | // func valueNew(v ref, args []ref) (ref, bool)
385 | "syscall/js.valueNew": (sp) => {
386 | sp >>>= 0;
387 | try {
388 | const v = loadValue(sp + 8);
389 | const args = loadSliceOfValues(sp + 16);
390 | const result = Reflect.construct(v, args);
391 | sp = this._inst.exports.getsp() >>> 0; // see comment above
392 | storeValue(sp + 40, result);
393 | this.mem.setUint8(sp + 48, 1);
394 | } catch (err) {
395 | sp = this._inst.exports.getsp() >>> 0; // see comment above
396 | storeValue(sp + 40, err);
397 | this.mem.setUint8(sp + 48, 0);
398 | }
399 | },
400 |
401 | // func valueLength(v ref) int
402 | "syscall/js.valueLength": (sp) => {
403 | sp >>>= 0;
404 | setInt64(sp + 16, parseInt(loadValue(sp + 8).length));
405 | },
406 |
407 | // valuePrepareString(v ref) (ref, int)
408 | "syscall/js.valuePrepareString": (sp) => {
409 | sp >>>= 0;
410 | const str = encoder.encode(String(loadValue(sp + 8)));
411 | storeValue(sp + 16, str);
412 | setInt64(sp + 24, str.length);
413 | },
414 |
415 | // valueLoadString(v ref, b []byte)
416 | "syscall/js.valueLoadString": (sp) => {
417 | sp >>>= 0;
418 | const str = loadValue(sp + 8);
419 | loadSlice(sp + 16).set(str);
420 | },
421 |
422 | // func valueInstanceOf(v ref, t ref) bool
423 | "syscall/js.valueInstanceOf": (sp) => {
424 | sp >>>= 0;
425 | this.mem.setUint8(sp + 24, (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0);
426 | },
427 |
428 | // func copyBytesToGo(dst []byte, src ref) (int, bool)
429 | "syscall/js.copyBytesToGo": (sp) => {
430 | sp >>>= 0;
431 | const dst = loadSlice(sp + 8);
432 | const src = loadValue(sp + 32);
433 | if (!(src instanceof Uint8Array || src instanceof Uint8ClampedArray)) {
434 | this.mem.setUint8(sp + 48, 0);
435 | return;
436 | }
437 | const toCopy = src.subarray(0, dst.length);
438 | dst.set(toCopy);
439 | setInt64(sp + 40, toCopy.length);
440 | this.mem.setUint8(sp + 48, 1);
441 | },
442 |
443 | // func copyBytesToJS(dst ref, src []byte) (int, bool)
444 | "syscall/js.copyBytesToJS": (sp) => {
445 | sp >>>= 0;
446 | const dst = loadValue(sp + 8);
447 | const src = loadSlice(sp + 16);
448 | if (!(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)) {
449 | this.mem.setUint8(sp + 48, 0);
450 | return;
451 | }
452 | const toCopy = src.subarray(0, dst.length);
453 | dst.set(toCopy);
454 | setInt64(sp + 40, toCopy.length);
455 | this.mem.setUint8(sp + 48, 1);
456 | },
457 |
458 | "debug": (value) => {
459 | console.log(value);
460 | },
461 | }
462 | };
463 | }
464 |
465 | async run(instance) {
466 | if (!(instance instanceof WebAssembly.Instance)) {
467 | throw new Error("Go.run: WebAssembly.Instance expected");
468 | }
469 | this._inst = instance;
470 | this.mem = new DataView(this._inst.exports.mem.buffer);
471 | this._values = [ // JS values that Go currently has references to, indexed by reference id
472 | NaN,
473 | 0,
474 | null,
475 | true,
476 | false,
477 | globalThis,
478 | this,
479 | ];
480 | this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id
481 | this._ids = new Map([ // mapping from JS values to reference ids
482 | [0, 1],
483 | [null, 2],
484 | [true, 3],
485 | [false, 4],
486 | [globalThis, 5],
487 | [this, 6],
488 | ]);
489 | this._idPool = []; // unused ids that have been garbage collected
490 | this.exited = false; // whether the Go program has exited
491 |
492 | // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory.
493 | let offset = 4096;
494 |
495 | const strPtr = (str) => {
496 | const ptr = offset;
497 | const bytes = encoder.encode(str + "\0");
498 | new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes);
499 | offset += bytes.length;
500 | if (offset % 8 !== 0) {
501 | offset += 8 - (offset % 8);
502 | }
503 | return ptr;
504 | };
505 |
506 | const argc = this.argv.length;
507 |
508 | const argvPtrs = [];
509 | this.argv.forEach((arg) => {
510 | argvPtrs.push(strPtr(arg));
511 | });
512 | argvPtrs.push(0);
513 |
514 | const keys = Object.keys(this.env).sort();
515 | keys.forEach((key) => {
516 | argvPtrs.push(strPtr(`${key}=${this.env[key]}`));
517 | });
518 | argvPtrs.push(0);
519 |
520 | const argv = offset;
521 | argvPtrs.forEach((ptr) => {
522 | this.mem.setUint32(offset, ptr, true);
523 | this.mem.setUint32(offset + 4, 0, true);
524 | offset += 8;
525 | });
526 |
527 | // The linker guarantees global data starts from at least wasmMinDataAddr.
528 | // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr.
529 | const wasmMinDataAddr = 4096 + 8192;
530 | if (offset >= wasmMinDataAddr) {
531 | throw new Error("total length of command line and environment variables exceeds limit");
532 | }
533 |
534 | this._inst.exports.run(argc, argv);
535 | if (this.exited) {
536 | this._resolveExitPromise();
537 | }
538 | await this._exitPromise;
539 | }
540 |
541 | _resume() {
542 | if (this.exited) {
543 | throw new Error("Go program has already exited");
544 | }
545 | this._inst.exports.resume();
546 | if (this.exited) {
547 | this._resolveExitPromise();
548 | }
549 | }
550 |
551 | _makeFuncWrapper(id) {
552 | const go = this;
553 | return function () {
554 | const event = { id: id, this: this, args: arguments };
555 | go._pendingEvent = event;
556 | go._resume();
557 | return event.result;
558 | };
559 | }
560 | }
561 | })();
562 |
--------------------------------------------------------------------------------
/filesTest.yaml:
--------------------------------------------------------------------------------
1 | # Example array of files:
2 | - file: ./testdata.xml # file path
3 | format: xml # File format
4 | label: filesTest # Label which will be used in the template {{ .filesTest }}
5 |
--------------------------------------------------------------------------------
/functions.go:
--------------------------------------------------------------------------------
1 | // Inspired by https://github.com/Masterminds/sprig
2 | package main
3 |
4 | import (
5 | "encoding/base32"
6 | "encoding/base64"
7 | "encoding/json"
8 | "fmt"
9 | "math"
10 | "math/rand"
11 | "reflect"
12 | "regexp"
13 | "strconv"
14 | "strings"
15 | "text/template"
16 | "time"
17 |
18 | "github.com/clbanning/mxj/v2"
19 | "github.com/google/uuid"
20 | "github.com/shopspring/decimal"
21 | "github.com/spf13/cast"
22 | lua "github.com/yuin/gopher-lua"
23 | "go.mongodb.org/mongo-driver/bson"
24 | "gopkg.in/yaml.v3"
25 | )
26 |
27 | // templateFunctions extends template functions
28 | func templateFunctions() template.FuncMap {
29 | return template.FuncMap{
30 | "add": add,
31 | "add1": add1,
32 | "sub": sub,
33 | "div": div,
34 | "mod": mod,
35 | "mul": mul,
36 | "addf": addf,
37 | "add1f": add1f,
38 | "subf": subf,
39 | "divf": divf,
40 | "mulf": mulf,
41 | "randInt": randInt,
42 | "round": round,
43 | "max": max,
44 | "min": min,
45 | "maxf": maxf,
46 | "minf": minf,
47 | "dateFormat": dateFormat,
48 | "dateFormatTZ": dateFormatTZ,
49 | "dateToInt": dateToInt,
50 | "intToDate": intToDate,
51 | "now": now,
52 | "b64enc": base64encode,
53 | "b64dec": base64decode,
54 | "b32enc": base32encode,
55 | "b32dec": base32decode,
56 | "uuid": newUUID,
57 | "replaceAll": replaceAll,
58 | "replaceAllRegex": replaceAllRegex,
59 | "regexMatch": regexMatch,
60 | "contains": contains,
61 | "upper": upper,
62 | "lower": lower,
63 | "addSubstring": addSubstring,
64 | "trim": trim,
65 | "trimAll": trimAll,
66 | "trimSuffix": trimSuffix,
67 | "indexOf": indexOf,
68 | "trimPrefix": trimPrefix,
69 | "atoi": atoi,
70 | "toBool": toBool,
71 | "toString": toString,
72 | "toInt": toInt,
73 | "toInt64": toInt64,
74 | "toFloat64": toFloat64,
75 | "toDecimal": toDecimal,
76 | "toDecimalString": toDecimalString,
77 | "toJSON": toJSON,
78 | "toBSON": toBSON,
79 | "toYAML": toYAML,
80 | "toXML": toXML,
81 | "isBool": isBool,
82 | "isInt": isInt,
83 | "isFloat64": isFloat64,
84 | "isString": isString,
85 | "isMap": isMap,
86 | "isArray": isArray,
87 | "mustArray": mustArray,
88 | "mapJSON": mapJSON,
89 | "lua": luaF,
90 | }
91 | }
92 |
93 | // add count
94 | func add(a ...interface{}) (r int64) {
95 | for i := range a {
96 | r += toInt64(a[i])
97 | }
98 | return
99 | }
100 |
101 | // add1 input+1
102 | func add1(a interface{}) int64 { return toInt64(a) + 1 }
103 |
104 | // sub substitute
105 | func sub(a, b interface{}) int64 { return toInt64(a) - toInt64(b) }
106 |
107 | // div divide
108 | func div(a, b interface{}) int64 { return toInt64(a) / toInt64(b) }
109 |
110 | // mod modulo
111 | func mod(a, b interface{}) int64 { return toInt64(a) % toInt64(b) }
112 |
113 | // mul multiply
114 | func mul(a ...interface{}) (r int64) {
115 | r = 1
116 | for i := range a {
117 | r = r * toInt64(a[i])
118 | }
119 | return r
120 | }
121 |
122 | // addf count float
123 | func addf(i ...interface{}) float64 {
124 | a := interface{}(float64(0))
125 | return execDecimalOp(a, i, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Add(d2) })
126 | }
127 |
128 | // add1f inputFloat+1
129 | func add1f(i interface{}) float64 {
130 | return execDecimalOp(i, []interface{}{1}, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Add(d2) })
131 | }
132 |
133 | // subf substitute float
134 | func subf(a interface{}, v ...interface{}) float64 {
135 | return execDecimalOp(a, v, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Sub(d2) })
136 | }
137 |
138 | // divide float
139 | func divf(a interface{}, v ...interface{}) float64 {
140 | return execDecimalOp(a, v, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Div(d2) })
141 | }
142 |
143 | // mulf multiply float
144 | func mulf(a interface{}, v ...interface{}) float64 {
145 | return execDecimalOp(a, v, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Mul(d2) })
146 | }
147 |
148 | // randInt returns random integer in defined range {{randInt min max}} e.g. {{randInt 1 10}}
149 | func randInt(min, max int) int { return rand.Intn(max-min) + min }
150 |
151 | // round float {{round .val 2}} -> 2 decimals or {{round .val 1 0.4}} 0.4 round point
152 | func round(a interface{}, p int, rOpt ...float64) float64 {
153 | roundOn := .5
154 | if len(rOpt) > 0 {
155 | roundOn = rOpt[0]
156 | }
157 | val := toFloat64(a)
158 | places := toFloat64(p)
159 |
160 | var round float64
161 | pow := math.Pow(10, places)
162 | digit := pow * val
163 | _, div := math.Modf(digit)
164 | if div >= roundOn {
165 | round = math.Ceil(digit)
166 | } else {
167 | round = math.Floor(digit)
168 | }
169 | return round / pow
170 | }
171 |
172 | // max return highest from numbers {{max .v1 .v2 .v3}}
173 | func max(a interface{}, numbers ...interface{}) int64 {
174 | aa := toInt64(a)
175 | for i := range numbers {
176 | bb := toInt64(numbers[i])
177 | if bb > aa {
178 | aa = bb
179 | }
180 | }
181 | return aa
182 | }
183 |
184 | // min return lowest from numbers {{min .v1 .v2 .v3}}
185 | func min(a interface{}, numbers ...interface{}) int64 {
186 | aa := toInt64(a)
187 | for i := range numbers {
188 | bb := toInt64(numbers[i])
189 | if bb < aa {
190 | aa = bb
191 | }
192 | }
193 | return aa
194 | }
195 |
196 | // maxf return highest from float numbers {{maxf .v1 .v2 .v3}}
197 | func maxf(a interface{}, numbers ...interface{}) float64 {
198 | aa := toFloat64(a)
199 | for i := range numbers {
200 | bb := toFloat64(numbers[i])
201 | aa = math.Max(aa, bb)
202 | }
203 | return aa
204 | }
205 |
206 | // minf return lowest from float numbers {{minf .v1 .v2 .v3}}
207 | func minf(a interface{}, numbers ...interface{}) float64 {
208 | aa := toFloat64(a)
209 | for i := range numbers {
210 | bb := toFloat64(numbers[i])
211 | aa = math.Min(aa, bb)
212 | }
213 | return aa
214 | }
215 |
216 | // dateFormat convert date format {{dateFormat "string", "inputPattern", "outputPattern"}} e.g. {{dateFormat "15.03.2021", "02.01.2006", "01022006"}}
217 | func dateFormat(date string, inputFormat string, outputFormat string) string {
218 | timeParsed, err := time.Parse(inputFormat, date)
219 | if err != nil {
220 | return date
221 | }
222 | return timeParsed.Format(outputFormat)
223 | }
224 |
225 | func dateFormatTZ(date string, inputFormat string, outputFormat string, timeZone string) string {
226 | location, err := time.LoadLocation(timeZone)
227 | if err != nil {
228 | return "err: unknownTimeZone"
229 | }
230 | timeParsed, err := time.Parse(inputFormat, date)
231 | if err != nil {
232 | return "err: wrongFormatDefinition"
233 | }
234 | timeParsed = timeParsed.In(location)
235 | return timeParsed.Format(outputFormat)
236 | }
237 |
238 | // convert date to unix timestamp
239 | func dateToInt(date string, inputFormat string) int64 {
240 | timeParsed, err := time.Parse(inputFormat, date)
241 | if err != nil {
242 | return 0
243 | }
244 | return timeParsed.Unix()
245 | }
246 |
247 | // convert unix timestamp to date
248 | func intToDate(unixTime interface{}, outputFormat string) string {
249 | return time.Unix(toInt64(unixTime), 0).Format(outputFormat)
250 | }
251 |
252 | // now return current date/time in specified format
253 | func now(format string) string {
254 | return time.Now().Format(format)
255 | }
256 |
257 | // base64encode encode to base64
258 | func base64encode(v string) string {
259 | return base64.StdEncoding.EncodeToString([]byte(v))
260 | }
261 |
262 | // base64decode decode from base64
263 | func base64decode(v string) string {
264 | data, err := base64.StdEncoding.DecodeString(v)
265 | if err != nil {
266 | return err.Error()
267 | }
268 | return string(data)
269 | }
270 |
271 | // base32encode encode to base32
272 | func base32encode(v string) string {
273 | return base32.StdEncoding.EncodeToString([]byte(v))
274 | }
275 |
276 | // base32decode decode from base32
277 | func base32decode(v string) string {
278 | data, err := base32.StdEncoding.DecodeString(v)
279 | if err != nil {
280 | return err.Error()
281 | }
282 | return string(data)
283 | }
284 |
285 | // newUUID returns UUID
286 | func newUUID() string { return uuid.New().String() }
287 |
288 | func replaceAll(old, new, src string) string {
289 | return strings.Replace(src, old, new, -1)
290 | }
291 |
292 | func replaceAllRegex(regex, new, src string) string {
293 | r := regexp.MustCompile(regex)
294 | return r.ReplaceAllString(src, new)
295 | }
296 |
297 | // regexMatch check regex e.g. {{regexMatch "a.b", "aaxbb"}}
298 | func regexMatch(regex string, s string) bool {
299 | match, _ := regexp.MatchString(regex, s)
300 | return match
301 | }
302 |
303 | // contains check if string contains substring e.g. {{contains "aaxbb" "xb"}}
304 | func contains(str string, substr string) bool {
305 | return strings.Contains(str, substr)
306 | }
307 |
308 | // upper string to uppercase
309 | func upper(s string) string {
310 | return strings.ToUpper(s)
311 | }
312 |
313 | // lower string to lowercase
314 | func lower(s string) string {
315 | return strings.ToLower(s)
316 | }
317 |
318 | // addSubstring add substring to string {{addSubstring "abcd", "efg", 2}} -> "abefgcd"
319 | func addSubstring(s string, ss string, pos interface{}) string {
320 | if toInt(pos) >= len(s) || -toInt(pos) >= len(s) {
321 | return "err:substringOutOfRange"
322 | }
323 | switch x := toInt(pos); {
324 | case x == 0:
325 | return s
326 | case x > 0:
327 | return fmt.Sprintf("%s%s%s", s[:len(s)-x], ss, s[len(s)-x:])
328 | case x < 0:
329 | return fmt.Sprintf("%s%s%s", s[:-x], ss, s[-x:])
330 | default:
331 | return "inputError"
332 | }
333 | }
334 |
335 | // trim remove leading and trailing white space
336 | func trim(s string) string {
337 | return strings.TrimSpace(s)
338 | }
339 |
340 | // trimAll remove leading and trailing whitespace
341 | func trimAll(a, b string) string { return strings.Trim(a, b) }
342 |
343 | // {{trimPrefix "!Hello World!" "!"}} - returns "Hello World!"
344 | func trimPrefix(a, b string) string { return strings.TrimPrefix(a, b) }
345 |
346 | // trimSuffix - {{trimSuffix "!Hello World!" "!"}} - returns "!HelloWorld"
347 | func trimSuffix(a, b string) string { return strings.TrimSuffix(a, b) }
348 |
349 | // indexOf {{indexOf "abcd", "bc"}} -> 1
350 | // TODO: add to tests and documentation
351 | func indexOf(a, b string) int {
352 | return strings.Index(a, b)
353 | }
354 |
355 | // atoi {{atoi "42"}} - string to int
356 | func atoi(a string) int { i, _ := strconv.Atoi(a); return i }
357 |
358 | func toBool(v interface{}) bool {
359 | return cast.ToBool(v)
360 | }
361 |
362 | // toInt convert to int
363 | func toInt(v interface{}) int {
364 | return cast.ToInt(v)
365 | }
366 |
367 | func toString(v interface{}) string {
368 | return cast.ToString(v)
369 | }
370 |
371 | // toInt64 converts integer types to 64-bit integers
372 | func toInt64(v interface{}) int64 {
373 | return cast.ToInt64(v)
374 | }
375 |
376 | // toFloat64 converts 64-bit floats
377 | func toFloat64(v interface{}) float64 {
378 | return cast.ToFloat64(v)
379 | }
380 |
381 | // toDecimal input to decimal
382 | func toDecimal(i interface{}) decimal.Decimal {
383 | value, err := convertDecimal(i)
384 | if err != nil {
385 | return decimal.Zero
386 | }
387 | return value
388 | }
389 |
390 | // toDecimalString input to decimal string
391 | func toDecimalString(i interface{}) string {
392 | value, err := convertDecimal(i)
393 | if err != nil {
394 | return fmt.Sprintf("err: %s", err.Error())
395 | }
396 | return value.String()
397 | }
398 |
399 | // toJSON convert to JSON
400 | func toJSON(data interface{}) string {
401 | out, err := json.MarshalIndent(data, "", " ")
402 | if err != nil {
403 | return fmt.Sprintf("err: %s", err.Error())
404 | }
405 | return string(out)
406 | }
407 |
408 | // toBSON convert to BSON
409 | func toBSON(data interface{}) string {
410 | out, err := bson.Marshal(data)
411 | if err != nil {
412 | return fmt.Sprintf("err: %s", err.Error())
413 | }
414 | return string(out)
415 | }
416 |
417 | // toYAML convert to YAML
418 | func toYAML(data interface{}) string {
419 | out, err := yaml.Marshal(data)
420 | if err != nil {
421 | return fmt.Sprintf("err: %s", err.Error())
422 | }
423 | return string(out)
424 | }
425 |
426 | // toXML convert to XML
427 | func toXML(data interface{}) string {
428 | var err error
429 | out := []byte("\r\n")
430 | if reflect.TypeOf(data).Kind() == reflect.Slice {
431 | if reflect.TypeOf(data).Kind() == reflect.Slice {
432 | for i := 0; i < reflect.ValueOf(data).Len(); i++ {
433 | x, err := mxj.AnyXmlIndent(data.([]map[string]interface{})[i], "", " ", "record")
434 | if err != nil {
435 | return fmt.Sprintf("err: %s", err.Error())
436 | }
437 | out = append(out, string(x)+"\r\n"...)
438 | }
439 | out = append(out, string("\r\n")...)
440 | out, err = mxj.BeautifyXml(out, "", " ")
441 | if err != nil {
442 | return fmt.Sprintf("err: %s", err.Error())
443 | }
444 | return string(out)
445 | }
446 |
447 | }
448 | out, err = mxj.AnyXmlIndent(data, "", " ", "doc")
449 | if err != nil {
450 | return fmt.Sprintf("err: %s", err.Error())
451 | }
452 | return string(out)
453 | }
454 |
455 | // isBool check if value is bool
456 | func isBool(i interface{}) bool {
457 | _, ok := i.(bool)
458 | return ok
459 | }
460 |
461 | // isInt check if value is int
462 | func isInt(i interface{}) bool {
463 | _, ok := i.(int)
464 | return ok
465 | }
466 |
467 | // isFloat64 check if value is float64
468 | func isFloat64(i interface{}) bool {
469 | _, ok := i.(float64)
470 | return ok
471 | }
472 |
473 | // isString check if value is string
474 | func isString(v interface{}) bool {
475 | _, ok := v.(string)
476 | return ok
477 | }
478 |
479 | // isMap check if value is map
480 | func isMap(v interface{}) bool {
481 | _, ok := v.(map[string]interface{})
482 | return ok
483 | }
484 |
485 | // isArray check if value is array
486 | func isArray(v interface{}) bool {
487 | _, ok := v.([]interface{})
488 | return ok
489 | }
490 |
491 | // mustArray - convert to array. Useful with XML where single record is not treated as array
492 | func mustArray(v interface{}) []interface{} { // convert to []interface{}
493 | if v == nil {
494 | return nil
495 | }
496 | if a, ok := v.([]interface{}); ok {
497 | return a
498 | }
499 | return []interface{}{v}
500 | }
501 |
502 | // mapJSON string JSON to map[string]interface{} so it can be used in pipline -> template
503 | func mapJSON(input string) map[string]interface{} {
504 | var mapData map[string]interface{}
505 | if err := json.Unmarshal([]byte(input), &mapData); err != nil {
506 | testData := make(map[string]interface{})
507 | testData["error"] = err.Error()
508 | return testData
509 | }
510 | return mapData
511 | }
512 |
513 | // luaF Call LUA function {{lua "functionName" input1 input2 input3 ...}
514 | // 1. Functions must be placed in ./lua/functions, 2. Inputs are passed as stringified json 3. Output of lua function must be string
515 | func luaF(i ...interface{}) string {
516 | if luaData == nil {
517 | return "error: ./lua/functions.lua file missing)"
518 | }
519 | strData, err := json.Marshal(i[1:])
520 | if err != nil {
521 | return fmt.Sprintf("luaInputError: %s\r\n", err.Error())
522 | }
523 | if err := luaData.CallByParam(
524 | lua.P{Fn: luaData.GetGlobal(i[0].(string)), NRet: 1, Protect: true}, lua.LString(string(strData))); err != nil {
525 | return fmt.Sprintf("luaError: %s\r\n", err.Error())
526 | }
527 | if str, ok := luaData.Get(-1).(lua.LString); ok {
528 | luaData.Pop(1)
529 | return str.String()
530 | }
531 | return "luaError: getResult"
532 | }
533 |
534 | // execDecimalOp convert float to decimal
535 | func execDecimalOp(a interface{}, b []interface{}, f func(d1, d2 decimal.Decimal) decimal.Decimal) float64 {
536 | prt := decimal.NewFromFloat(toFloat64(a))
537 | for _, x := range b {
538 | dx := decimal.NewFromFloat(toFloat64(x))
539 | prt = f(prt, dx)
540 | }
541 | rslt, _ := prt.Float64()
542 | return rslt
543 | }
544 |
545 | // convertDecimal converts a number to a decimal.Decimal
546 | func convertDecimal(i interface{}) (decimal.Decimal, error) {
547 | switch v := i.(type) {
548 | case decimal.Decimal:
549 | return v, nil
550 | case float64:
551 | return decimal.NewFromFloat(v), nil
552 | case int:
553 | return decimal.NewFromFloat(float64(v)), nil
554 | case int64:
555 | return decimal.NewFromFloat(float64(v)), nil
556 | case string:
557 | value, err := decimal.NewFromString(v)
558 | if err != nil {
559 | return decimal.Zero, err
560 | }
561 | return value, nil
562 | default:
563 | return decimal.Zero, fmt.Errorf("unsupported type: %T", i)
564 | }
565 | }
566 |
--------------------------------------------------------------------------------
/functions_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "regexp"
7 | "strings"
8 | "testing"
9 | "text/template"
10 | "time"
11 |
12 | "github.com/shopspring/decimal"
13 | )
14 |
15 | func runt(tpl, expect string) error {
16 | return runtv(tpl, expect, map[string]string{})
17 | }
18 |
19 | func runtv(tpl, expect string, vars interface{}) error {
20 | t := template.Must(template.New("test").Funcs(templateFunctions()).Parse(tpl))
21 | var b bytes.Buffer
22 | err := t.Execute(&b, vars)
23 | if err != nil {
24 | return err
25 | }
26 | if expect != b.String() {
27 | return fmt.Errorf("Expected '%s', got '%s'", expect, b.String())
28 | }
29 | return nil
30 | }
31 |
32 | func TestTemplateFunctions(t *testing.T) {
33 | if err := runt(`{{ add "1" 2 }}`, "3"); err != nil {
34 | t.Errorf("templateError: %v", err.Error())
35 | }
36 | }
37 |
38 | func TestAdd(t *testing.T) {
39 | result := add("6", 4)
40 | if result != 10 {
41 | t.Errorf("result: %v", result)
42 | }
43 | }
44 | func TestAdd1(t *testing.T) {
45 | result := add1("6")
46 | if result != 7 {
47 | t.Errorf("result: %v", result)
48 | }
49 | }
50 |
51 | func TestSub(t *testing.T) {
52 | result := sub("6", 2)
53 | if result != 4 {
54 | t.Errorf("result: %v", result)
55 | }
56 | }
57 |
58 | func TestDiv(t *testing.T) {
59 | result := div("6", 2)
60 | if result != 3 {
61 | t.Errorf("result: %v", result)
62 | }
63 | }
64 |
65 | func TestMod(t *testing.T) {
66 | result := mod("6", 5)
67 | if result != 1 {
68 | t.Errorf("result: %v", result)
69 | }
70 | }
71 | func TestMul(t *testing.T) {
72 | result := mul("6", 4)
73 | if result != 24 {
74 | t.Errorf("result: %v", result)
75 | }
76 | }
77 |
78 | func TestAddf(t *testing.T) {
79 | result := addf("6.14", 4.12)
80 | if result != 10.26 {
81 | t.Errorf("result: %v", result)
82 | }
83 | }
84 |
85 | func TestAdd1f(t *testing.T) {
86 | result := add1f("6.14")
87 | if result != 7.14 {
88 | t.Errorf("result: %v", result)
89 | }
90 | }
91 |
92 | func TestSubf(t *testing.T) {
93 | result := subf("6.12487", 2.347511)
94 | if result != 3.777359 {
95 | t.Errorf("result: %v", result)
96 | }
97 | }
98 |
99 | func TestDivf(t *testing.T) {
100 | result := divf("6.12487", 2.347511)
101 | if result != 2.6090910756115733 {
102 | t.Errorf("result: %v", result)
103 | }
104 | }
105 |
106 | func TestMulf(t *testing.T) {
107 | result := mulf("6.12487", 2.347511)
108 | if result != 14.37819969857 {
109 | t.Errorf("result: %v", result)
110 | }
111 | }
112 |
113 | func TestRandInt(t *testing.T) {
114 | result := randInt(50, 55)
115 | if result < 50 || result > 55 {
116 | t.Errorf("result: %v", result)
117 | }
118 | }
119 |
120 | func TestRound(t *testing.T) {
121 | if round("6.32487", 2) != 6.32 {
122 | t.Errorf("result: %v", round("6.32487", 2))
123 | }
124 | if round("6.35", 1, 0.6) != 6.3 {
125 | t.Errorf("result: %v", round("6.35", 1, 0.6))
126 | }
127 | if round("6.35", 1, 0.4) != 6.4 {
128 | t.Errorf("result: %v", round("6.35", 1, 0.4))
129 | }
130 | }
131 |
132 | func TestMax(t *testing.T) {
133 | if max("6", 4, "12", "5") != 12 {
134 | t.Errorf("result: %v", max("6", 4, "12", "5"))
135 | }
136 | }
137 |
138 | func TestMin(t *testing.T) {
139 | if min("6", 4, "12", "5") != 4 {
140 | t.Errorf("result: %v", min("6", 4, "12", "5"))
141 | }
142 | }
143 |
144 | func TestMaxf(t *testing.T) {
145 | if maxf("6.32", 4.15, "12.3128", "5") != 12.3128 {
146 | t.Errorf("result: %v", maxf("6.32", 4.15, "12.3128", "5"))
147 | }
148 | }
149 |
150 | func TestMinf(t *testing.T) {
151 | if minf("6.32", 4.15, "12.3128", "5") != 4.15 {
152 | t.Errorf("result: %v", minf("6.32", 4.15, "12.3128", "5"))
153 | }
154 | }
155 |
156 | func TestDateFormat(t *testing.T) {
157 | if dateFormat("15.03.2021", "02.01.2006", "01022006") != "03152021" {
158 | t.Errorf("result: %s", dateFormat("15.03.2021", "02.01.2006", "01022006"))
159 | }
160 | if dateFormat("Hello", "World", "01022006") != "Hello" {
161 | t.Errorf("result: %s", dateFormat("Hello", "World", "01022006"))
162 | }
163 | }
164 |
165 | func TestDateFormatTZ(t *testing.T) {
166 | if dateFormatTZ("2021-08-26T03:35:00.000+04:00", "2006-01-02T15:04:05.000-07:00", "15:04", "Europe/Prague") != "01:35" {
167 | t.Errorf("result: %s", dateFormatTZ("2021-08-26T03:35:00.000+04:00", "2006-01-02T15:04:05.000-07:00", "15:04", "Europe/Prague"))
168 | }
169 | if dateFormatTZ("Hello", "World", "01022006", "42") != "err: unknownTimeZone" {
170 | t.Errorf("result: %s", dateFormatTZ("Hello", "World", "01022006", "42"))
171 | }
172 | if dateFormatTZ("Hello", "World", "01022006", "Europe/Prague") != "err: wrongFormatDefinition" {
173 | t.Errorf("result: %s", dateFormatTZ("Hello", "World", "01022006", "Europe/Prague"))
174 | }
175 | }
176 |
177 | func TestDateToInt(t *testing.T) {
178 | if dateToInt("15.03.2021", "02.01.2006") != 1615766400 {
179 | t.Errorf("result: %v", dateToInt("15.03.2021", "02.01.2006"))
180 | }
181 | }
182 |
183 | func TestIntToDate(t *testing.T) {
184 | if intToDate(1615766400, "02.01.2006") != "15.03.2021" {
185 | t.Errorf("result: %v", intToDate(1615766400, "02.01.2006"))
186 | }
187 | }
188 |
189 | func TestNow(t *testing.T) {
190 | if now("2006-01-02 15:04") != time.Now().Format("2006-01-02 15:04") {
191 | t.Errorf("result: %s", now("2006-01-02 15:04"))
192 | }
193 | }
194 |
195 | func TestBase64encode(t *testing.T) {
196 | if base64encode("Hello World!") != "SGVsbG8gV29ybGQh" {
197 | t.Errorf("result: %v", base64encode("Hello World!"))
198 | }
199 | }
200 |
201 | func TestBase64decode(t *testing.T) {
202 | if base64decode("SGVsbG8gV29ybGQh") != "Hello World!" {
203 | t.Errorf("result: %v", base64decode("SGVsbG8gV29ybGQh"))
204 | }
205 | if base64decode("Hello") != "illegal base64 data at input byte 4" {
206 | t.Errorf("result: %v", base64decode("Hello"))
207 | }
208 | }
209 |
210 | func TestBase32encode(t *testing.T) {
211 | if base32encode("Hello World!") != "JBSWY3DPEBLW64TMMQQQ====" {
212 | t.Errorf("result: %v", base32encode("Hello World!"))
213 | }
214 | }
215 |
216 | func TestBase32decode(t *testing.T) {
217 | if base32decode("JBSWY3DPEBLW64TMMQQQ====") != "Hello World!" {
218 | t.Errorf("result: %v", base32decode("JBSWY3DPEBLW64TMMQQQ===="))
219 | }
220 | if base32decode("Hello") != "illegal base32 data at input byte 1" {
221 | t.Errorf("result: %v", base32decode("Hello"))
222 | }
223 | }
224 |
225 | func TestRegexMatch(t *testing.T) {
226 | if !regexMatch("a.b", "aaxbb") {
227 | t.Errorf("result: %v", regexMatch(`^a.b$`, "aaxbb"))
228 | }
229 | }
230 |
231 | func TestContains(t *testing.T) {
232 | if contains("aaxbb", "a.b") {
233 | t.Errorf("result: %v", contains("aaxbb", "a.b"))
234 | }
235 | if !contains("aaxbb", "ax") {
236 | t.Errorf("result: %v", contains("aaxbb", "a.b"))
237 | }
238 | }
239 |
240 | func TestReplaceAll(t *testing.T) {
241 | if replaceAll("x", "Z", "aaxbb") != "aaZbb" {
242 | t.Errorf("result: %v", replaceAll("aaxbb", "x", "Z"))
243 | }
244 | }
245 |
246 | func TestReplaceAllRegex(t *testing.T) {
247 | if replaceAllRegex("[a-d]", "Z", "aaxbb") != "ZZxZZ" {
248 | t.Errorf("result: %v", replaceAllRegex("aaxbb", `[a-d]`, "Z"))
249 | }
250 | }
251 |
252 | func TestUUID(t *testing.T) {
253 | testUUID := newUUID()
254 | r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
255 | if !r.MatchString(testUUID) {
256 | t.Errorf("result: %v", testUUID)
257 | }
258 | }
259 |
260 | func TestUpper(t *testing.T) {
261 | if upper("Hello") != "HELLO" {
262 | t.Errorf("result: %v", upper("Hello"))
263 | }
264 | }
265 |
266 | func TestLower(t *testing.T) {
267 | if lower("World") != "world" {
268 | t.Errorf("result: %v", lower("World"))
269 | }
270 | }
271 |
272 | func TestAddSubstring(t *testing.T) {
273 | if addSubstring("Hello!!!", " World", "3") != "Hello World!!!" {
274 | t.Errorf("result: %v", addSubstring("Hello!!!", " World", "3"))
275 | }
276 | if addSubstring("Hello!!!", " World", "-5") != "Hello World!!!" {
277 | t.Errorf("result: %v", addSubstring("Hello!!!", " World", "-5"))
278 | }
279 | if addSubstring("Hello!!!", " World", "0") != "Hello!!!" {
280 | t.Errorf("result: %v", addSubstring("Hello!!!", " World", "0"))
281 | }
282 | if addSubstring("Hello!!!", " World", "15") != "err:substringOutOfRange" {
283 | t.Errorf("result: %v", addSubstring("Hello!!!", " World", "15"))
284 | }
285 | }
286 |
287 | func TestTrim(t *testing.T) {
288 | result := trim("\r\nHello World\r\n")
289 | if result != "Hello World" {
290 | t.Errorf("result: %v", result)
291 | }
292 | }
293 |
294 | func TestTrimAll(t *testing.T) {
295 | result := trimAll("!Hello World!", "!")
296 | if result != "Hello World" {
297 | t.Errorf("result: %v", result)
298 | }
299 | }
300 |
301 | func TestPrefix(t *testing.T) {
302 | result := trimPrefix("!Hello World!", "!")
303 | if result != "Hello World!" {
304 | t.Errorf("result: %v", result)
305 | }
306 | }
307 |
308 | func TestSuffix(t *testing.T) {
309 | result := trimSuffix("!Hello World!", "!")
310 | if result != "!Hello World" {
311 | t.Errorf("result: %v", result)
312 | }
313 | }
314 |
315 | func TestAtoi(t *testing.T) {
316 | result := atoi("42")
317 | if result != 42 {
318 | t.Errorf("result: %v", result)
319 | }
320 | }
321 |
322 | func TestToBool(t *testing.T) {
323 | result := toBool("true")
324 | if !result {
325 | t.Errorf("result: %v", result)
326 | }
327 | result = toBool("false")
328 | if result {
329 | t.Errorf("result: %v", result)
330 | }
331 | }
332 |
333 | func TestToString(t *testing.T) {
334 | result := toString(42)
335 | if result != "42" {
336 | t.Errorf("result: %v", result)
337 | }
338 | }
339 |
340 | func TestToInt(t *testing.T) {
341 | if toInt("42") != 42 {
342 | t.Errorf("result: %d", toInt("42"))
343 | }
344 | }
345 |
346 | func TestToInt64(t *testing.T) {
347 | if toInt64("42") != 42 {
348 | t.Errorf("result: %d", toInt64("42"))
349 | }
350 | }
351 |
352 | func TestToFloat64(t *testing.T) {
353 | if toFloat64("1234567.151234") != 1234567.151234 {
354 | t.Errorf("result: %v", toFloat64("1234567.151234"))
355 | }
356 | }
357 |
358 | func TestToDecimal(t *testing.T) {
359 | x, _ := decimal.NewFromString("1234567.151234")
360 | if !x.Equal(toDecimal("1234567.151234")) {
361 | t.Errorf("result: %v", toDecimal("1234567.151234"))
362 | }
363 | x, _ = decimal.NewFromString("0")
364 | if !x.Equal(toDecimal("1234567.151234a")) {
365 | t.Errorf("result: %v", toDecimal("1234567.151234a"))
366 | }
367 | }
368 |
369 | func TestToDecimalString(t *testing.T) {
370 | if toDecimalString("1234567.151234a") != "err: can't convert 1234567.151234a to decimal" {
371 | t.Errorf("result: %v", toDecimalString("1234567.151234a"))
372 | }
373 | if toDecimalString("1234567.151234") != "1234567.151234" {
374 | t.Errorf("result: %v", toDecimalString("1234567.151234"))
375 | }
376 | }
377 |
378 | func TestLuaF(t *testing.T) {
379 | if luaF("sum", "5", "5") != "10" {
380 | t.Errorf("result: %s", luaF("sum", "5", "5"))
381 | }
382 | if !strings.Contains(luaF("Unknown", "5", "5"), `attempt to call a non-function object`) {
383 | t.Errorf("result: %s", luaF("Unknown", "5", "5"))
384 | }
385 | testData := make(map[string]interface{})
386 | testData["Hello"] = make(chan int)
387 | if !strings.Contains(luaF("sum", testData), "luaInputError: json: unsupported type: chan int") {
388 | t.Errorf("result: %v", luaF("sum", testData))
389 | }
390 | }
391 |
392 | func TestToJSON(t *testing.T) {
393 | testData := make(map[string]interface{})
394 | testData["Hello"] = "World"
395 | result := toJSON(testData)
396 | if !strings.Contains(result, `"Hello": "World"`) {
397 | t.Errorf("result: %v", result)
398 | }
399 | testData["Hello"] = make(chan int)
400 | result = toJSON(testData)
401 | if result != "err: json: unsupported type: chan int" {
402 | t.Errorf("result: %v", result)
403 | }
404 | }
405 |
406 | func TestToBSON(t *testing.T) {
407 | testData := make(map[string]interface{})
408 | testData["h"] = "w"
409 | result := toBSON(testData)
410 | if result != string([]byte{14, 0, 0, 0, 2, 104, 0, 2, 0, 0, 0, 119, 0, 0}) {
411 | t.Errorf("result: %v", []byte(result))
412 | }
413 | testData["Hello"] = make(chan int)
414 | result = toBSON(testData)
415 | if result != "err: no encoder found for chan int" {
416 | t.Errorf("result: %v", result)
417 | }
418 | }
419 |
420 | func TestToYAML(t *testing.T) {
421 | testData := make(map[string]interface{})
422 | testData["Hello"] = "World"
423 | result := toYAML(testData)
424 | if result != `Hello: World
425 | ` {
426 | t.Errorf("result: %v", result)
427 | }
428 | }
429 |
430 | func TestToXML(t *testing.T) {
431 | testData := make(map[string]interface{})
432 | testData["Hello"] = "World"
433 | result := toXML(testData)
434 | if !strings.Contains(result, "World") {
435 | t.Errorf("result: %v", result)
436 | }
437 | testData2 := make([]map[string]interface{}, 1)
438 | testData2[0] = make(map[string]interface{})
439 | testData2[0]["Hello"] = "World"
440 | result = toXML(testData2)
441 | if !strings.Contains(result, "World") {
442 | t.Errorf("result: %v", result)
443 | }
444 |
445 | }
446 |
447 | func TestIsBool(t *testing.T) {
448 | if !isBool(true) {
449 | t.Errorf("result: %v", true)
450 | }
451 | if isBool("false") {
452 | t.Errorf("result: %v", false)
453 | }
454 | }
455 |
456 | func TestIsInt(t *testing.T) {
457 | if !isInt(42) {
458 | t.Errorf("result: %v", true)
459 | }
460 | if isInt("42") {
461 | t.Errorf("result: %v", false)
462 | }
463 | }
464 |
465 | func TestIsFloat64(t *testing.T) {
466 | if !isFloat64(42.0) {
467 | t.Errorf("result: %v", true)
468 | }
469 | if isFloat64("42.0") {
470 | t.Errorf("result: %v", false)
471 | }
472 | }
473 |
474 | func TestIsString(t *testing.T) {
475 | if !isString("Hello") {
476 | t.Errorf("result: %v", true)
477 | }
478 | if isString(42) {
479 | t.Errorf("result: %v", false)
480 | }
481 | }
482 |
483 | func TestIsMap(t *testing.T) {
484 | if !isMap(map[string]interface{}{"Hello": "World"}) {
485 | t.Errorf("result: %v", true)
486 | }
487 | if isMap(42) {
488 | t.Errorf("result: %v", false)
489 | }
490 | }
491 |
492 | func TestIsArray(t *testing.T) {
493 | if !isArray([]interface{}{"Hello", "World"}) {
494 | t.Errorf("result: %v", true)
495 | }
496 | if isArray(42) {
497 | t.Errorf("result: %v", false)
498 | }
499 | }
500 |
501 | func TestMustArray(t *testing.T) {
502 | if !isArray(mustArray(nil)) {
503 | t.Errorf("result: %v", true)
504 | }
505 | if !isArray(mustArray([]interface{}{"Hello", "World"})) {
506 | t.Errorf("result: %v", true)
507 | }
508 | if !isArray(mustArray("Hello")) {
509 | t.Errorf("result: %v", false)
510 | }
511 | }
512 |
513 | func TestMapJSON(t *testing.T) {
514 | testData := "{\"Hello\":\"World\"}"
515 | result := mapJSON(testData)
516 | if result["Hello"] != "World" {
517 | t.Errorf("result: %v", result["Hello"])
518 | }
519 | testData = "{\"Hello\" World\"}"
520 | result = mapJSON(testData)
521 | if !strings.Contains(result["error"].(string), `invalid character 'W'`) {
522 | t.Errorf("result: %v", result)
523 | }
524 | }
525 |
526 | func TestExecDecimalOp(t *testing.T) {
527 | testMulf := func(a interface{}, v ...interface{}) float64 {
528 | return execDecimalOp(a, v, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Mul(d2) })
529 | }
530 | if testMulf(6.2154, "4.35") != 27.03699 {
531 | t.Errorf("result: %v", testMulf(6.2154, "4.35"))
532 | }
533 | testDivf := func(a interface{}, v ...interface{}) float64 {
534 | return execDecimalOp(a, v, func(d1, d2 decimal.Decimal) decimal.Decimal { return d1.Div(d2) })
535 | }
536 | if testDivf(6.2154, "4.35") != 1.4288275862068966 {
537 | t.Errorf("result: %v", testDivf(6.2154, "4.35"))
538 | }
539 | }
540 |
541 | func TestConvertDecimal(t *testing.T) {
542 | result, err := convertDecimal(decimal.RequireFromString("457842.123845"))
543 | if err != nil {
544 | t.Errorf("result: %v", err)
545 | }
546 | if result.String() != "457842.123845" {
547 | t.Errorf("result: %v", result)
548 | }
549 | _, err = convertDecimal("457842.123845a")
550 | if err.Error() != "can't convert 457842.123845a to decimal" {
551 | t.Errorf("result: %v", err)
552 | }
553 | result, err = convertDecimal("457842.123845")
554 | if err != nil {
555 | t.Errorf("result: %v", err)
556 | }
557 | if result.String() != "457842.123845" {
558 | t.Errorf("result: %v", result)
559 | }
560 | result, err = convertDecimal(42)
561 | if err != nil {
562 | t.Errorf("result: %v", err)
563 | }
564 | if result.String() != "42" {
565 | t.Errorf("result: %v", result)
566 | }
567 | result, err = convertDecimal(457842.123845)
568 | if err != nil {
569 | t.Errorf("result: %v", err)
570 | }
571 | if result.String() != "457842.123845" {
572 | t.Errorf("result: %v", result)
573 | }
574 |
575 | }
576 |
577 | // go test -coverprofile cover.out
578 | // go tool cover -html='cover.out'
579 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/mmalcek/bafi
2 |
3 | go 1.23
4 |
5 | require (
6 | github.com/clbanning/mxj/v2 v2.7.0
7 | github.com/google/uuid v1.6.0
8 | github.com/mmalcek/mt940 v0.1.1
9 | github.com/sashabaranov/go-openai v1.38.0
10 | github.com/shopspring/decimal v1.4.0
11 | github.com/spf13/cast v1.7.1
12 | github.com/yuin/gopher-lua v1.1.1
13 | go.mongodb.org/mongo-driver v1.17.3
14 | gopkg.in/yaml.v3 v3.0.1
15 | )
16 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME=
2 | github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s=
3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
6 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
7 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
8 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
9 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
10 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
11 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
12 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
13 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
14 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
15 | github.com/mmalcek/mt940 v0.1.1 h1:w0LYJk4nQWnMeTtuLL1dY2oNMdtHTnWNVY+y7k0KMQU=
16 | github.com/mmalcek/mt940 v0.1.1/go.mod h1:IzQU3xpykKw6QEHn0i75Xxds7eapEEmwYn5L4B28ZZ8=
17 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
18 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
19 | github.com/sashabaranov/go-openai v1.20.2 h1:nilzF2EKzaHyK4Rk2Dbu/aJEZbtIvskDIXvfS4yx+6M=
20 | github.com/sashabaranov/go-openai v1.20.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
21 | github.com/sashabaranov/go-openai v1.38.0 h1:hNN5uolKwdbpiqOn7l+Z2alch/0n0rSFyg4n+GZxR5k=
22 | github.com/sashabaranov/go-openai v1.38.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
23 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
24 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
25 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
26 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
27 | github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
28 | github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
29 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
30 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
31 | github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
32 | github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
33 | go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
34 | go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
35 | go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ=
36 | go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
37 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
38 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
39 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
40 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
41 |
--------------------------------------------------------------------------------
/lua/functions.lua:
--------------------------------------------------------------------------------
1 | json = require './lua/json'
2 |
3 | function sum(incomingData)
4 | dataTable = json.decode(incomingData)
5 | return tostring(tonumber(dataTable[1]) + tonumber(dataTable[2]))
6 | end
7 |
8 | function mul(incomingData)
9 | dataTable = json.decode(incomingData)
10 | return tostring(tonumber(dataTable[1]) * tonumber(dataTable[2]))
11 | end
--------------------------------------------------------------------------------
/lua/json.lua:
--------------------------------------------------------------------------------
1 | -- (https://github.com/rxi/json.lua)
2 | --
3 | -- json.lua
4 | --
5 | -- Copyright (c) 2020 rxi
6 | --
7 | -- Permission is hereby granted, free of charge, to any person obtaining a copy of
8 | -- this software and associated documentation files (the "Software"), to deal in
9 | -- the Software without restriction, including without limitation the rights to
10 | -- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
11 | -- of the Software, and to permit persons to whom the Software is furnished to do
12 | -- so, subject to the following conditions:
13 | --
14 | -- The above copyright notice and this permission notice shall be included in all
15 | -- copies or substantial portions of the Software.
16 | --
17 | -- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 | -- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 | -- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 | -- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 | -- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 | -- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 | -- SOFTWARE.
24 | --
25 |
26 | local json = { _version = "0.1.2" }
27 |
28 | -------------------------------------------------------------------------------
29 | -- Encode
30 | -------------------------------------------------------------------------------
31 |
32 | local encode
33 |
34 | local escape_char_map = {
35 | [ "\\" ] = "\\",
36 | [ "\"" ] = "\"",
37 | [ "\b" ] = "b",
38 | [ "\f" ] = "f",
39 | [ "\n" ] = "n",
40 | [ "\r" ] = "r",
41 | [ "\t" ] = "t",
42 | }
43 |
44 | local escape_char_map_inv = { [ "/" ] = "/" }
45 | for k, v in pairs(escape_char_map) do
46 | escape_char_map_inv[v] = k
47 | end
48 |
49 |
50 | local function escape_char(c)
51 | return "\\" .. (escape_char_map[c] or string.format("u%04x", c:byte()))
52 | end
53 |
54 |
55 | local function encode_nil(val)
56 | return "null"
57 | end
58 |
59 |
60 | local function encode_table(val, stack)
61 | local res = {}
62 | stack = stack or {}
63 |
64 | -- Circular reference?
65 | if stack[val] then error("circular reference") end
66 |
67 | stack[val] = true
68 |
69 | if rawget(val, 1) ~= nil or next(val) == nil then
70 | -- Treat as array -- check keys are valid and it is not sparse
71 | local n = 0
72 | for k in pairs(val) do
73 | if type(k) ~= "number" then
74 | error("invalid table: mixed or invalid key types")
75 | end
76 | n = n + 1
77 | end
78 | if n ~= #val then
79 | error("invalid table: sparse array")
80 | end
81 | -- Encode
82 | for i, v in ipairs(val) do
83 | table.insert(res, encode(v, stack))
84 | end
85 | stack[val] = nil
86 | return "[" .. table.concat(res, ",") .. "]"
87 |
88 | else
89 | -- Treat as an object
90 | for k, v in pairs(val) do
91 | if type(k) ~= "string" then
92 | error("invalid table: mixed or invalid key types")
93 | end
94 | table.insert(res, encode(k, stack) .. ":" .. encode(v, stack))
95 | end
96 | stack[val] = nil
97 | return "{" .. table.concat(res, ",") .. "}"
98 | end
99 | end
100 |
101 |
102 | local function encode_string(val)
103 | return '"' .. val:gsub('[%z\1-\31\\"]', escape_char) .. '"'
104 | end
105 |
106 |
107 | local function encode_number(val)
108 | -- Check for NaN, -inf and inf
109 | if val ~= val or val <= -math.huge or val >= math.huge then
110 | error("unexpected number value '" .. tostring(val) .. "'")
111 | end
112 | return string.format("%.14g", val)
113 | end
114 |
115 |
116 | local type_func_map = {
117 | [ "nil" ] = encode_nil,
118 | [ "table" ] = encode_table,
119 | [ "string" ] = encode_string,
120 | [ "number" ] = encode_number,
121 | [ "boolean" ] = tostring,
122 | }
123 |
124 |
125 | encode = function(val, stack)
126 | local t = type(val)
127 | local f = type_func_map[t]
128 | if f then
129 | return f(val, stack)
130 | end
131 | error("unexpected type '" .. t .. "'")
132 | end
133 |
134 |
135 | function json.encode(val)
136 | return ( encode(val) )
137 | end
138 |
139 |
140 | -------------------------------------------------------------------------------
141 | -- Decode
142 | -------------------------------------------------------------------------------
143 |
144 | local parse
145 |
146 | local function create_set(...)
147 | local res = {}
148 | for i = 1, select("#", ...) do
149 | res[ select(i, ...) ] = true
150 | end
151 | return res
152 | end
153 |
154 | local space_chars = create_set(" ", "\t", "\r", "\n")
155 | local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",")
156 | local escape_chars = create_set("\\", "/", '"', "b", "f", "n", "r", "t", "u")
157 | local literals = create_set("true", "false", "null")
158 |
159 | local literal_map = {
160 | [ "true" ] = true,
161 | [ "false" ] = false,
162 | [ "null" ] = nil,
163 | }
164 |
165 |
166 | local function next_char(str, idx, set, negate)
167 | for i = idx, #str do
168 | if set[str:sub(i, i)] ~= negate then
169 | return i
170 | end
171 | end
172 | return #str + 1
173 | end
174 |
175 |
176 | local function decode_error(str, idx, msg)
177 | local line_count = 1
178 | local col_count = 1
179 | for i = 1, idx - 1 do
180 | col_count = col_count + 1
181 | if str:sub(i, i) == "\n" then
182 | line_count = line_count + 1
183 | col_count = 1
184 | end
185 | end
186 | error( string.format("%s at line %d col %d", msg, line_count, col_count) )
187 | end
188 |
189 |
190 | local function codepoint_to_utf8(n)
191 | -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa
192 | local f = math.floor
193 | if n <= 0x7f then
194 | return string.char(n)
195 | elseif n <= 0x7ff then
196 | return string.char(f(n / 64) + 192, n % 64 + 128)
197 | elseif n <= 0xffff then
198 | return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128)
199 | elseif n <= 0x10ffff then
200 | return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128,
201 | f(n % 4096 / 64) + 128, n % 64 + 128)
202 | end
203 | error( string.format("invalid unicode codepoint '%x'", n) )
204 | end
205 |
206 |
207 | local function parse_unicode_escape(s)
208 | local n1 = tonumber( s:sub(1, 4), 16 )
209 | local n2 = tonumber( s:sub(7, 10), 16 )
210 | -- Surrogate pair?
211 | if n2 then
212 | return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000)
213 | else
214 | return codepoint_to_utf8(n1)
215 | end
216 | end
217 |
218 |
219 | local function parse_string(str, i)
220 | local res = ""
221 | local j = i + 1
222 | local k = j
223 |
224 | while j <= #str do
225 | local x = str:byte(j)
226 |
227 | if x < 32 then
228 | decode_error(str, j, "control character in string")
229 |
230 | elseif x == 92 then -- `\`: Escape
231 | res = res .. str:sub(k, j - 1)
232 | j = j + 1
233 | local c = str:sub(j, j)
234 | if c == "u" then
235 | local hex = str:match("^[dD][89aAbB]%x%x\\u%x%x%x%x", j + 1)
236 | or str:match("^%x%x%x%x", j + 1)
237 | or decode_error(str, j - 1, "invalid unicode escape in string")
238 | res = res .. parse_unicode_escape(hex)
239 | j = j + #hex
240 | else
241 | if not escape_chars[c] then
242 | decode_error(str, j - 1, "invalid escape char '" .. c .. "' in string")
243 | end
244 | res = res .. escape_char_map_inv[c]
245 | end
246 | k = j + 1
247 |
248 | elseif x == 34 then -- `"`: End of string
249 | res = res .. str:sub(k, j - 1)
250 | return res, j + 1
251 | end
252 |
253 | j = j + 1
254 | end
255 |
256 | decode_error(str, i, "expected closing quote for string")
257 | end
258 |
259 |
260 | local function parse_number(str, i)
261 | local x = next_char(str, i, delim_chars)
262 | local s = str:sub(i, x - 1)
263 | local n = tonumber(s)
264 | if not n then
265 | decode_error(str, i, "invalid number '" .. s .. "'")
266 | end
267 | return n, x
268 | end
269 |
270 |
271 | local function parse_literal(str, i)
272 | local x = next_char(str, i, delim_chars)
273 | local word = str:sub(i, x - 1)
274 | if not literals[word] then
275 | decode_error(str, i, "invalid literal '" .. word .. "'")
276 | end
277 | return literal_map[word], x
278 | end
279 |
280 |
281 | local function parse_array(str, i)
282 | local res = {}
283 | local n = 1
284 | i = i + 1
285 | while 1 do
286 | local x
287 | i = next_char(str, i, space_chars, true)
288 | -- Empty / end of array?
289 | if str:sub(i, i) == "]" then
290 | i = i + 1
291 | break
292 | end
293 | -- Read token
294 | x, i = parse(str, i)
295 | res[n] = x
296 | n = n + 1
297 | -- Next token
298 | i = next_char(str, i, space_chars, true)
299 | local chr = str:sub(i, i)
300 | i = i + 1
301 | if chr == "]" then break end
302 | if chr ~= "," then decode_error(str, i, "expected ']' or ','") end
303 | end
304 | return res, i
305 | end
306 |
307 |
308 | local function parse_object(str, i)
309 | local res = {}
310 | i = i + 1
311 | while 1 do
312 | local key, val
313 | i = next_char(str, i, space_chars, true)
314 | -- Empty / end of object?
315 | if str:sub(i, i) == "}" then
316 | i = i + 1
317 | break
318 | end
319 | -- Read key
320 | if str:sub(i, i) ~= '"' then
321 | decode_error(str, i, "expected string for key")
322 | end
323 | key, i = parse(str, i)
324 | -- Read ':' delimiter
325 | i = next_char(str, i, space_chars, true)
326 | if str:sub(i, i) ~= ":" then
327 | decode_error(str, i, "expected ':' after key")
328 | end
329 | i = next_char(str, i + 1, space_chars, true)
330 | -- Read value
331 | val, i = parse(str, i)
332 | -- Set
333 | res[key] = val
334 | -- Next token
335 | i = next_char(str, i, space_chars, true)
336 | local chr = str:sub(i, i)
337 | i = i + 1
338 | if chr == "}" then break end
339 | if chr ~= "," then decode_error(str, i, "expected '}' or ','") end
340 | end
341 | return res, i
342 | end
343 |
344 |
345 | local char_func_map = {
346 | [ '"' ] = parse_string,
347 | [ "0" ] = parse_number,
348 | [ "1" ] = parse_number,
349 | [ "2" ] = parse_number,
350 | [ "3" ] = parse_number,
351 | [ "4" ] = parse_number,
352 | [ "5" ] = parse_number,
353 | [ "6" ] = parse_number,
354 | [ "7" ] = parse_number,
355 | [ "8" ] = parse_number,
356 | [ "9" ] = parse_number,
357 | [ "-" ] = parse_number,
358 | [ "t" ] = parse_literal,
359 | [ "f" ] = parse_literal,
360 | [ "n" ] = parse_literal,
361 | [ "[" ] = parse_array,
362 | [ "{" ] = parse_object,
363 | }
364 |
365 |
366 | parse = function(str, idx)
367 | local chr = str:sub(idx, idx)
368 | local f = char_func_map[chr]
369 | if f then
370 | return f(str, idx)
371 | end
372 | decode_error(str, idx, "unexpected character '" .. chr .. "'")
373 | end
374 |
375 |
376 | function json.decode(str)
377 | if type(str) ~= "string" then
378 | error("expected argument of type string, got " .. type(str))
379 | end
380 | local res, idx = parse(str, next_char(str, 1, space_chars, true))
381 | idx = next_char(str, idx, space_chars, true)
382 | if idx <= #str then
383 | decode_error(str, idx, "trailing garbage")
384 | end
385 | return res
386 | end
387 |
388 |
389 | return json
390 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "encoding/csv"
7 | "encoding/hex"
8 | "encoding/json"
9 | "flag"
10 | "fmt"
11 | "io"
12 | "log"
13 | "math/rand"
14 | "os"
15 | "path/filepath"
16 | "strings"
17 | "text/template"
18 | "time"
19 |
20 | "github.com/clbanning/mxj/v2"
21 | "github.com/mmalcek/mt940"
22 | "github.com/sashabaranov/go-openai"
23 | lua "github.com/yuin/gopher-lua"
24 | "go.mongodb.org/mongo-driver/bson"
25 | "gopkg.in/yaml.v3"
26 | )
27 |
28 | const version = "1.2.1"
29 |
30 | var (
31 | luaData *lua.LState
32 | )
33 |
34 | type tParams struct {
35 | inputFile *string
36 | outputFile *string
37 | textTemplate *string
38 | inputFormat *string
39 | inputDelimiter *string
40 | getVersion *bool
41 | getHelp *bool
42 | chatGPTkey *string
43 | chatGPTmodel *string
44 | chatGPTquery *string
45 | }
46 |
47 | func init() {
48 | rand.Seed(time.Now().UTC().UnixNano())
49 | if _, err := os.Stat("./lua/functions.lua"); !os.IsNotExist(err) {
50 | luaData = lua.NewState()
51 | if err := luaData.DoFile("./lua/functions.lua"); err != nil {
52 | log.Fatal("loadLuaFunctions", err.Error())
53 | }
54 | }
55 | }
56 |
57 | func main() {
58 | params := tParams{
59 | inputFile: flag.String("i", "", `input file
60 | -if not defined read from stdin (pipe mode)
61 | -if prefixed with "?" app will expect yaml file with multiple files description. `),
62 | outputFile: flag.String("o", "", `output file,
63 | -if not defined write to stdout (pipe mode)`),
64 | textTemplate: flag.String("t", "", `template, file or inline.
65 | -Inline template should start with ? e.g. -t "?{{.MyValue}}" `),
66 | inputFormat: flag.String("f", "", "input format: json, bson, yaml, csv, mt940, xml(default)"),
67 | inputDelimiter: flag.String("d", "", "input delimiter: CSV only, default is comma -d ';' or -d 0x09"),
68 | getVersion: flag.Bool("v", false, "show version (Project page: https://github.com/mmalcek/bafi)"),
69 | getHelp: flag.Bool("h", false, "show help"),
70 | chatGPTkey: flag.String("gk", "", "OpenAI API key"),
71 | chatGPTmodel: flag.String("gm", "gpt35", "OpenAI GPT-3 model (gpt35, gpt4)"),
72 | chatGPTquery: flag.String("gq", "", "OpenAI query"),
73 | }
74 | flag.Parse()
75 |
76 | if err := processTemplate(params); err != nil {
77 | log.Fatal(err.Error())
78 | }
79 | if luaData != nil {
80 | luaData.Close()
81 | }
82 | }
83 |
84 | func processTemplate(params tParams) error {
85 | if *params.getVersion {
86 | fmt.Printf("Version: %s\r\nProject page: https://github.com/mmalcek/bafi\r\n", version)
87 | return nil
88 | }
89 | if *params.getHelp {
90 | fmt.Println("Usage: bafi -i input.json -t template.tmpl -o output.txt")
91 | flag.PrintDefaults()
92 | return nil
93 | }
94 | if *params.textTemplate == "" && *params.chatGPTkey == "" {
95 | fmt.Println("template file must be defined: -t template.tmpl")
96 | return nil
97 | }
98 | data, files, err := getInputData(params.inputFile)
99 | if err != nil {
100 | return err
101 | }
102 | // Try identify file format by extension. Input parameter -f has priority
103 | if *params.inputFormat == "" {
104 | switch strings.ToLower(filepath.Ext(*params.inputFile)) {
105 | case ".json":
106 | *params.inputFormat = "json"
107 | case ".bson":
108 | *params.inputFormat = "bson"
109 | case ".yaml", ".yml":
110 | *params.inputFormat = "yaml"
111 | case ".csv":
112 | *params.inputFormat = "csv"
113 | case ".sta":
114 | *params.inputFormat = "mt940"
115 | case ".xml", ".cdf", ".cdf3":
116 | *params.inputFormat = "xml"
117 | default:
118 | *params.inputFormat = ""
119 | }
120 | }
121 |
122 | // If list of file map them one by one else map incoming []byte to mapData
123 | var mapData interface{}
124 | if data == nil && files != nil {
125 | filesStruct := make(map[string]interface{})
126 | for _, file := range files {
127 | data, err := os.ReadFile(file["file"].(string))
128 | if err != nil {
129 | return err
130 | }
131 | *params.inputFormat = file["format"].(string)
132 | if filesStruct[file["label"].(string)], err = mapInputData(data, params); err != nil {
133 | return err
134 | }
135 | }
136 | mapData = &filesStruct
137 | } else {
138 | if mapData, err = mapInputData(data, params); err != nil {
139 | return err
140 | }
141 | }
142 |
143 | if *params.chatGPTkey != "" {
144 | if *params.chatGPTquery == "" {
145 | fmt.Println("OpenAI query must be defined: -gq \"What is the weather like?\"")
146 | return nil
147 | }
148 | response, err := chatGPTprocess(mapData, params)
149 | if err != nil {
150 | return err
151 | }
152 | if *params.outputFile == "" {
153 | fmt.Println(response.Choices[0].Message.Content)
154 | } else {
155 | output, err := os.Create(*params.outputFile)
156 | if err != nil {
157 | return fmt.Errorf("createOutputFile: %s", err.Error())
158 | }
159 | defer output.Close()
160 | output.WriteString(response.Choices[0].Message.Content)
161 | }
162 | return nil
163 | }
164 |
165 | templateFile, err := readTemplate(*params.textTemplate)
166 | if err != nil {
167 | return err
168 | }
169 | if err := writeOutputData(mapData, params.outputFile, templateFile); err != nil {
170 | return err
171 | }
172 | return nil
173 | }
174 |
175 | // getInputData get the data from stdin/pipe or from file or forward list of multiple input files
176 | func getInputData(input *string) (data []byte, files []map[string]interface{}, errorMsg error) {
177 | var err error
178 | inputFile := *input
179 | switch {
180 | case inputFile == "":
181 | fi, err := os.Stdin.Stat()
182 | if err != nil {
183 | return nil, nil, fmt.Errorf("getStdin: %s", err.Error())
184 | }
185 | if fi.Mode()&os.ModeNamedPipe == 0 {
186 | return nil, nil, fmt.Errorf("stdin: Error-noPipe")
187 | }
188 | if data, err = io.ReadAll(os.Stdin); err != nil {
189 | return nil, nil, fmt.Errorf("readStdin: %s", err.Error())
190 | }
191 | case inputFile[:1] == "?":
192 | files = make([]map[string]interface{}, 0)
193 | configFile, err := os.ReadFile(inputFile[1:])
194 | if err != nil {
195 | return nil, nil, fmt.Errorf("readFileList: %s", err.Error())
196 | }
197 | if err := yaml.Unmarshal(configFile, &files); err != nil {
198 | return nil, nil, fmt.Errorf("yaml.UnmarshalFileList: %s", err.Error())
199 | }
200 | return nil, files, nil
201 | default:
202 | if data, err = os.ReadFile(inputFile); err != nil {
203 | return nil, nil, fmt.Errorf("readFile: %s", err.Error())
204 | }
205 | }
206 | return cleanBOM(data), nil, nil
207 | }
208 |
209 | // mapInputData map input data to map[string]interface{}
210 | func mapInputData(data []byte, params tParams) (interface{}, error) {
211 | switch strings.ToLower(*params.inputFormat) {
212 | case "json":
213 | var mapData map[string]interface{}
214 | if err := json.Unmarshal(data, &mapData); err != nil {
215 | if strings.Contains(err.Error(), "cannot unmarshal array") {
216 | mapDataArray := make([]map[string]interface{}, 0)
217 | if err := json.Unmarshal(data, &mapDataArray); err != nil {
218 | return nil, fmt.Errorf("jsonArray: %s", err.Error())
219 | }
220 | return mapDataArray, nil
221 | }
222 | return nil, fmt.Errorf("mapJSON: %s", err.Error())
223 | }
224 | return mapData, nil
225 | case "bson":
226 | var mapData map[string]interface{}
227 | if err := bson.Unmarshal(data, &mapData); err != nil {
228 | // If error try parse as mongoDump
229 | if strings.Contains(err.Error(), "invalid document length") {
230 | var rawData bson.Raw
231 | mapDataArray := make([]map[string]interface{}, 0)
232 | i := 0
233 | for len(data) > 0 {
234 | var x map[string]interface{}
235 | if err := bson.Unmarshal(data, &rawData); err != nil {
236 | return nil, fmt.Errorf("mapBSONArray1: %s", err.Error())
237 | }
238 | if err := bson.Unmarshal(rawData, &x); err != nil {
239 | return nil, fmt.Errorf("mapBSONArray2: %s", err.Error())
240 | }
241 | mapDataArray = append(mapDataArray, x)
242 | data = data[len(rawData):]
243 | i++
244 | }
245 | return mapDataArray, nil
246 | }
247 | return nil, fmt.Errorf("mapBSON: %s", err.Error())
248 | }
249 | return mapData, nil
250 | case "yaml":
251 | var mapData map[string]interface{}
252 | if err := yaml.Unmarshal(data, &mapData); err != nil {
253 | if strings.Contains(err.Error(), "cannot unmarshal !!") {
254 | mapDataArray := make([]map[string]interface{}, 0)
255 | if err := yaml.Unmarshal(data, &mapDataArray); err != nil {
256 | return nil, fmt.Errorf("yamlArray: %s", err.Error())
257 | }
258 | return mapDataArray, nil
259 | }
260 | return nil, fmt.Errorf("mapYAML: %s", err.Error())
261 | }
262 | return mapData, nil
263 | case "csv":
264 | var mapData []map[string]interface{}
265 | r := csv.NewReader(strings.NewReader(string(data)))
266 | r.Comma = prepareDelimiter(*params.inputDelimiter)
267 | lines, err := r.ReadAll()
268 | if err != nil {
269 | return nil, fmt.Errorf("mapCSV: %s", err.Error())
270 | }
271 | mapData = make([]map[string]interface{}, len(lines[1:]))
272 | headers := make([]string, len(lines[0]))
273 | copy(headers, lines[0])
274 | for i, line := range lines[1:] {
275 | x := make(map[string]interface{})
276 | for j, value := range line {
277 | x[headers[j]] = value
278 | }
279 | mapData[i] = x
280 | }
281 | return mapData, nil
282 | case "xml":
283 | mapData, err := mxj.NewMapXml(data)
284 | if err != nil {
285 | return nil, fmt.Errorf("mapXML: %s", err.Error())
286 | }
287 | return mapData, nil
288 | case "mt940":
289 | if *params.inputDelimiter == "" {
290 | return mt940.Parse(data)
291 | } else {
292 | *params.inputDelimiter = strings.Replace(*params.inputDelimiter, `\r`, "\r", -1)
293 | *params.inputDelimiter = strings.Replace(*params.inputDelimiter, `\n`, "\n", -1)
294 | return mt940.ParseMultimessage(data, *params.inputDelimiter)
295 | }
296 | default:
297 | return nil, fmt.Errorf("unknown input format: use parameter -f to define input format e.g. -f json (accepted values are json, bson, yaml, csv, mt940, xml)")
298 | }
299 | }
300 |
301 | // Delimiter can be defined as string or as HEX value eg. 0x09
302 | func prepareDelimiter(inputString string) rune {
303 | if inputString != "" {
304 | if len(inputString) == 4 && inputString[0:2] == "0x" {
305 | bytes, err := hex.DecodeString(inputString[2:4])
306 | if err != nil {
307 | log.Fatalf(fmt.Sprintf("error CSV delimiter: %s", err.Error()))
308 | }
309 | return rune(string(bytes)[0])
310 | }
311 | return rune(inputString[0])
312 | }
313 | return rune(',')
314 | }
315 |
316 | // readTemplate get template from file or from input
317 | func readTemplate(textTemplate string) ([]byte, error) {
318 | var templateFile []byte
319 | var err error
320 | if textTemplate[:1] == "?" {
321 | templateFile = []byte(textTemplate[1:])
322 | } else {
323 | templateFile, err = os.ReadFile(textTemplate)
324 | if err != nil {
325 | return nil, fmt.Errorf("readFile: %s", err.Error())
326 | }
327 | }
328 | return templateFile, nil
329 | }
330 |
331 | // writeOutputData process template and write output
332 | func writeOutputData(mapData interface{}, outputFile *string, templateFile []byte) error {
333 | var err error
334 | template, err := template.New("new").Funcs(templateFunctions()).Parse(string(templateFile))
335 | if err != nil {
336 | return fmt.Errorf("parseTemplate: %s", err.Error())
337 | }
338 | if *outputFile == "" {
339 | output := new(bytes.Buffer)
340 | if err = template.Execute(output, mapData); err != nil {
341 | return fmt.Errorf("writeStdout: %s", err.Error())
342 | }
343 | fmt.Print(output)
344 | } else {
345 | output, err := os.Create(*outputFile)
346 | if err != nil {
347 | return fmt.Errorf("createOutputFile: %s", err.Error())
348 | }
349 | defer output.Close()
350 | if err = template.Execute(output, mapData); err != nil {
351 | return fmt.Errorf("writeOutputFile: %s", err.Error())
352 | }
353 | }
354 | return nil
355 | }
356 |
357 | func chatGPTprocess(mapData interface{}, params tParams) (response openai.ChatCompletionResponse, err error) {
358 | jsonData, err := json.Marshal(mapData)
359 | if err != nil {
360 | return response, fmt.Errorf("jsonMarshal: %s", err.Error())
361 | }
362 | model := openai.GPT3Dot5Turbo
363 | switch *params.chatGPTmodel {
364 | case "gpt35":
365 | model = openai.GPT3Dot5Turbo
366 | case "gpt4":
367 | model = openai.GPT4
368 | case "gpt4o":
369 | model = openai.GPT4o
370 | case "gpt4o-mini":
371 | model = openai.GPT4oMini
372 | default:
373 | model = openai.GPT3Dot5Turbo
374 | }
375 |
376 | client := openai.NewClient(*params.chatGPTkey)
377 | return client.CreateChatCompletion(
378 | context.Background(),
379 | openai.ChatCompletionRequest{
380 | Model: model,
381 | Messages: []openai.ChatCompletionMessage{
382 | {
383 | Role: openai.ChatMessageRoleUser,
384 | Content: *params.chatGPTquery + "\n" + string(jsonData),
385 | },
386 | },
387 | },
388 | )
389 | }
390 |
391 | // cleanBOM remove UTF-8 Byte Order Mark if present
392 | func cleanBOM(b []byte) []byte {
393 | if len(b) >= 3 &&
394 | b[0] == 0xef &&
395 | b[1] == 0xbb &&
396 | b[2] == 0xbf {
397 | return b[3:]
398 | }
399 | return b
400 | }
401 |
--------------------------------------------------------------------------------
/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "encoding/base64"
5 | "strings"
6 | "testing"
7 |
8 | "github.com/clbanning/mxj/v2"
9 | )
10 |
11 | const bsonDump = `JgAAAAdfaWQAYQAVOXB0lMEH42tuAm5hbWUABgAAAEhlbGxvAAAmAAAAB19pZABhCmAZAkO5o4cVWbsCbmFtZQAGAAAAV29ybGQAAA==`
12 |
13 | func TestProcessTemplate(t *testing.T) {
14 | inputFile := ""
15 | inputFormat := ""
16 | inputDelimiter := ","
17 | outputFile := ""
18 | textTemplate := `?{{define content}}`
19 | getVersion := false
20 | chatGPTkey := ""
21 | chatGPTmodel := ""
22 | chatGPTquery := ""
23 |
24 | params := tParams{
25 | inputFile: &inputFile,
26 | inputFormat: &inputFormat,
27 | inputDelimiter: &inputDelimiter,
28 | outputFile: &outputFile,
29 | textTemplate: &textTemplate,
30 | getVersion: &getVersion,
31 | chatGPTkey: &chatGPTkey,
32 | chatGPTmodel: &chatGPTmodel,
33 | chatGPTquery: &chatGPTquery,
34 | }
35 | err := processTemplate(params)
36 | if !strings.Contains(err.Error(), "stdin: Error-noPipe") {
37 | t.Errorf("result: %v", err.Error())
38 | }
39 | inputFile = "testdata.xml"
40 | textTemplate = "hello.tmpl"
41 | err = processTemplate(params)
42 | if !strings.Contains(err.Error(), `readFile: open hello.tmpl:`) {
43 | t.Errorf("result: %v", err.Error())
44 | }
45 | inputFile = "testdata.xml"
46 | textTemplate = "?{{define content}}"
47 | err = processTemplate(params)
48 | if !strings.Contains(err.Error(), `unexpected "content" in define`) {
49 | t.Errorf("result: %v", err.Error())
50 | }
51 | inputFormat = "json"
52 | err = processTemplate(params)
53 | if !strings.Contains(err.Error(), `mapJSON: invalid character`) {
54 | t.Errorf("result: %v", err.Error())
55 | }
56 | textTemplate = `?Output template test: description = {{index .TOP_LEVEL "-description"}} {{print "\r\n"}}`
57 | inputFormat = ""
58 | err = processTemplate(params)
59 | if err != nil {
60 | t.Errorf("result: %v", err.Error())
61 | }
62 |
63 | textTemplate = `?Output template test: description = {{index .filesTest.TOP_LEVEL "-description"}} {{print "\r\n"}}`
64 | inputFile = "?filesTest.yaml"
65 | err = processTemplate(params)
66 | if err != nil {
67 | t.Errorf("result: %v", err.Error())
68 | }
69 |
70 | inputFile = "?filesTest.yamlx"
71 | err = processTemplate(params)
72 | if !strings.Contains(err.Error(), "readFileList: open filesTest.yamlx") {
73 | t.Errorf("result: %v", err.Error())
74 | }
75 |
76 | getVersion = true
77 | err = processTemplate(params)
78 | if err != nil {
79 | t.Errorf("result: %v", err.Error())
80 | }
81 | textTemplate = ""
82 | getVersion = false
83 | err = processTemplate(params)
84 | if err != nil {
85 | t.Errorf("result: %v", err.Error())
86 | }
87 | }
88 |
89 | func TestGetInputData(t *testing.T) {
90 | inputFile := "testdata.xml"
91 | data, _, _ := getInputData(&inputFile)
92 | if !strings.Contains(string(data), ``) {
93 | t.Errorf("result: %v", string(data))
94 | }
95 | inputFile = "Hello.xml"
96 | _, _, err := getInputData(&inputFile)
97 | if !strings.Contains(err.Error(), `readFile: open Hello.xml:`) {
98 | t.Errorf("result: %v", err.Error())
99 | }
100 | inputFile = ""
101 | _, _, err = getInputData(&inputFile)
102 | if !strings.Contains(err.Error(), `stdin: Error-noPipe`) {
103 | t.Errorf("result: %v", err.Error())
104 | }
105 | }
106 |
107 | func TestMapInputData(t *testing.T) {
108 | inputFile := ""
109 | inputFormat := ""
110 | inputDelimiter := ","
111 | outputFile := ""
112 | textTemplate := `?{{define content}}`
113 | getVersion := false
114 | params := tParams{
115 | inputFile: &inputFile,
116 | inputFormat: &inputFormat,
117 | inputDelimiter: &inputDelimiter,
118 | outputFile: &outputFile,
119 | textTemplate: &textTemplate,
120 | getVersion: &getVersion,
121 | }
122 | // Test map json
123 | input := []byte(`{"name": "John","age": 30}`)
124 | inputFormat = "json"
125 | result, _ := mapInputData(input, params)
126 | if result.(map[string]interface{})["name"] != "John" || result.(map[string]interface{})["age"] != float64(30) {
127 | t.Errorf("resultJSON: %v", result)
128 | }
129 | input = []byte(`[{"name": "John","age": 30}, {"name": "Hanz","age": 28}]`)
130 | result, _ = mapInputData(input, params)
131 | if result.([]map[string]interface{})[0]["name"] != "John" {
132 | t.Errorf("resultJSONarray: %v", result.([]map[string]interface{})[0]["name"])
133 | }
134 | input = []byte(`[{"name": "John","age": 30}, {"name": Hanz","age": 28}]`)
135 | _, err := mapInputData(input, params)
136 | if !strings.Contains(err.Error(), "invalid character 'H'") {
137 | t.Errorf("resultJSONerr: %v", err.Error())
138 | }
139 | input = []byte(`{"name" John","age": 30}`)
140 | _, err = mapInputData(input, params)
141 | if !strings.Contains(err.Error(), "invalid character 'J'") {
142 | t.Errorf("resultJSONerr: %v", err.Error())
143 | }
144 | // Test map bson
145 | input = []byte{14, 0, 0, 0, 2, 104, 0, 2, 0, 0, 0, 119, 0, 0}
146 | inputFormat = "bson"
147 | result, _ = mapInputData(input, params)
148 | if result.(map[string]interface{})["h"] != "w" {
149 | t.Errorf("resultBSON: %v", result)
150 | }
151 | input, err = base64.StdEncoding.DecodeString(bsonDump)
152 | if err != nil {
153 | t.Errorf("base64BSONdump: %v", err.Error())
154 | }
155 | result, _ = mapInputData(input, params)
156 | if result.([]map[string]interface{})[0]["name"] != `Hello` {
157 | t.Errorf("resultBSONdump: %v", result.([]map[string]interface{})[0]["name"])
158 | }
159 | input = []byte{14, 0, 0, 1, 2, 104, 0, 2, 0, 0, 0, 119, 0, 0}
160 | _, err = mapInputData(input, params)
161 | if !strings.Contains(err.Error(), "EOF") {
162 | t.Errorf("resultBSONerr: %v", err.Error())
163 | }
164 | // Test map yaml
165 | input = []byte(`name: John`)
166 | inputFormat = "yaml"
167 | result, _ = mapInputData(input, params)
168 | if result.(map[string]interface{})["name"] != "John" {
169 | t.Errorf("resultYAML: %v", result)
170 | }
171 | input = []byte("- name: John\r\n- name: Peter")
172 | result, _ = mapInputData(input, params)
173 | if result.([]map[string]interface{})[0]["name"] != "John" {
174 | t.Errorf("resultYAMLarray: %v", result)
175 | }
176 | input = []byte("- name John\r\n- name: Peter")
177 | _, err = mapInputData(input, params)
178 | if !strings.Contains(err.Error(), "cannot unmarshal !!str `name John`") {
179 | t.Errorf("resultYAMLarrayErr: %v", err.Error())
180 | }
181 | input = []byte(`name John`)
182 | _, err = mapInputData(input, params)
183 | if !strings.Contains(err.Error(), "cannot unmarshal !!str `name John`") {
184 | t.Errorf("resultYAMLerr: %v", err.Error())
185 | }
186 | // Test map csv
187 | input = []byte("name,surname\r\nHello,World")
188 | inputFormat = "csv"
189 | result, _ = mapInputData(input, params)
190 | if result.([]map[string]interface{})[0]["name"] != "Hello" {
191 | t.Errorf("result: %v", result.([]map[string]interface{})[0]["name"])
192 | }
193 | input = []byte("name,surname\r\nHello,World")
194 | inputDelimiter = "0x2C"
195 | result, _ = mapInputData(input, params)
196 | if result.([]map[string]interface{})[0]["name"] != "Hello" {
197 | t.Errorf("result: %v", result.([]map[string]interface{})[0]["name"])
198 | }
199 | input = []byte("name,surname\r\nHello,World,!!!")
200 | _, err = mapInputData(input, params)
201 | if !strings.Contains(err.Error(), "wrong number of fields") {
202 | t.Errorf("result: %v", err.Error())
203 | }
204 |
205 | // Test map xml
206 | input = []byte(`John`)
207 | inputFormat = "xml"
208 | result, _ = mapInputData(input, params)
209 | if result.(mxj.Map)["name"] != "John" {
210 | t.Errorf("result: %v", result)
211 | }
212 | input = []byte(`John`)
213 | _, err = mapInputData(input, params)
214 | if !strings.Contains(err.Error(), "xml.Decoder.Token() - XML syntax error on line 1") {
215 | t.Errorf("result: %v", err.Error())
216 | }
217 | }
218 |
219 | func TestReadTemplate(t *testing.T) {
220 | // Test inline template
221 | result, err := readTemplate("?{{toXML .}}")
222 | if string(result) != "{{toXML .}}" {
223 | t.Errorf("result: %v", string(result))
224 | }
225 | if err != nil {
226 | t.Errorf("err: %v", err)
227 | }
228 | // Test template file
229 | result, err = readTemplate("template.tmpl")
230 | if !strings.Contains(string(result), "CSV formatted data:") {
231 | t.Errorf("result: %v", string(result))
232 | }
233 | if err != nil {
234 | t.Errorf("err: %v", err)
235 | }
236 | // Test nonExisting file
237 | _, err = readTemplate("hello.tmpl")
238 | if !strings.Contains(err.Error(), "readFile: open hello.tmpl:") {
239 | t.Errorf("err: %v", err)
240 | }
241 | }
242 |
243 | func TestWriteOutputData(t *testing.T) {
244 | testData := make(map[string]interface{})
245 | testData["Hello"] = "World"
246 | outputFile := ""
247 | templateFile := []byte(`{{define content}}`)
248 | err := writeOutputData(testData, &outputFile, templateFile)
249 | if !strings.Contains(err.Error(), `new:1: unexpected "content"`) {
250 | t.Errorf("result: %v", err.Error())
251 | }
252 | templateFile = []byte(`Output test: Hello {{.Hello}} {{print "\r\n"}}`)
253 | if err := writeOutputData(testData, &outputFile, templateFile); err != nil {
254 | t.Errorf("result: %v", err.Error())
255 | }
256 | outputFile = "output.txt"
257 | if err := writeOutputData(testData, &outputFile, templateFile); err != nil {
258 | t.Errorf("result: %v", err.Error())
259 | }
260 | testData["Hello"] = make(chan int, 1)
261 | err = writeOutputData(testData, &outputFile, templateFile)
262 | if !strings.Contains(err.Error(), "can't print {{.Hello}} of type chan int") {
263 | t.Errorf("result: %v", err.Error())
264 | }
265 | outputFile = "out*he\\ll//o/./txt"
266 | err = writeOutputData(testData, &outputFile, templateFile)
267 | if !strings.Contains(err.Error(), "createOutputFile:") {
268 | t.Errorf("result: %v", err.Error())
269 | }
270 | }
271 |
272 | func TestCleanBOM(t *testing.T) {
273 | input := "\xef\xbb\xbf" + "Hello"
274 | result := string(cleanBOM([]byte(input)))
275 | if result != "Hello" {
276 | t.Errorf("result: %v", result)
277 | }
278 | input = "Hello"
279 | result = string(cleanBOM([]byte(input)))
280 | if result != "Hello" {
281 | t.Errorf("result: %v", result)
282 | }
283 | }
284 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: BaFi
2 | site_url: https://mmalcek.github.io/bafi
3 | nav:
4 | - Application: README.md
5 | - Examples: examples.md
6 | - About: about.md
7 | theme:
8 | name: mkdocs
9 | highlightjs: true
10 | hljs_languages:
11 | - lua
12 | - powershell
13 | navigation_depth: 4
14 | nav_style: light
15 | plugins:
16 | - search
17 | analytics:
18 | gtag: G-GE1PDVJERL
19 |
--------------------------------------------------------------------------------
/template.tmpl:
--------------------------------------------------------------------------------
1 | ------------------------------------------------------------------------------
2 | CSV formatted data:
3 | Employee,Date,val1,val2,val3,SUM,LuaMultiply,linkedText
4 | {{- /* {{- ...}} - minus trim whitespace */}}
5 | {{- range (mustArray .TOP_LEVEL.DATA_LINE)}} {{- /* mustArray - force .DATA_LINE to array even if there is just one record (nil if not exists) */}}
6 | {{index .Employee "-ID"}},{{dateFormat .Trans_Date "2006-01-02" "02.01.2006"}},{{.val1}},{{.val2}},{{.val3}},{{add .val1 .val2}},{{lua "mul" .val1 .val2}},"{{index .Linked_Text "-VALUE"}}"
7 | {{- end}}
8 | ------------------------------------------------------------------------------
9 | Count totals:
10 | {{- /* Use variable and iterate over lines to get SUM */}}
11 | {{- $TotalV1 := 0 }}
12 | {{- range .TOP_LEVEL.DATA_LINE}}{{$TotalV1 = lua "sum" $TotalV1 .val1}}{{end}}
13 | {{- $TotalV3 := 0 }}
14 | {{- /* addf - will provide decimal count */}}
15 | {{- range .TOP_LEVEL.DATA_LINE}}{{$TotalV3 = addf $TotalV3 .val3}}{{end}}
16 | Total:
17 | {{- /* if functions and,or,not,eq(equal),lt(lessThen),gt(greatherThen) */}}
18 | Val1: {{$TotalV1}} - {{if gt (toInt $TotalV1) 50}}Over Budget{{else}}Under Buget{{end}}
19 | Val3: {{$TotalV3}}
20 | Created at: {{now "02.01.2006 - 15:04:05"}}
21 | ------------------------------------------------------------------------------
22 | JSON formatted data:
23 | {{- $new := "{\"employees\": [" }}
24 | {{- range .TOP_LEVEL.DATA_LINE}}
25 | {{- $new = print $new "{\"employeeID\":\"" (index .Employee "-ID") "\", \"val1\":" .val1 "}," }}
26 | {{- end}}
27 | {{- /* Trim trailing comma, alternatively you can remove last char by "(slice $new 0 (sub (len $new) 1))" */}}
28 | {{- $new = print (trimSuffix $new "," ) "]}"}}
29 | {{$new}}
30 | ------------------------------------------------------------------------------
31 | JSON Converted to map and marshal to YAML:
32 | {{toYAML (mapJSON $new) -}}
33 | ------------------------------------------------------------------------------
34 |
--------------------------------------------------------------------------------
/testdata.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | 2021-05-21
7 | 5
8 | 12
9 | 18.3285
10 |
11 |
12 |
13 |
14 |
15 |
16 | 2021-05-23
17 | 43
18 | 9
19 | 5.67343
20 |
21 |
22 |
23 |
24 |
25 |
26 | 2021-05-23
27 | 14
28 | 4
29 | 2.97984
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/workspace.code-workspace:
--------------------------------------------------------------------------------
1 | {
2 | "folders": [
3 | {
4 | "name": "BaFi",
5 | "path": "."
6 | },
7 | ],
8 | "settings": {
9 | "editor.formatOnSave": true,
10 | "editor.minimap.enabled": false,
11 | "files.autoSave": "onFocusChange",
12 | "files.eol": "\r\n",
13 | "git.confirmSync": false,
14 | "git.autofetch": true,
15 | "git.enableSmartCommit": true,
16 | "go.useLanguageServer": true,
17 | }
18 | }
--------------------------------------------------------------------------------