├── .github └── workflows │ └── build-and-push-docker.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── internal ├── classifier │ └── classifier.go ├── config │ ├── config.go │ └── config_test.go ├── firefly │ └── firefly.go ├── handlers │ └── handler.go └── router │ ├── router.go │ └── router_test.go └── main.go /.github/workflows/build-and-push-docker.yml: -------------------------------------------------------------------------------- 1 | # Initial maintaner's setup: 2 | # 1. Create a [DockerHub personal access token](https://docs.docker.com/docker-hub/access-tokens/) 3 | # 2. Set [Github actions repository secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository) 4 | # `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` 5 | # with values of your Docker hub username and the personal token respectively. 6 | # 3. Replace the user tag in the "Build and push" action with your Docker hub user. 7 | 8 | name: Build and Push Docker Image 9 | on: 10 | push: 11 | branches: 12 | - main 13 | 14 | jobs: 15 | build: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@v4 20 | 21 | - name: Set up QEMU 22 | uses: docker/setup-qemu-action@v3 23 | 24 | - name: Set up Docker Buildx 25 | uses: docker/setup-buildx-action@v3 26 | with: 27 | platforms: linux/amd64,linux/arm64 28 | 29 | - name: Login to Docker Hub 30 | uses: docker/login-action@v3 31 | with: 32 | username: ${{ secrets.DOCKER_HUB_USER }} 33 | password: ${{ secrets.DOCKER_HUB_TOKEN }} 34 | 35 | - name: Build and push 36 | uses: docker/build-push-action@v5 37 | with: 38 | context: . 39 | platforms: linux/amd64,linux/arm64 40 | push: true 41 | tags: akopulko/ffiiitc:latest 42 | - name: Update Docker Hub Overview 43 | uses: peter-evans/dockerhub-description@v3 44 | with: 45 | username: ${{ secrets.DOCKER_HUB_USER }} 46 | password: ${{ secrets.DOCKER_HUB_TOKEN }} 47 | repository: akopulko/ffiiitc 48 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.yaml 3 | test-data/ 4 | bin/ 5 | .gitignore 6 | .DS_Store 7 | *.code-workspace 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Multiarch Docker cross-compiled build. 2 | # Ref: https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ 3 | 4 | FROM --platform=$BUILDPLATFORM golang:1.21-alpine as build 5 | WORKDIR /src 6 | COPY internal/ ./internal 7 | COPY go.mod ./ 8 | COPY go.sum ./ 9 | COPY *.go ./ 10 | RUN go mod download 11 | RUN go test ./... 12 | ARG TARGETPLATFORM TARGETARCH TARGETOS 13 | RUN echo "Building for ${TARGETPLATFORM}..." 14 | RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /out/ffiiitc 15 | 16 | FROM alpine:latest as release 17 | WORKDIR /app 18 | RUN mkdir -p /app/data 19 | COPY --from=build /out/ffiiitc /app/ffiiitc 20 | EXPOSE 8080 21 | ENTRYPOINT ["/app/ffiiitc"] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kostya 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FireFly III Transaction Classification 2 | 3 | This little web service performs transaction classification and integrates with [FireFly III](https://github.com/firefly-iii/firefly-iii) (A free and open source personal finance manager) via web hooks. 4 | 5 | ### What is does? 6 | 7 | Every time you add new transaction to FireFly III, either manually or via import tool, web hook will trigger and provide transaction description to `ffiiitc`. It will then be classified using [Naive Bayesian Classification](https://en.wikipedia.org/wiki/Naive_Bayes_classifier) and transaction will be updated with matching category. 8 | 9 | > Naive Bayesian classifier go package used by `ffiiitc` is available [here](https://github.com/navossoc/bayesian). Please read the [license](https://github.com/navossoc/bayesian/blob/master/LICENSE). 10 | 11 | ### How to run? 12 | 13 | #### Pre-requisites 14 | 15 | - [Docker desktop](https://www.docker.com/products/docker-desktop/) or any other form of running containers on your computer 16 | - [FireFly III](https://github.com/firefly-iii/firefly-iii) up and running as per [docs](https://docs.firefly-iii.org/firefly-iii/installation/docker/?mtm_campaign=docu-internal&mtm_kwd=docker) 17 | - At least **one or two statements** imported into FireFly with transactions manually **categorised**. This is required for classifier to train on your dataset and is very important. 18 | - Have personal access token (PAT) generated in FireFly III. Go to `Options->Profile->OAuth` click `Create new token` 19 | 20 | ##### Docker Compose 21 | 22 | - `git clone https://github.com/akopulko/ffiiitc.git` 23 | - `docker buildx build --load --platform=linux/amd64 -t ffiiitc:latest .` 24 | 25 | #### Run 26 | 27 | ##### Docker Compose 28 | 29 | - Stop `docker compose -f docker-compose.yml down` 30 | - Modify your FireFly III docker compose file add the following 31 | 32 | ```yaml 33 | fftc: 34 | image: akopulko/ffiiitc:latest 35 | hostname: fftc 36 | networks: 37 | - firefly_iii 38 | restart: always 39 | container_name: ffiiitc 40 | environment: 41 | - FF_API_KEY= 42 | - FF_APP_URL= 43 | volumes: 44 | - ffiiitc-data:/app/data 45 | ports: 46 | - ':8080' 47 | depends_on: 48 | - app 49 | volumes: 50 | ... 51 | ffiiitc-data: 52 | ``` 53 | 54 | You can also append your environment variable names with `_FILE` instead, having their value point to the file where tha actual sensitive value is stored. This works with any environment variable. 55 | 56 | ```yaml 57 | secrets: 58 | ffiiitc-personal-access-token: 59 | file: "/ffiiitc-personal-access-token" 60 | 61 | services: 62 | ... 63 | fftc: 64 | image: akopulko/ffiiitc:latest 65 | hostname: fftc 66 | networks: 67 | - firefly_iii 68 | restart: always 69 | container_name: ffiiitc 70 | secrets: 71 | - "ffiiitc-personal-access-token" 72 | environment: 73 | - FF_API_KEY_FILE="/run/secrets/ffiiitc-personal-access-token" 74 | - FF_APP_URL= 75 | volumes: 76 | - ffiiitc-data:/app/data 77 | ports: 78 | - ':8080' 79 | depends_on: 80 | - app 81 | 82 | volumes: 83 | ... 84 | ffiiitc-data: 85 | ``` 86 | 87 | - Start `docker compose -f docker-compose.yml up -d` 88 | 89 | #### Docker 90 | 91 | ``` 92 | docker run 93 | -d 94 | --name='ffiiitc' 95 | -e 'FF_API_KEY'='' 96 | -e 'FF_APP_URL'='' 97 | -p ':8080' 98 | -v '':'/app/data':'rw' 'ffiiitc' 99 | ``` 100 | 101 | #### Configure Web Hooks in FireFly 102 | 103 | In `FireFly` go to `Automation -> Webhooks` and click `Create new webhook` 104 | 105 | - Create webhook for transaction classification 106 | 107 | ```yaml 108 | title: classify 109 | trigger: after transaction creation 110 | response: transaction details 111 | delivery: json 112 | url: http://fftc:/classify 113 | active: checked 114 | ``` 115 | 116 | ### Troubleshooting 117 | 118 | #### Logs 119 | You can check `ffiiitc` logs to see if there are any errors:
`docker compose logs fftc -f` 120 | 121 | #### Forced training of your model 122 | There is also option available to force train the model from your transactions if required. 123 | To trigger force train run the following command and restart `fftc` container: 124 | `curl -i http://localhost:/train` where `EXPOSED_PORT` is the port you provided in your docker compose for `fftc`. 125 | As always, you can check logs to see if model was successfully regenerated. 126 | 127 | You can also provide optional `start` and `end` date query parameters (in `yyyy-mm-dd` format) to limit the transactions used for training. For example: 128 | 129 | ``` 130 | curl -i "http://localhost:/train?start=2024-01-01&end=2024-06-01" 131 | ``` 132 | 133 | If `start` and/or `end` are omitted, all available transactions will be used for training. -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module ffiiitc 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/go-pkgz/lgr v0.11.0 7 | github.com/navossoc/bayesian v0.0.0-20230423142728-ab66f8feaf97 8 | github.com/stretchr/testify v1.8.4 9 | ) 10 | 11 | require ( 12 | github.com/davecgh/go-spew v1.1.1 // indirect 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | gopkg.in/yaml.v3 v3.0.1 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/go-pkgz/lgr v0.11.0 h1:9XH5o+vj09L0sRWEswIGK1lJ6g07xVB4/Z28RV9Z+qM= 4 | github.com/go-pkgz/lgr v0.11.0/go.mod h1:4rdRmMSs4yGFjnUg0rSDbKx21LmFNZoH4y8OLl3qDnU= 5 | github.com/navossoc/bayesian v0.0.0-20230423142728-ab66f8feaf97 h1:6CBjPos6l0GnoMCMiQPbhPRhHXFaa7RFAGwpRwveVlI= 6 | github.com/navossoc/bayesian v0.0.0-20230423142728-ab66f8feaf97/go.mod h1:P1c1lcW3JeYIRbVw98K6qNHJq/3hX4ru5SCQc84ZbZo= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 10 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 11 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 12 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 13 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 14 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 15 | -------------------------------------------------------------------------------- /internal/classifier/classifier.go: -------------------------------------------------------------------------------- 1 | package classifier 2 | 3 | import ( 4 | "regexp" 5 | "slices" 6 | "strings" 7 | 8 | "github.com/go-pkgz/lgr" 9 | "github.com/navossoc/bayesian" 10 | ) 11 | 12 | // classifier implementation 13 | type TrnClassifier struct { 14 | Classifier *bayesian.Classifier 15 | logger *lgr.Logger 16 | } 17 | 18 | type TransactionDataSet [][]string 19 | 20 | // init classifier with model file 21 | func NewTrnClassifierFromFile(modelFile string, l *lgr.Logger) (*TrnClassifier, error) { 22 | cls, err := bayesian.NewClassifierFromFile(modelFile) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return &TrnClassifier{ 27 | Classifier: cls, 28 | logger: l, 29 | }, nil 30 | } 31 | 32 | // init classifier with training data set 33 | func NewTrnClassifierWithTraining(dataSet TransactionDataSet, l *lgr.Logger) (*TrnClassifier, error) { 34 | trainingMap := convertDatasetToTrainingMap(dataSet) 35 | catList := getCategoriesFromTrainingMap(trainingMap) 36 | //catList := maps.Keys(trainingMap) 37 | cls := bayesian.NewClassifier(catList...) 38 | for _, cat := range catList { 39 | cls.Learn(trainingMap[string(cat)], cat) 40 | } 41 | return &TrnClassifier{ 42 | Classifier: cls, 43 | logger: l, 44 | }, nil 45 | } 46 | 47 | // save classifier to model file 48 | func (tc *TrnClassifier) SaveClassifierToFile(modelFile string) error { 49 | err := tc.Classifier.WriteToFile(modelFile) 50 | return err 51 | } 52 | 53 | // perform transaction classification 54 | // in: transaction description 55 | // out: likely transaction category 56 | func (tc *TrnClassifier) ClassifyTransaction(t string) string { 57 | features := extractTransactionFeatures(t) 58 | _, likely, _ := tc.Classifier.LogScores(features) 59 | return string(tc.Classifier.Classes[likely]) 60 | } 61 | 62 | // function to get category and list of 63 | // unique features from line of transaction data set 64 | // in: [cat, trn description] 65 | // out: cat, [features...] 66 | func getCategoryAndFeatures(data []string) (string, []string) { 67 | category := data[0] 68 | words := strings.Split(data[1], " ") 69 | var features []string 70 | for _, word := range words { 71 | if (validFeature(word)) && (!slices.Contains(features, word)) { 72 | features = append(features, word) 73 | } 74 | } 75 | return category, features 76 | 77 | } 78 | 79 | // get slice of categories from training map 80 | func getCategoriesFromTrainingMap(training map[string][]string) []bayesian.Class { 81 | var result []bayesian.Class 82 | for key := range training { 83 | result = append(result, bayesian.Class(key)) 84 | } 85 | return result 86 | } 87 | 88 | // checks if feature is valid 89 | // should be not single symbol and not pure number 90 | func validFeature(feature string) bool { 91 | return len(feature) > 1 && !isStringNumeric(feature) 92 | } 93 | 94 | // checks if string is pure number: int or float 95 | func isStringNumeric(s string) bool { 96 | numericPattern := `^-?\d+(\.\d+)?$` 97 | match, err := regexp.MatchString(numericPattern, s) 98 | return err == nil && match 99 | } 100 | 101 | // build training map from transactions data set 102 | // in: [ [cat, trn description], [cat, trn description]... ] 103 | // out: map[Category] = [feature1, feature2, ...] 104 | func convertDatasetToTrainingMap(dataSet TransactionDataSet) map[string][]string { 105 | resultMap := make(map[string][]string) 106 | var features []string 107 | var category string 108 | for _, line := range dataSet { 109 | category, features = getCategoryAndFeatures(line) 110 | _, exist := resultMap[category] 111 | if exist { 112 | resultMap[category] = append(resultMap[category], features...) 113 | } else { 114 | resultMap[category] = features 115 | } 116 | } 117 | return resultMap 118 | } 119 | 120 | // extract unique words from transaction description that are not numeric 121 | func extractTransactionFeatures(transaction string) []string { 122 | var transFeatures []string 123 | features := strings.Split(transaction, " ") 124 | for _, feature := range features { 125 | if (len(feature) > 1) && (!slices.Contains(transFeatures, feature)) && (!isStringNumeric(feature)) { 126 | transFeatures = append(transFeatures, feature) 127 | } 128 | } 129 | return transFeatures 130 | } 131 | -------------------------------------------------------------------------------- /internal/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/go-pkgz/lgr" 10 | ) 11 | 12 | const ( 13 | FireflyAppTimeout = 10 // 10 sec for fftc to app service timeout 14 | ModelFile = "data/model.gob" //file name to store model 15 | apiKeyEnvVar = "FF_API_KEY" 16 | appUrlEnvVar = "FF_APP_URL" 17 | ) 18 | 19 | type Config struct { 20 | APIKey string 21 | FFApp string 22 | } 23 | 24 | var envVars = []string{ 25 | "FF_API_KEY", 26 | "FF_APP_URL", 27 | } 28 | 29 | func EnvVarExist(varName string) bool { 30 | _, present := os.LookupEnv(varName) 31 | return present 32 | } 33 | 34 | func EnvVarIsSet(varName string) bool { 35 | return os.Getenv(varName) != "" 36 | } 37 | 38 | func LookupEnvVarValueFromFile(path string, logger *lgr.Logger) (string, bool) { 39 | if path == "" { 40 | logger.Logf("WARN file path is empty!") 41 | return "", false 42 | } 43 | 44 | logger.Logf("DEBUG reading file...") 45 | valueBytes, e := os.ReadFile(path) 46 | 47 | if e != nil { 48 | panic(e) 49 | } 50 | value := string(valueBytes) 51 | value = strings.TrimSuffix(value, "\n") 52 | 53 | return string(value), true 54 | } 55 | 56 | func LookupEnvVar(variableName string, logger *lgr.Logger) (string, bool) { 57 | logger.Logf("DEBUG looking for env var %s", variableName) 58 | 59 | // Try get value from a file, like Docker secrets. 60 | if EnvVarIsSet(variableName + "_FILE") { 61 | logger.Logf("DEBUG var %s is set as file, getting file path...", variableName) 62 | path := os.Getenv(variableName + "_FILE") 63 | logger.Logf("DEBUG file path is '%s'", path) 64 | return LookupEnvVarValueFromFile(path, logger) 65 | } 66 | 67 | // Try get value ist stored in variable directly. 68 | logger.Logf("DEBUG extracting value directly from env var") 69 | return os.LookupEnv(variableName) 70 | } 71 | 72 | func FormatEnvNotSetErrorMessage(variableName string) string { 73 | return fmt.Sprintf("Environment vars '%s' or '%s' not set!", variableName, variableName+"_FILE") 74 | } 75 | 76 | func NewConfig(logger *lgr.Logger) (*Config, error) { 77 | apiKey, apiKeyExists := LookupEnvVar(apiKeyEnvVar, logger) 78 | 79 | if !apiKeyExists { 80 | return nil, errors.New(FormatEnvNotSetErrorMessage(apiKeyEnvVar)) 81 | } 82 | 83 | appUrl, appUrlExists := LookupEnvVar(appUrlEnvVar, logger) 84 | if !appUrlExists { 85 | return nil, errors.New(FormatEnvNotSetErrorMessage(appUrlEnvVar)) 86 | } 87 | 88 | cfg := Config{ 89 | APIKey: apiKey, 90 | FFApp: appUrl, 91 | } 92 | 93 | return &cfg, nil 94 | } 95 | -------------------------------------------------------------------------------- /internal/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/go-pkgz/lgr" 8 | ) 9 | 10 | func TestEnvVarExist(t *testing.T) { 11 | t.Run("ExistingEnvVar", func(t *testing.T) { 12 | // Set up a temporary environment variable for testing 13 | os.Setenv("TEST_ENV_VAR", "value") 14 | 15 | // Check if the environment variable exists 16 | if !EnvVarExist("TEST_ENV_VAR") { 17 | t.Error("Expected environment variable to exist, but it does not") 18 | } 19 | 20 | // Clean up the environment variable after the test 21 | os.Unsetenv("TEST_ENV_VAR") 22 | }) 23 | 24 | t.Run("NonExistingEnvVar", func(t *testing.T) { 25 | // Check if the environment variable exists 26 | if EnvVarExist("NON_EXISTING_ENV_VAR") { 27 | t.Error("Expected environment variable to not exist, but it does") 28 | } 29 | }) 30 | } 31 | 32 | func TestNewConfig(t *testing.T) { 33 | logger := lgr.New(lgr.Debug, lgr.CallerFunc) 34 | 35 | t.Run("AllEnvVarsExist", func(t *testing.T) { 36 | 37 | // Set up temporary environment variables for testing 38 | os.Setenv("FF_API_KEY", "test_api_key") 39 | os.Setenv("FF_APP_URL", "test_app_url") 40 | 41 | // Create a new config 42 | cfg, err := NewConfig(logger) 43 | 44 | // Check if there is no error 45 | if err != nil { 46 | t.Errorf("Expected no error, but got: %v", err) 47 | } 48 | 49 | // Check if the APIKey and FFApp fields are correct 50 | if cfg.APIKey != "test_api_key" { 51 | t.Errorf("Expected APIKey to be 'test_api_key', but got: %s", cfg.APIKey) 52 | } 53 | if cfg.FFApp != "test_app_url" { 54 | t.Errorf("Expected FFApp to be 'test_app_url', but got: %s", cfg.FFApp) 55 | } 56 | 57 | // Clean up the environment variables after the test 58 | os.Unsetenv("FF_API_KEY") 59 | os.Unsetenv("FF_APP_URL") 60 | }) 61 | 62 | t.Run("MissingEnvVars", func(t *testing.T) { 63 | // Create a new config 64 | cfg, err := NewConfig(logger) 65 | 66 | // Check if there is an error 67 | if err == nil { 68 | t.Error("Expected error due to missing environment variables, but got no error") 69 | } 70 | 71 | // Check if the config is nil 72 | if cfg != nil { 73 | t.Error("Expected config to be nil, but it is not") 74 | } 75 | }) 76 | } 77 | -------------------------------------------------------------------------------- /internal/firefly/firefly.go: -------------------------------------------------------------------------------- 1 | package firefly 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/go-pkgz/lgr" 12 | ) 13 | 14 | const ( 15 | fireflyAPIPrefix = "api/v1" 16 | ) 17 | 18 | type Timeout time.Duration 19 | 20 | // struct for firefly http client 21 | type FireFlyHttpClient struct { 22 | AppURL string 23 | //Timeout Timeout 24 | Timeout Timeout 25 | Token string 26 | logger *lgr.Logger 27 | } 28 | 29 | // set of structs for firefly transaction json data 30 | type FireFlyTransaction struct { 31 | Description string `json:"description"` 32 | Category string `json:"category_name"` 33 | TransactionID string `json:"transaction_journal_id"` 34 | Tags []string `json:"tags"` 35 | } 36 | 37 | type FireFlyTransactions struct { 38 | FireWebHooks bool `json:"fire_webhooks"` 39 | Id string `json:"id"` 40 | Transactions []FireFlyTransaction `json:"transactions"` 41 | } 42 | 43 | type FireFlyTransactionAttributes struct { 44 | Attributes FireFlyTransactions `json:"attributes"` 45 | } 46 | 47 | // set of structs for firefly pagination json 48 | type FireFlyPaginationData struct { 49 | Total int `json:"total"` 50 | Count int `json:"count"` 51 | PerPage int `json:"per_page"` 52 | CurrentPage int `json:"current_page"` 53 | TotalPages int `json:"total_pages"` 54 | } 55 | 56 | type FireFlyPagination struct { 57 | Pagination FireFlyPaginationData `json:"pagination"` 58 | } 59 | 60 | // firefly transaction api response json 61 | type FireFlyTransactionsResponse struct { 62 | Data []FireFlyTransactionAttributes `json:"data"` 63 | Meta FireFlyPagination `json:"meta"` 64 | } 65 | 66 | func NewFireFlyHttpClient(url, token string, timeout Timeout, l *lgr.Logger) *FireFlyHttpClient { 67 | return &FireFlyHttpClient{ 68 | AppURL: url, 69 | Token: token, 70 | Timeout: timeout, 71 | logger: l, 72 | } 73 | } 74 | 75 | // helper function to make http request to firefly api 76 | // returns body 77 | func (fc *FireFlyHttpClient) sendRequestWithToken(method, url, token string, data []byte) ([]byte, error) { 78 | client := http.Client{ 79 | Timeout: time.Duration(fc.Timeout) * time.Second, // Set a reasonable timeout for the request. 80 | } 81 | 82 | req, err := http.NewRequest(method, url, bytes.NewBuffer(data)) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | // Set the Authorization header with the Bearer token. 88 | req.Header.Set("Authorization", "Bearer "+token) 89 | req.Header.Set("Content-Type", "application/json") 90 | req.Header.Set("accept", "application/vnd.api+json") 91 | 92 | resp, err := client.Do(req) 93 | if err != nil { 94 | return nil, err 95 | } 96 | defer resp.Body.Close() 97 | 98 | if resp.StatusCode != http.StatusOK { 99 | return nil, fmt.Errorf("request failed with status: %d", resp.StatusCode) 100 | } 101 | 102 | bodyBytes, err := io.ReadAll(resp.Body) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | return bodyBytes, nil 108 | } 109 | 110 | // SendGetRequestWithToken sends an HTTP GET request to the FireFly API with a token. 111 | func (fc *FireFlyHttpClient) SendGetRequestWithToken(url, token string) ([]byte, error) { 112 | return fc.sendRequestWithToken(http.MethodGet, url, token, nil) 113 | } 114 | 115 | // SendPutRequestWithToken sends an HTTP PUT request to the FireFly API with a token and data. 116 | func (fc *FireFlyHttpClient) SendPutRequestWithToken(url, token string, data []byte) ([]byte, error) { 117 | return fc.sendRequestWithToken(http.MethodPut, url, token, data) 118 | } 119 | 120 | func (fc *FireFlyHttpClient) UpdateTransactionCategory(id, trans_id, category string, tags []string) error { 121 | //log.Printf("updating transaction: %s", id) 122 | 123 | trn := FireFlyTransactions{ 124 | FireWebHooks: false, 125 | Id: id, 126 | Transactions: []FireFlyTransaction{ 127 | { 128 | TransactionID: trans_id, 129 | Category: category, 130 | Tags: append(tags, []string{"ffiiitc"}...), 131 | }, 132 | }, 133 | } 134 | 135 | //log.Printf("trn data: %v", trn) 136 | fc.logger.Logf("DEBUG trn data: %v", trn) 137 | 138 | jsonData, err := json.Marshal(trn) 139 | if err != nil { 140 | //log.Fatalf("Error marshaling JSON data: %v", err) 141 | return err 142 | } 143 | 144 | var prettyJSON bytes.Buffer 145 | json.Indent(&prettyJSON, jsonData, "", " ") 146 | //log.Printf("json sent: %s", prettyJSON.String()) 147 | fc.logger.Logf("DEBUG json sent: %s", prettyJSON.String()) 148 | 149 | res, err := fc.SendPutRequestWithToken( 150 | fmt.Sprintf("%s/%s/transactions/%s", fc.AppURL, fireflyAPIPrefix, id), 151 | fc.Token, 152 | jsonData, 153 | ) 154 | 155 | if err != nil { 156 | return err 157 | } 158 | 159 | //debug 160 | prettyJSON = bytes.Buffer{} 161 | json.Indent(&prettyJSON, res, "", " ") 162 | //log.Println(prettyJSON.String()) 163 | fc.logger.Logf("DEBUG json received: %s", prettyJSON.String()) 164 | 165 | return nil 166 | } 167 | 168 | func buildCategoryDescriptionSlice(data FireFlyTransactionsResponse) []string { 169 | var res []string 170 | for _, value := range data.Data { 171 | for _, trnval := range value.Attributes.Transactions { 172 | trn := fmt.Sprintf("%s,%s", trnval.Category, trnval.Description) 173 | res = append(res, trn) 174 | } 175 | } 176 | return res 177 | } 178 | 179 | func buildTransactionsDataset(data FireFlyTransactionsResponse) [][]string { 180 | var res [][]string 181 | for _, value := range data.Data { 182 | for _, trnval := range value.Attributes.Transactions { 183 | //trn := fmt.Sprintf("%s,%s", trnval.Category, trnval.Description) 184 | trn := []string{trnval.Category, trnval.Description} 185 | res = append(res, trn) 186 | } 187 | } 188 | return res 189 | } 190 | 191 | // get all transactions 192 | // returns slice of strings "transaction description, category" 193 | func (fc *FireFlyHttpClient) GetTransactions() ([]string, error) { 194 | var pageIndex int 195 | //log.Println("get first page of transactions") 196 | fc.logger.Logf("INFO get first page of transactions") 197 | res, err := fc.SendGetRequestWithToken( 198 | fmt.Sprintf("%s/api/v1/transactions?page=1", fc.AppURL), 199 | fc.Token, 200 | ) 201 | if err != nil { 202 | return nil, err 203 | } 204 | 205 | var data FireFlyTransactionsResponse 206 | err = json.Unmarshal(res, &data) 207 | if err != nil { 208 | return nil, err 209 | } 210 | resSlice := buildCategoryDescriptionSlice(data) 211 | 212 | //log.Printf("transactions total pages: %d", data.Meta.Pagination.TotalPages) 213 | fc.logger.Logf("INFO transactions total pages: %d", data.Meta.Pagination.TotalPages) 214 | if data.Meta.Pagination.TotalPages > 1 { 215 | //log.Println("transactions more than 1 page available. iterating") 216 | fc.logger.Logf("INFO transactions more than 1 page available. iterating") 217 | pageIndex = 2 218 | for pageIndex <= data.Meta.Pagination.TotalPages { 219 | res, err := fc.SendGetRequestWithToken( 220 | fmt.Sprintf("%s/%s/transactions?page=%d", fc.AppURL, fireflyAPIPrefix, pageIndex), 221 | fc.Token, 222 | ) 223 | if err != nil { 224 | return nil, err 225 | } 226 | var data FireFlyTransactionsResponse 227 | err = json.Unmarshal(res, &data) 228 | if err != nil { 229 | return nil, err 230 | } 231 | resSlice = append(resSlice, buildCategoryDescriptionSlice(data)...) 232 | //log.Printf("page %d...", pageIndex) 233 | fc.logger.Logf("INFO page %d...", pageIndex) 234 | pageIndex++ 235 | } 236 | } 237 | // return transactions, err 238 | return resSlice, err 239 | } 240 | 241 | func (fc *FireFlyHttpClient) GetTransactionsDataset(startStr, endStr string) ([][]string, error) { 242 | var pageIndex int 243 | fc.logger.Logf("INFO get first page of transactions") 244 | dateRangeQuery := "" 245 | if startStr != "" { 246 | _, err := time.Parse("2006-01-02", startStr) 247 | if err == nil { 248 | dateRangeQuery = fmt.Sprintf("&start=%s", startStr) 249 | fc.logger.Logf("DEBUG start date: %s", startStr) 250 | } else { 251 | fc.logger.Logf("WARN invalid start date format: %v", err) 252 | } 253 | } 254 | 255 | if endStr != "" { 256 | _, err := time.Parse("2006-01-02", endStr) 257 | if err == nil { 258 | dateRangeQuery += fmt.Sprintf("&end=%s", endStr) 259 | fc.logger.Logf("DEBUG end date: %s", endStr) 260 | } else { 261 | fc.logger.Logf("WARN invalid end date format: %v", err) 262 | } 263 | } 264 | 265 | res, err := fc.SendGetRequestWithToken( 266 | fmt.Sprintf("%s/%s/transactions?page=1%s", fc.AppURL, fireflyAPIPrefix, dateRangeQuery), 267 | fc.Token, 268 | ) 269 | if err != nil { 270 | return nil, err 271 | } 272 | 273 | var data FireFlyTransactionsResponse 274 | err = json.Unmarshal(res, &data) 275 | if err != nil { 276 | return nil, err 277 | } 278 | fc.logger.Logf("DEBUG raw transactions data: %v", data) 279 | 280 | resSlice := buildTransactionsDataset(data) 281 | 282 | fc.logger.Logf("INFO transactions total pages: %d", data.Meta.Pagination.TotalPages) 283 | if data.Meta.Pagination.TotalPages > 1 { 284 | fc.logger.Logf("INFO transactions more than 1 page available. iterating") 285 | pageIndex = 2 286 | for pageIndex <= data.Meta.Pagination.TotalPages { 287 | res, err := fc.SendGetRequestWithToken( 288 | fmt.Sprintf("%s/%s/transactions?page=%d%s", fc.AppURL, fireflyAPIPrefix, pageIndex, dateRangeQuery), 289 | fc.Token, 290 | ) 291 | if err != nil { 292 | return nil, err 293 | } 294 | var data FireFlyTransactionsResponse 295 | err = json.Unmarshal(res, &data) 296 | if err != nil { 297 | return nil, err 298 | } 299 | resSlice = append(resSlice, buildTransactionsDataset(data)...) 300 | fc.logger.Logf("INFO page %d...", pageIndex) 301 | pageIndex++ 302 | } 303 | } 304 | return resSlice, err 305 | } 306 | -------------------------------------------------------------------------------- /internal/handlers/handler.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "encoding/json" 5 | "ffiiitc/internal/classifier" 6 | "ffiiitc/internal/config" 7 | "ffiiitc/internal/firefly" 8 | 9 | "net/http" 10 | "strconv" 11 | 12 | "github.com/go-pkgz/lgr" 13 | ) 14 | 15 | type WebHookHandler struct { 16 | Classifier *classifier.TrnClassifier 17 | FireflyClient *firefly.FireFlyHttpClient 18 | Logger *lgr.Logger 19 | } 20 | 21 | // structs to handle payload from new transaction web hook 22 | type FireflyTrn struct { 23 | Id string `json:"transaction_journal_id"` 24 | Description string `json:"description"` 25 | Category string `json:"category_name"` 26 | Tags []string `json:"tags"` 27 | } 28 | 29 | type FireFlyContent struct { 30 | Id int64 `json:"id"` 31 | Transactions []FireflyTrn `json:"transactions"` 32 | } 33 | 34 | type FireflyWebHook struct { 35 | Content FireFlyContent `json:"content"` 36 | } 37 | 38 | func NewWebHookHandler(c *classifier.TrnClassifier, f *firefly.FireFlyHttpClient, l *lgr.Logger) *WebHookHandler { 39 | return &WebHookHandler{ 40 | Classifier: c, 41 | FireflyClient: f, 42 | Logger: l, 43 | } 44 | } 45 | 46 | // http handler for new transaction 47 | func (wh *WebHookHandler) HandleNewTransactionWebHook(w http.ResponseWriter, r *http.Request) { 48 | 49 | // only allow post method 50 | if r.Method != http.MethodPost { 51 | http.Error(w, "bad request", http.StatusBadRequest) 52 | return 53 | } 54 | 55 | // decode payload 56 | decoder := json.NewDecoder(r.Body) 57 | var hookData FireflyWebHook 58 | err := decoder.Decode(&hookData) 59 | if err != nil { 60 | wh.Logger.Logf("ERROR decoding webhook payload: %v", err) 61 | http.Error(w, "bad data", http.StatusBadRequest) 62 | return 63 | } 64 | 65 | // perform classification 66 | for _, trn := range hookData.Content.Transactions { 67 | wh.Logger.Logf( 68 | "INFO hook new trn: received (id: %v) (description: %s)", 69 | hookData.Content.Id, 70 | trn.Description, 71 | ) 72 | cat := wh.Classifier.ClassifyTransaction(trn.Description) 73 | wh.Logger.Logf("INFO hook new trn: classified (id: %v) (category: %s)", hookData.Content.Id, cat) 74 | err = wh.FireflyClient.UpdateTransactionCategory(strconv.FormatInt(hookData.Content.Id, 10), trn.Id, cat, trn.Tags) 75 | if err != nil { 76 | wh.Logger.Logf("ERROR hook new trn: error updating (id: %v) %v", hookData.Content.Id, err) 77 | } 78 | wh.Logger.Logf("INFO hook new trn: updated (id: %v)", hookData.Content.Id) 79 | 80 | } 81 | w.WriteHeader(http.StatusOK) 82 | } 83 | 84 | // http handler for forcing to train model 85 | func (wh *WebHookHandler) HandleForceTrainingModel(w http.ResponseWriter, r *http.Request) { 86 | 87 | // only allow get method 88 | if r.Method != http.MethodGet { 89 | http.Error(w, "bad request", http.StatusBadRequest) 90 | return 91 | } 92 | 93 | query := r.URL.Query() 94 | startStr := query.Get("start") 95 | endStr := query.Get("end") 96 | 97 | wh.Logger.Logf("INFO Received request to perform force training") 98 | wh.Logger.Logf("INFO Requesting transactions data from Firefly") 99 | trnDataset, err := wh.FireflyClient.GetTransactionsDataset(startStr, endStr) 100 | if err != nil || len(trnDataset) == 0 { 101 | wh.Logger.Logf("ERROR: Error while getting transactions data\n %v", err) 102 | } else { 103 | wh.Logger.Logf("DEBUG Got training data\n %v", trnDataset) 104 | cls, err := classifier.NewTrnClassifierWithTraining(trnDataset, wh.Logger) 105 | if err != nil { 106 | wh.Logger.Logf("ERROR creating classifier from dataset:\n %v", err) 107 | } else { 108 | wh.Logger.Logf("INFO forced training completed...") 109 | wh.Logger.Logf("INFO saving data to model...") 110 | err = cls.SaveClassifierToFile(config.ModelFile) 111 | if err != nil { 112 | wh.Logger.Logf("ERROR saving model to file:\n %v", err) 113 | } else { 114 | wh.Logger.Logf("INFO: forced training completed and model saved. Please restart 'ffiiitc' container") 115 | } 116 | } 117 | } 118 | 119 | w.WriteHeader(http.StatusOK) 120 | } 121 | 122 | // http handler for updated transaction 123 | // func (wh *WebHookHandler) HandleUpdateTransactionWebHook(w http.ResponseWriter, r *http.Request) { 124 | 125 | // // only allow post method 126 | // if r.Method != http.MethodPost { 127 | // http.Error(w, "bad request", http.StatusBadRequest) 128 | // return 129 | // } 130 | 131 | // // decode payload 132 | // decoder := json.NewDecoder(r.Body) 133 | // var hookData FireflyWebHook 134 | // err := decoder.Decode(&hookData) 135 | // if err != nil { 136 | // http.Error(w, "bad data", http.StatusBadRequest) 137 | // return 138 | // } 139 | 140 | // // perform training 141 | // for _, trn := range hookData.Content.Transactions { 142 | // wh.Logger.Logf( 143 | // "hook update trn: received (id: %v) (desc: %s) (cat: %s)", 144 | // trn.Id, 145 | // trn.Description, 146 | // trn.Category, 147 | // ) 148 | 149 | // if trn.Category != "" { 150 | // err := wh.Classifier.Train(trn.Description, trn.Category) 151 | // if err != nil { 152 | // wh.Logger.Logf("hook update trn: error updating model: %v", err) 153 | // } 154 | // wh.Logger.Logf( 155 | // "hook update trn: (cat: %s) (features: %v)", 156 | // trn.Category, 157 | // wh.Classifier.Classifier.WordsByClass(bayesian.Class(trn.Category)), 158 | // ) 159 | // } else { 160 | // wh.Logger.Logf("skip training. Category is empty") 161 | // } 162 | 163 | // // } 164 | // w.WriteHeader(http.StatusOK) 165 | // } 166 | -------------------------------------------------------------------------------- /internal/router/router.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | ) 8 | 9 | type Router struct { 10 | Mux *http.ServeMux 11 | } 12 | 13 | func NewRouter() *Router { 14 | return &Router{ 15 | Mux: http.NewServeMux(), 16 | } 17 | } 18 | 19 | func (r *Router) AddRoute(pattern string, handler func(w http.ResponseWriter, r *http.Request)) { 20 | r.Mux.HandleFunc(pattern, handler) 21 | } 22 | 23 | func (r *Router) logRoute(handler http.Handler) http.Handler { 24 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 25 | log.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL) 26 | handler.ServeHTTP(w, r) 27 | }) 28 | } 29 | 30 | func (r *Router) Run(port int) error { 31 | ps := fmt.Sprintf(":%d", port) 32 | return http.ListenAndServe(ps, r.logRoute(r.Mux)) 33 | } 34 | -------------------------------------------------------------------------------- /internal/router/router_test.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestRouter(t *testing.T) { 13 | 14 | router := NewRouter() 15 | 16 | // Create a test HTTP server 17 | server := httptest.NewServer(router.Mux) 18 | defer server.Close() 19 | 20 | t.Run("AddRoute", func(t *testing.T) { 21 | // Set up a test route 22 | router.AddRoute("/test", func(w http.ResponseWriter, r *http.Request) { 23 | w.WriteHeader(http.StatusOK) 24 | w.Write([]byte("Test Response")) 25 | }) 26 | 27 | // Perform a request to the test route 28 | resp, err := http.Get(server.URL + "/test") 29 | assert.NoError(t, err) 30 | defer resp.Body.Close() 31 | 32 | // Check if the response status code is 200 33 | assert.Equal(t, http.StatusOK, resp.StatusCode) 34 | 35 | // Check if the response body is correct 36 | expectedBody := "Test Response" 37 | body, err := ioutil.ReadAll(resp.Body) 38 | assert.NoError(t, err) 39 | assert.Equal(t, expectedBody, string(body)) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "ffiiitc/internal/classifier" 5 | "ffiiitc/internal/config" 6 | "ffiiitc/internal/firefly" 7 | "ffiiitc/internal/handlers" 8 | "ffiiitc/internal/router" 9 | 10 | "github.com/go-pkgz/lgr" 11 | ) 12 | 13 | func main() { 14 | 15 | // make logger 16 | l := lgr.New(lgr.Debug, lgr.CallerFunc) 17 | l.Logf("INFO Firefly transaction classification started") 18 | 19 | // get the config 20 | l.Logf("INFO getting config") 21 | cfg, err := config.NewConfig(l) 22 | if err != nil { 23 | l.Logf("FATAL getting config: %v", err) 24 | } 25 | 26 | // make firefly http client for rest api 27 | fc := firefly.NewFireFlyHttpClient(cfg.FFApp, cfg.APIKey, config.FireflyAppTimeout, l) 28 | 29 | // make classifier 30 | // on first run, classifier will take all your 31 | // transactions and learn their categories 32 | // subsequent start classifier will load trained model from file 33 | l.Logf("INFO loading classifier from model: %s", config.ModelFile) 34 | cls, err := classifier.NewTrnClassifierFromFile(config.ModelFile, l) 35 | if err != nil { 36 | l.Logf("ERROR %v", err) 37 | l.Logf("INFO looks like we need to do some training...") 38 | // get transactions in data set 39 | //[ [cat, trn description], [cat, trn description]... ] 40 | // Empty string for start and end date means all transactions 41 | trnDataset, err := fc.GetTransactionsDataset("", "") 42 | l.Logf("DEBUG data set:\n %v", trnDataset) 43 | 44 | if err != nil { 45 | l.Logf("FATAL: unable to get list of transactions %v", err) 46 | } 47 | 48 | // byesian package requires at least 2 transactions with different categories to start training 49 | 50 | // we fail if no transactions in Firefly 51 | if len(trnDataset) == 0 { 52 | l.Logf("FATAL: no transactions in Firefly. At least 2 manually categorised transactions with different categories are required %v", trnDataset) 53 | } 54 | 55 | // we also check for at least 2 different categories available if transactions exist 56 | categories := make(map[string]int) 57 | for i, data := range trnDataset { 58 | if len(data) == 0 { 59 | l.Logf("WARN skipping empty transaction data at index %d", i) 60 | continue 61 | } 62 | category := data[0] 63 | if category == "" { 64 | l.Logf("WARN skipping transaction with empty category at index %d", i) 65 | continue 66 | } 67 | categories[category]++ 68 | } 69 | 70 | l.Logf("INFO found %d different categories: %v", len(categories), categories) 71 | 72 | if len(categories) < 2 { 73 | l.Logf("FATAL: classifier needs at least 2 different categories in transactions, got %d. Categories found: %v", 74 | len(categories), 75 | categories) 76 | return 77 | } 78 | 79 | cls, err = classifier.NewTrnClassifierWithTraining(trnDataset, l) 80 | if err != nil { 81 | l.Logf("FATAL: %v", err) 82 | } 83 | l.Logf("INFO training completed...") 84 | err = cls.SaveClassifierToFile(config.ModelFile) 85 | if err != nil { 86 | l.Logf("FATAL: %v", err) 87 | } 88 | l.Logf("INFO classifier saved to: %s", config.ModelFile) 89 | } 90 | 91 | l.Logf("DEBUG learned classes: %v", cls.Classifier.Classes) 92 | 93 | // init handlers 94 | h := handlers.NewWebHookHandler(cls, fc, l) 95 | 96 | // init router 97 | r := router.NewRouter() 98 | 99 | // add handlers 100 | r.AddRoute("/classify", h.HandleNewTransactionWebHook) 101 | r.AddRoute("/train", h.HandleForceTrainingModel) 102 | // temporary remove this handle 103 | //r.AddRoute("/learn", h.HandleUpdateTransactionWebHook) 104 | 105 | //run 106 | err = r.Run(8080) 107 | if err != nil { 108 | panic(err) 109 | } 110 | } 111 | --------------------------------------------------------------------------------