├── Setup.hs ├── .ghci ├── bin ├── build-and-test ├── show-latest-release-info ├── show-release-downloads ├── tag-release ├── time-check ├── fetch-latest-hledger └── release-tarball ├── step-by-step ├── img │ └── mopane-worm-meal.jpg ├── README.org ├── Downloads │ └── Bogart │ │ ├── 123456789_2016-04-28.csv │ │ ├── 123456789_2016-05-28.csv │ │ └── 123456789_2016-03-30.csv ├── part3.org ├── part2.org └── part1.org ├── .gitignore ├── test ├── CSVImport │ ├── Unit.hs │ ├── ImportHelperTests.hs │ └── Integration.hs ├── Spec.hs ├── PathHelpers │ └── Unit.hs ├── Common │ ├── Integration.hs │ └── Unit.hs ├── TestHelpersTurtle.hs ├── TestHelpers.hs └── BaseDir │ └── Integration.hs ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── src └── Hledger │ └── Flow │ ├── DateTime.hs │ ├── DocHelpers.hs │ ├── Import │ ├── Types.hs │ ├── ImportHelpers.hs │ ├── ImportHelpersTurtle.hs │ └── CSVImport.hs │ ├── Types.hs │ ├── RuntimeOptions.hs │ ├── Internals.hs │ ├── Logging.hs │ ├── PathHelpers.hs │ ├── BaseDir.hs │ ├── Reports.hs │ └── Common.hs ├── stack.yaml.lock ├── TODO.org ├── app ├── Parsing.hs └── Main.hs ├── package.yaml ├── stack.yaml ├── .circleci └── config.yml ├── CONTRIBUTING.org ├── README.org ├── ChangeLog.md └── docs └── README.org /Setup.hs: -------------------------------------------------------------------------------- 1 | import Distribution.Simple 2 | main = defaultMain 3 | -------------------------------------------------------------------------------- /.ghci: -------------------------------------------------------------------------------- 1 | :set -XOverloadedStrings 2 | :set prompt "\ESC[1;34m%s\n\ESC[0;34mλ> \ESC[m" 3 | import Turtle 4 | -------------------------------------------------------------------------------- /bin/build-and-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | stack test --interleaved-output && stack install 6 | -------------------------------------------------------------------------------- /bin/show-latest-release-info: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | curl https://api.github.com/repos/apauley/hledger-flow/releases/latest 4 | -------------------------------------------------------------------------------- /step-by-step/img/mopane-worm-meal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apauley/hledger-flow/HEAD/step-by-step/img/mopane-worm-meal.jpg -------------------------------------------------------------------------------- /bin/show-release-downloads: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | curl https://api.github.com/repos/apauley/hledger-flow/releases | jq '.[].assets[] | .name, .browser_download_url, .download_count' 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .stack-work/ 2 | dist/ 3 | dist-newstyle/ 4 | hledger-flow.cabal 5 | *~ 6 | scratch 7 | scratch.org 8 | docs/my-finances 9 | releases/ 10 | *.c.* 11 | .idea 12 | *.iml 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /test/CSVImport/Unit.hs: -------------------------------------------------------------------------------- 1 | module CSVImport.Unit where 2 | 3 | import qualified CSVImport.ImportHelperTests 4 | import qualified CSVImport.ImportHelperTurtleTests 5 | import Test.HUnit 6 | 7 | tests :: Test 8 | tests = TestList [CSVImport.ImportHelperTests.tests, CSVImport.ImportHelperTurtleTests.tests] 9 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | After reading the text below, please replace it with your own PR description. 2 | 3 | **Thank you for contributing to hledger-flow!** 4 | 5 | There are some helpful information in our [Contibutor's Guide](https://github.com/apauley/hledger-flow/blob/master/CONTRIBUTING.org) 6 | 7 | Please check it out! 8 | -------------------------------------------------------------------------------- /src/Hledger/Flow/DateTime.hs: -------------------------------------------------------------------------------- 1 | module Hledger.Flow.DateTime where 2 | 3 | import Data.Time.Calendar 4 | import Data.Time.Clock 5 | 6 | currentDate :: IO (Integer, Int, Int) -- :: (year,month,day) 7 | currentDate = toGregorian . utctDay <$> getCurrentTime 8 | 9 | currentYear :: IO Integer 10 | currentYear = do 11 | (y, _, _) <- currentDate 12 | return y 13 | -------------------------------------------------------------------------------- /src/Hledger/Flow/DocHelpers.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module Hledger.Flow.DocHelpers where 4 | 5 | import qualified Data.Text as T (Text) 6 | import Turtle ((%)) 7 | import qualified Turtle as Turtle (Line, format, l) 8 | 9 | docURL :: Turtle.Line -> T.Text 10 | docURL = Turtle.format ("https://github.com/apauley/hledger-flow/tree/master/docs#" % Turtle.l) 11 | -------------------------------------------------------------------------------- /bin/tag-release: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | BASEDIR="$(pwd)" 6 | 7 | GITHASH="$(git log -1 --format=tformat:%H)" 8 | PACKAGE_VERSION="$(grep '^version:' ${BASEDIR}/package.yaml|awk '{print $2}')" 9 | VERSION="v${PACKAGE_VERSION}" 10 | MSG=$(echo -e "Release version ${PACKAGE_VERSION}\n\nSee ChangeLog for details - https://github.com/apauley/hledger-flow/blob/master/ChangeLog.md") 11 | 12 | git tag --sign --message="${MSG}" ${VERSION} ${GITHASH} 13 | -------------------------------------------------------------------------------- /stack.yaml.lock: -------------------------------------------------------------------------------- 1 | # This file was autogenerated by Stack. 2 | # You should not edit this file by hand. 3 | # For more information, please see the documentation at: 4 | # https://docs.haskellstack.org/en/stable/topics/lock_files 5 | 6 | packages: [] 7 | snapshots: 8 | - completed: 9 | sha256: d347039f81388e16ea93ddaf9ff1850abfba8f8680ff75fbdd177692542ceb26 10 | size: 726286 11 | url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/24/8.yaml 12 | original: lts-24.8 13 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Import/Types.hs: -------------------------------------------------------------------------------- 1 | module Hledger.Flow.Import.Types where 2 | 3 | import qualified Data.Map.Strict as Map 4 | import Hledger.Flow.PathHelpers (RelFile, TurtlePath) 5 | 6 | type TurtleFileBundle = Map.Map TurtlePath [TurtlePath] 7 | 8 | type InputFileBundle = Map.Map RelFile [RelFile] 9 | 10 | data ImportDirs = ImportDirs 11 | { importDir :: TurtlePath, 12 | ownerDir :: TurtlePath, 13 | bankDir :: TurtlePath, 14 | accountDir :: TurtlePath, 15 | stateDir :: TurtlePath, 16 | yearDir :: TurtlePath 17 | } 18 | deriving (Show) 19 | -------------------------------------------------------------------------------- /step-by-step/README.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | 3 | * Hledger Flow: Step By Step 4 | 5 | This is a series of step-by-step instructions. 6 | 7 | They are intended to be read in sequence: 8 | 9 | 1. [[file:part1.org][Part 1: The First CSV Statement]] 10 | 2. [[file:part2.org][Part 2: An hledger report]] 11 | 3. [[file:part3.org][Part 3: Adding More Statements]] 12 | 13 | You can see the example imported financial transactions as it was generated by the step-by-step 14 | instructions here: 15 | 16 | https://github.com/apauley/hledger-flow-example 17 | -------------------------------------------------------------------------------- /TODO.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: content 2 | 3 | * The TODO List 4 | ** Generate reports 5 | - Monthly reports for the past 6 months 6 | - Quarterly reports for the past year 7 | - Yearly reports for the past 5 years 8 | - How much money went towards expenses, assets and liabilities, expressed as a percentage of income 9 | - My personal inflation: percentage increase in expenses per year 10 | - Overall reports, and reports per owner 11 | - More useful reports? 12 | ** Performance 13 | - Don't re-process statements that have already been processed 14 | ** Documentation 15 | - Expand the step-by-step instructions with more scenarios 16 | -------------------------------------------------------------------------------- /test/Spec.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module Main where 5 | 6 | import qualified BaseDir.Integration 7 | import qualified CSVImport.Integration 8 | import qualified CSVImport.Unit 9 | import qualified Common.Integration 10 | import qualified Common.Unit 11 | import qualified PathHelpers.Unit 12 | import Test.HUnit 13 | import Turtle 14 | 15 | tests :: Test 16 | tests = TestList [Common.Unit.tests, Common.Integration.tests, PathHelpers.Unit.tests, BaseDir.Integration.tests, CSVImport.Unit.tests, CSVImport.Integration.tests] 17 | 18 | main :: IO Counts 19 | main = do 20 | errCounts <- runTestTT tests 21 | if (errors errCounts > 0 || failures errCounts > 0) 22 | then exit $ ExitFailure 1 23 | else return errCounts 24 | -------------------------------------------------------------------------------- /app/Parsing.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE PartialTypeSignatures #-} 2 | 3 | module Parsing ( 4 | parseStartYear 5 | ) where 6 | 7 | import Hledger.Flow.DateTime 8 | import System.Exit (die) 9 | import Text.Read (readMaybe) 10 | 11 | parseStartYear :: Maybe String -> IO (Maybe Integer) 12 | parseStartYear y = case y of 13 | Nothing -> return Nothing 14 | Just "current" -> Just <$> currentYear 15 | Just s -> Just <$> parseInt s "Unable to parse year" 16 | 17 | parseInt :: String -> String -> IO Integer 18 | parseInt s errPrefix = case safeParseInt s errPrefix of 19 | Right i -> return i 20 | Left err -> die err 21 | 22 | safeParseInt :: String -> String -> Either String Integer 23 | safeParseInt s errPrefix = case (readMaybe s :: Maybe Integer) of 24 | Nothing -> Left $ errPrefix ++ " " ++ s 25 | Just i -> Right i 26 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: ['https://explorer.cardano.org/en/address?address=addr1qy5hlk7a5cmrr9khnvtmufsa053f4kvc6t3x9f8cer4hzzwnayxnuy0ytxkqm36sumwvl7jsqhvazmup2xn7hrcjfvrq9q5gxj'] # Replace with up to 4 custom sponsorship URLs 13 | -------------------------------------------------------------------------------- /test/PathHelpers/Unit.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE QuasiQuotes #-} 2 | 3 | module PathHelpers.Unit where 4 | 5 | import Hledger.Flow.PathHelpers 6 | import Path 7 | import Test.HUnit 8 | 9 | testPathSize :: Test 10 | testPathSize = 11 | TestCase 12 | ( do 13 | let d0 = [reldir|.|] 14 | let d1 = [reldir|d1|] 15 | let d1ond0 = d0 [reldir|d1|] 16 | let d2 = d1 [reldir|d2|] 17 | let d3 = d2 [reldir|d3|] 18 | assertEqual "Calculate the path size correctly" 0 (pathSize d0) 19 | assertEqual "Calculate the path size correctly" 1 (pathSize d1) 20 | assertEqual "Calculate the path size correctly" 1 (pathSize d1ond0) 21 | assertEqual "Calculate the path size correctly" 2 (pathSize d2) 22 | assertEqual "Calculate the path size correctly" 3 (pathSize d3) 23 | ) 24 | 25 | tests :: Test 26 | tests = TestList [testPathSize] 27 | -------------------------------------------------------------------------------- /bin/time-check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BASEDIRS="$@" 4 | 5 | date 6 | echo -e "A simple script to get a rough idea of execution time." 7 | echo -e "Each script argument is a base directory.\n" 8 | 9 | hledger-flow --version|head -n 1 10 | echo 11 | 12 | for bd in ${BASEDIRS} 13 | do 14 | DIRNAME=$(basename ${bd}) 15 | echo "$(date) BEGIN: ${DIRNAME}" 16 | for subcommand in $(echo "import report") 17 | do 18 | /usr/bin/time --format="${DIRNAME} ${subcommand} sequential real %es" hledger-flow --sequential ${subcommand} ${bd} 2>&1 | grep 'Generated.*reports\|Imported.*journals\|real' 19 | echo 20 | /usr/bin/time --format="${DIRNAME} ${subcommand} parallel real %es" hledger-flow ${subcommand} ${bd} 2>&1 | grep 'Generated.*reports\|Imported.*journals\|real' 21 | echo 22 | done 23 | echo -e "$(date) END: ${DIRNAME}\n" 24 | done 25 | 26 | echo "***" 27 | -------------------------------------------------------------------------------- /step-by-step/Downloads/Bogart/123456789_2016-04-28.csv: -------------------------------------------------------------------------------- 1 | 2,123456789,'MNR GAWIE DE GROOT','BOGART TJEKREKENING' 2 | 3,,'Staat' 3 | 3,'Staatnommer','Vanaf Datum','Tot Datum' 4 | 3,75,'29 Maart 2016','27 April 2016' 5 | 4,,'Opsomming' 6 | 5,,'Transaksies' 7 | 5,'Nommer','Datum','Beskrywing1','Beskrywing2','Beskrywing3','Bedrag','Saldo','Opgeloopte Koste' 8 | 5,1,'29 Mrt',"#Monthly Bank Fee",,"",-550.23,40471.62, 9 | 5,2,'01 Apr',"Gereeld Bet Na","Rent","",-5000.00,35471.62, 10 | 5,3,'01 Apr',"Gereeld Bet Na","MyBTCEx","",-2500.00,32971.62, 11 | 5,4,'02 Apr',"Magband Debiet","Traditional Boring Investments","",-2500,30471.62, 12 | 5,5,'02 Apr',"Magband Debiet","All-in-one Insurance","",-5234.43,25237.19, 13 | 5,6,'05 Apr',"Gereeld Bet Na","My Charity","",-4500.00,20737.19, 14 | 5,7,'10 Apr',"Smart-ap Transfer To","Credit Card Account","",-15000.00,5737.19, 15 | 5,8,'15 Apr',"ATM","Cash Withdrawal","",-5000.00,737.19, 16 | 5,9,'25 Apr',"Debiet Order Krediet","Grillerige Salary",,37256.28,37993.47, 17 | 6,'END' 18 | -------------------------------------------------------------------------------- /step-by-step/Downloads/Bogart/123456789_2016-05-28.csv: -------------------------------------------------------------------------------- 1 | 2,123456789,'MNR GAWIE DE GROOT','BOGART TJEKREKENING' 2 | 3,,'Staat' 3 | 3,'Staatnommer','Vanaf Datum','Tot Datum' 4 | 3,75,'28 April 2016','27 Mei 2016' 5 | 4,,'Opsomming' 6 | 5,,'Transaksies' 7 | 5,'Nommer','Datum','Beskrywing1','Beskrywing2','Beskrywing3','Bedrag','Saldo','Opgeloopte Koste' 8 | 5,1,'29 Apr',"#Monthly Bank Fee",,"",-599.06,37394.41, 9 | 5,2,'01 Mei',"Gereeld Bet Na","Rent","",-5000.00,32394.41, 10 | 5,3,'01 Mei',"Gereeld Bet Na","MyBTCEx","",-2500.00,29894.41, 11 | 5,4,'02 Mei',"Magband Debiet","Traditional Boring Investments","",-2500,27394.41, 12 | 5,5,'02 Mei',"Magband Debiet","All-in-one Insurance","",-5234.43,22159.98, 13 | 5,6,'05 Mei',"Gereeld Bet Na","My Charity","",-4000.00,18159.98, 14 | 5,7,'10 Mei',"Smart-ap Transfer To","Credit Card Account","",-14000.00,4159.98, 15 | 5,8,'15 Mei',"ATM","Cash Withdrawal","",-4000.00,159.98, 16 | 5,9,'25 Mei',"Debiet Order Krediet","Grillerige Salary",,37256.28,37416.26, 17 | 6,'END' 18 | -------------------------------------------------------------------------------- /step-by-step/Downloads/Bogart/123456789_2016-03-30.csv: -------------------------------------------------------------------------------- 1 | 2,123456789,'MNR GAWIE DE GROOT','BOGART TJEKREKENING' 2 | 3,,'Staat' 3 | 3,'Staatnommer','Vanaf Datum','Tot Datum' 4 | 3,75,'28 Februarie 2016','28 Maart 2016' 5 | 4,,'Opsomming' 6 | 5,,'Transaksies' 7 | 5,'Nommer','Datum','Beskrywing1','Beskrywing2','Beskrywing3','Bedrag','Saldo','Opgeloopte Koste' 8 | 5,1,'01 Mrt',"#Monthly Bank Fee",,"",-500.00,40000.00, 9 | 5,2,'01 Mrt',"Gereeld Bet Na","Rent","",-5000.00,35000.00, 10 | 5,3,'01 Mrt',"Gereeld Bet Na","MyBTCEx","",-2500.00,32500.00, 11 | 5,4,'02 Mrt',"Magband Debiet","Traditional Boring Investments","",-2500,30000.00, 12 | 5,5,'02 Mrt',"Magband Debiet","All-in-one Insurance","",-5234.43,24765.57, 13 | 5,6,'05 Mrt',"Gereeld Bet Na","My Charity","",-4000.00,20765.57, 14 | 5,7,'10 Mrt',"Smart-ap Transfer To","Credit Card Account","",-14000.00,6765.57, 15 | 5,8,'15 Mrt',"ATM","Cash Withdrawal","",-3000.00,3765.57, 16 | 5,9,'25 Mrt',"Debiet Order Krediet","Grillerige Salary",,37256.28,41021.85, 17 | 6,'END' 18 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Types.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE FlexibleInstances #-} 2 | 3 | module Hledger.Flow.Types where 4 | 5 | import qualified Data.Text as T 6 | import Hledger.Flow.PathHelpers 7 | import qualified Turtle (ExitCode, Line, NominalDiffTime, Shell) 8 | 9 | type BaseDir = AbsDir 10 | 11 | type RunDir = RelDir 12 | 13 | data LogMessage = StdOut T.Text | StdErr T.Text | Terminate deriving (Show) 14 | 15 | type FullOutput = (Turtle.ExitCode, T.Text, T.Text) 16 | 17 | type FullTimedOutput = (FullOutput, Turtle.NominalDiffTime) 18 | 19 | type ProcFun = T.Text -> [T.Text] -> Turtle.Shell Turtle.Line -> IO FullOutput 20 | 21 | type ProcInput = (T.Text, [T.Text], Turtle.Shell Turtle.Line) 22 | 23 | data HledgerInfo = HledgerInfo 24 | { hlPath :: AbsFile, 25 | hlVersion :: T.Text 26 | } 27 | deriving (Show) 28 | 29 | class HasVerbosity a where 30 | verbose :: a -> Bool 31 | 32 | class HasBaseDir a where 33 | baseDir :: a -> BaseDir 34 | 35 | class HasRunDir a where 36 | importRunDir :: a -> RunDir 37 | 38 | class HasSequential a where 39 | sequential :: a -> Bool 40 | 41 | class HasBatchSize a where 42 | batchSize :: a -> Int 43 | 44 | class HasPrettyReports a where 45 | prettyReports :: a -> Bool 46 | -------------------------------------------------------------------------------- /src/Hledger/Flow/RuntimeOptions.hs: -------------------------------------------------------------------------------- 1 | module Hledger.Flow.RuntimeOptions where 2 | 3 | import qualified Data.Text as T 4 | import Hledger.Flow.Internals (SystemInfo) 5 | import Hledger.Flow.Types 6 | import Prelude hiding (putStrLn) 7 | 8 | data RuntimeOptions = RuntimeOptions 9 | { baseDir :: BaseDir, 10 | importRunDir :: RunDir, 11 | importStartYear :: Maybe Integer, 12 | onlyNewFiles :: Bool, 13 | hfVersion :: T.Text, 14 | hledgerInfo :: HledgerInfo, 15 | sysInfo :: SystemInfo, 16 | verbose :: Bool, 17 | showOptions :: Bool, 18 | sequential :: Bool, 19 | batchSize :: Int, 20 | prettyReports :: Bool 21 | } 22 | deriving (Show) 23 | 24 | instance HasVerbosity RuntimeOptions where 25 | verbose (RuntimeOptions _ _ _ _ _ _ _ v _ _ _ _) = v 26 | 27 | instance HasSequential RuntimeOptions where 28 | sequential (RuntimeOptions _ _ _ _ _ _ _ _ _ sq _ _) = sq 29 | 30 | instance HasBatchSize RuntimeOptions where 31 | batchSize (RuntimeOptions _ _ _ _ _ _ _ _ _ _ bs _) = bs 32 | 33 | instance HasBaseDir RuntimeOptions where 34 | baseDir (RuntimeOptions bd _ _ _ _ _ _ _ _ _ _ _) = bd 35 | 36 | instance HasRunDir RuntimeOptions where 37 | importRunDir (RuntimeOptions _ rd _ _ _ _ _ _ _ _ _ _) = rd 38 | 39 | instance HasPrettyReports RuntimeOptions where 40 | prettyReports (RuntimeOptions _ _ _ _ _ _ _ _ _ _ _ pr) = pr 41 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Internals.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE TemplateHaskell #-} 2 | 3 | module Hledger.Flow.Internals where 4 | 5 | import qualified Data.Text as T 6 | import Data.Version (Version, showVersion) 7 | import Development.GitRev 8 | import GHC.Conc (getNumCapabilities, getNumProcessors) 9 | import Paths_hledger_flow (version) 10 | import qualified System.Info as Sys 11 | 12 | data SystemInfo = SystemInfo 13 | { os :: String, 14 | arch :: String, 15 | compilerName :: String, 16 | compilerVersion :: Version, 17 | cores :: Int, 18 | availableCores :: Int 19 | } 20 | deriving (Show) 21 | 22 | versionInfo :: SystemInfo -> T.Text 23 | versionInfo sysInfo = 24 | T.pack 25 | ( "hledger-flow " 26 | ++ showVersion version 27 | ++ " " 28 | ++ os sysInfo 29 | ++ " " 30 | ++ arch sysInfo 31 | ++ " " 32 | ++ compilerName sysInfo 33 | ++ " " 34 | ++ showVersion (compilerVersion sysInfo) 35 | ++ " " 36 | ++ $(gitHash) 37 | ) 38 | 39 | systemInfo :: IO SystemInfo 40 | systemInfo = do 41 | processors <- getNumProcessors 42 | available <- getNumCapabilities 43 | return 44 | SystemInfo 45 | { os = Sys.os, 46 | arch = Sys.arch, 47 | compilerName = Sys.compilerName, 48 | compilerVersion = Sys.compilerVersion, 49 | cores = processors, 50 | availableCores = available 51 | } 52 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Version and Runtime Information** 14 | 15 | Please mention the version number of `hledger-flow` you are using: 16 | ``` 17 | $ hledger-flow --version 18 | ``` 19 | 20 | Is this the [latest version](https://github.com/apauley/hledger-flow/releases)? 21 | 22 | If your request includes commands you ran and the output, please also include 23 | the runtime options with `--show-options` e.g: 24 | 25 | ``` 26 | $ hledger-flow --show-options import 27 | ``` 28 | 29 | **Our Example Statements Repository** 30 | 31 | FYI, we have a repo with some example transactions which you can use to run `hledger-flow` on: 32 | https://github.com/apauley/hledger-flow-example 33 | 34 | Can you give examples of what you would like by running `hledger-flow` on these files? 35 | 36 | **Describe the solution you'd like** 37 | A clear and concise description of what you want to happen. 38 | 39 | **Describe alternatives you've considered** 40 | A clear and concise description of any alternative solutions or features you've considered. 41 | 42 | **Additional context** 43 | Add any other context or screenshots about the feature request here. 44 | -------------------------------------------------------------------------------- /bin/fetch-latest-hledger: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | USAGE="${0} [OS] [TARGETDIR]" 6 | 7 | if [ "$#" -ne 2 ] 8 | then 9 | echo ${USAGE} > /dev/stderr 10 | exit 1 11 | fi 12 | 13 | OS=${1} 14 | TARGETDIR=${2} 15 | 16 | # OS should work with whatever `uname -s` returns, or in Travis-CI which has ${TRAVIS_OS_NAME} 17 | case ${OS} in 18 | "linux" | "Linux") 19 | FILENAME="hledger-linux-static-x64.zip" 20 | ;; 21 | "osx" | "Darwin") 22 | FILENAME="hledger-macos.zip" 23 | ;; 24 | "windows") 25 | FILENAME="hledger-windows.zip" 26 | ;; 27 | *) 28 | echo "Unknown OS ${OS}. Valid values: linux/osx/windows (or `uname -s`)" > /dev/stderr 29 | exit 1 30 | ;; 31 | esac 32 | 33 | if [ -d ${TARGETDIR} ] 34 | then 35 | echo "Downloading and extracting ${FILENAME} to ${TARGETDIR} (os: ${OS})" 36 | URL=$(curl --silent "https://api.github.com/repos/simonmichael/hledger/releases/latest" | grep "browser_download_url.*${FILENAME}" | awk '{print $2}'| sed 's/"//g') 37 | echo "Fetching '${URL}'" 38 | curl -L ${URL} > /tmp/${FILENAME} 39 | 40 | EXTRACTDIR=$(mktemp -d) 41 | unzip -o -d ${EXTRACTDIR} /tmp/${FILENAME} 42 | chmod 755 ${EXTRACTDIR}/* 43 | 44 | echo "Moving files to ${TARGETDIR}" 45 | mv ${EXTRACTDIR}/* ${TARGETDIR} 46 | 47 | rmdir ${EXTRACTDIR} 48 | else 49 | echo "Specified target directory is not a directory: ${TARGETDIR}" > /dev/stderr 50 | echo ${USAGE} > /dev/stderr 51 | exit 1 52 | fi 53 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Version and Runtime Information** 14 | 15 | Please mention the version number of `hledger-flow` you are using: 16 | ``` 17 | $ hledger-flow --version 18 | ``` 19 | 20 | Is this the [latest version](https://github.com/apauley/hledger-flow/releases)? 21 | Please confirm your issue using the latest version of `hledger-flow`, maybe it has already been fixed. 22 | 23 | Also include the runtime options of the command you ran, e.g: 24 | ``` 25 | $ hledger-flow --show-options import 26 | ``` 27 | 28 | **To Reproduce** 29 | 30 | FYI, we have a repo with some example transactions which you can use to run `hledger-flow` on: 31 | https://github.com/apauley/hledger-flow-example 32 | 33 | Can you reproduce your issue on these example files? 34 | 35 | Steps to reproduce the behavior: 36 | 1. Given this input (files or other input) 37 | 2. And when running this exact command (with `--show-options`) 38 | 3. Under these extra conditions 39 | 4. See error - please paste the full error 40 | 41 | **Expected behavior** 42 | A clear and concise description of what you expected to happen. 43 | 44 | **Desktop (please complete the following information):** 45 | - OS: [e.g. Linux or OSX] 46 | - Version [e.g. 0.42.7.8] 47 | 48 | **Additional context** 49 | Add any other context about the problem here. 50 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Logging.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module Hledger.Flow.Logging where 4 | 5 | import Control.Concurrent.STM 6 | import Control.Monad (when) 7 | import qualified Data.Text as T 8 | import qualified Data.Text.IO as T 9 | import Data.Time.LocalTime (getZonedTime) 10 | import qualified GHC.IO.Handle.FD as H 11 | import Hledger.Flow.Types 12 | import Turtle ((%)) 13 | import qualified Turtle 14 | 15 | dummyLogger :: TChan LogMessage -> T.Text -> IO () 16 | dummyLogger _ _ = return () 17 | 18 | channelOut :: TChan LogMessage -> T.Text -> IO () 19 | channelOut ch txt = atomically $ writeTChan ch $ StdOut txt 20 | 21 | channelOutLn :: TChan LogMessage -> T.Text -> IO () 22 | channelOutLn ch txt = channelOut ch (txt <> "\n") 23 | 24 | channelErr :: TChan LogMessage -> T.Text -> IO () 25 | channelErr ch txt = atomically $ writeTChan ch $ StdErr txt 26 | 27 | channelErrLn :: TChan LogMessage -> T.Text -> IO () 28 | channelErrLn ch txt = channelErr ch (txt <> "\n") 29 | 30 | logToChannel :: TChan LogMessage -> T.Text -> IO () 31 | logToChannel ch msg = do 32 | ts <- timestampPrefix msg 33 | channelErrLn ch ts 34 | 35 | timestampPrefix :: T.Text -> IO T.Text 36 | timestampPrefix txt = do 37 | t <- getZonedTime 38 | return $ Turtle.format (Turtle.s % "\thledger-flow " % Turtle.s) (Turtle.repr t) txt 39 | 40 | consoleChannelLoop :: TChan LogMessage -> IO () 41 | consoleChannelLoop ch = do 42 | logMsg <- atomically $ readTChan ch 43 | case logMsg of 44 | StdOut msg -> do 45 | T.hPutStr H.stdout msg 46 | consoleChannelLoop ch 47 | StdErr msg -> do 48 | T.hPutStr H.stderr msg 49 | consoleChannelLoop ch 50 | Terminate -> return () 51 | 52 | terminateChannelLoop :: TChan LogMessage -> IO () 53 | terminateChannelLoop ch = atomically $ writeTChan ch Terminate 54 | 55 | logVerbose :: (HasVerbosity o) => o -> TChan LogMessage -> T.Text -> IO () 56 | logVerbose opts ch msg = when (verbose opts) $ logToChannel ch msg 57 | -------------------------------------------------------------------------------- /bin/release-tarball: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | USAGE="USAGE: ${0} hledger-flow-executable" 6 | 7 | if [ $# != 1 ] 8 | then 9 | >&2 echo $USAGE 10 | exit 1 11 | fi 12 | 13 | EXECUTABLE=${1} 14 | if [ ! -f ${EXECUTABLE} ] 15 | then 16 | >&2 echo "File does not exist: ${EXECUTABLE}" 17 | exit 1 18 | fi 19 | 20 | BASEDIR="$(pwd)" 21 | 22 | if [ -z ${STACK_YAML} ] 23 | then 24 | STACK_YAML="${BASEDIR}/stack.yaml" 25 | fi 26 | 27 | if [ ! -f ${STACK_YAML} ] 28 | then 29 | >&2 echo "stack yaml file does not exist: ${STACK_YAML}" 30 | exit 1 31 | fi 32 | export STACK_YAML 33 | 34 | TARGETBASE=${BASEDIR}/releases 35 | mkdir -p ${TARGETBASE} 36 | 37 | ABBRHASH="$(git log -1 --format=tformat:%h)" 38 | PACKAGE_VERSION="$(grep '^version:' ${BASEDIR}/package.yaml|awk '{print $2}')" 39 | VERSION="v${PACKAGE_VERSION}" 40 | STACK_RESOLVER="$(grep '^resolver:' ${STACK_YAML}|cut -d' ' -f2)" 41 | GHC_VERSION="$(stack exec -- ghc --version)" 42 | CLI_NAME="$(basename ${EXECUTABLE})" 43 | PLATFORM=`uname -s` 44 | ARCH=`uname -m` 45 | 46 | if [ ${PLATFORM} == "Darwin" ] 47 | then 48 | OS="MacOSX_${PLATFORM}" 49 | else 50 | OS=${PLATFORM} 51 | fi 52 | 53 | VNAME="${CLI_NAME}_${OS}_${ARCH}_${VERSION}_${ABBRHASH}" 54 | TARNAME="${VNAME}.tar.gz" 55 | 56 | TARGET=${TARGETBASE}/${VNAME} 57 | rm -rf ${TARGET} 58 | mkdir ${TARGET} 59 | BUILDINFO=${TARGET}/buildinfo.txt 60 | date > ${BUILDINFO} 61 | 62 | echo >> ${BUILDINFO} 63 | uname -a >> ${BUILDINFO} 64 | echo -e "\nhledger-flow version: $(${EXECUTABLE} --version)" >> ${BUILDINFO} 65 | echo -e "hledger-flow package.yaml version: ${PACKAGE_VERSION}\n" >> ${BUILDINFO} 66 | echo -e "stack.yaml resolver: ${STACK_RESOLVER}" >> ${BUILDINFO} 67 | echo -e "GHC Version: ${GHC_VERSION}\n" >> ${BUILDINFO} 68 | 69 | shasum --algorithm 256 ${EXECUTABLE} >> ${BUILDINFO} 70 | echo >> ${BUILDINFO} 71 | git log -1 >> ${BUILDINFO} 72 | 73 | cp ${EXECUTABLE} ${TARGET} 74 | cd ${TARGET} 75 | shasum --algorithm 256 * > sha256-${VNAME}.txt 76 | cd ${TARGETBASE} 77 | tar zcf ${TARNAME} ${VNAME} 78 | 79 | echo ${TARNAME} 80 | -------------------------------------------------------------------------------- /package.yaml: -------------------------------------------------------------------------------- 1 | name: hledger-flow 2 | version: 0.16.1 3 | synopsis: An hledger workflow focusing on automated statement import and classification. 4 | category: Finance, Console 5 | license: GPL-3 6 | author: "Andreas Pauley " 7 | maintainer: "Andreas Pauley " 8 | copyright: "2023 Andreas Pauley" 9 | github: "apauley/hledger-flow" 10 | bug-reports: https://github.com/apauley/hledger-flow/issues 11 | 12 | extra-source-files: 13 | - README.org 14 | - ChangeLog.md 15 | 16 | # Metadata used when publishing your package 17 | # synopsis: Short description of your package 18 | # category: Finance, Console 19 | 20 | # To avoid duplicated efforts in documentation and dealing with the 21 | # complications of embedding Haddock markup inside cabal files, it is 22 | # common to point users to the README file. 23 | description: Please see the README on GitHub at 24 | 25 | dependencies: 26 | - base >= 4.7 && < 5 27 | 28 | library: 29 | source-dirs: src 30 | ghc-options: 31 | - -Wall 32 | dependencies: 33 | - turtle 34 | - path 35 | - path-io 36 | - filepath 37 | - exceptions 38 | - text 39 | - foldl 40 | - containers 41 | - time 42 | - stm 43 | - gitrev 44 | 45 | executables: 46 | hledger-flow: 47 | main: Main.hs 48 | source-dirs: app 49 | ghc-options: 50 | - -threaded 51 | - -rtsopts 52 | - -with-rtsopts=-N 53 | - -Wall 54 | - -fPIC 55 | ld-options: 56 | - -dynamic # change to -static for a statically linked executable. Does not work on osx 57 | 58 | dependencies: 59 | - hledger-flow 60 | - path 61 | - turtle 62 | - text 63 | - optparse-applicative 64 | 65 | tests: 66 | hledger-flow-test: 67 | main: Spec.hs 68 | source-dirs: test 69 | ghc-options: 70 | - -threaded 71 | - -rtsopts 72 | - -with-rtsopts=-N 73 | dependencies: 74 | - hledger-flow 75 | - path 76 | - path-io 77 | - turtle 78 | - HUnit 79 | - containers 80 | - foldl 81 | - text 82 | - stm 83 | -------------------------------------------------------------------------------- /test/Common/Integration.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module Common.Integration (tests) where 5 | 6 | import qualified Data.List as List (sort) 7 | import Hledger.Flow.Common 8 | import Hledger.Flow.PathHelpers (TurtlePath) 9 | import Test.HUnit 10 | import TestHelpersTurtle 11 | import Turtle 12 | 13 | testHiddenFiles :: Test 14 | testHiddenFiles = 15 | TestCase 16 | ( sh 17 | ( do 18 | let tmpbase = "." "test" "tmp" 19 | mktree tmpbase 20 | tmpdir <- using (mktempdir tmpbase "hlflowtest") 21 | let tmpJournals = map (tmpdir ) journalFiles :: [TurtlePath] 22 | let tmpExtras = map (tmpdir ) extraFiles :: [TurtlePath] 23 | let tmpHidden = map (tmpdir ) hiddenFiles :: [TurtlePath] 24 | let onDisk = List.sort $ tmpJournals ++ tmpExtras ++ tmpHidden 25 | touchAll onDisk 26 | filtered <- fmap List.sort $ shellToList $ onlyFiles $ select onDisk 27 | let expected = List.sort $ tmpExtras ++ tmpJournals 28 | liftIO $ assertEqual "Hidden files should be excluded" expected filtered 29 | ) 30 | ) 31 | 32 | testFilterPaths :: Test 33 | testFilterPaths = 34 | TestCase 35 | ( sh 36 | ( do 37 | let tmpbase = "." "test" "tmp" 38 | mktree tmpbase 39 | tmpdir <- using (mktempdir tmpbase "hlflowtest") 40 | let tmpJournals = map (tmpdir ) journalFiles :: [TurtlePath] 41 | let tmpExtras = map (tmpdir ) extraFiles :: [TurtlePath] 42 | let tmpHidden = map (tmpdir ) hiddenFiles :: [TurtlePath] 43 | let onDisk = List.sort $ tmpJournals ++ tmpExtras ++ tmpHidden 44 | touchAll onDisk 45 | 46 | let nonExistant = map (tmpdir ) ["where", "is", "my", "mind"] 47 | let toFilter = nonExistant ++ onDisk 48 | filtered <- single $ filterPaths testfile toFilter 49 | let actual = List.sort filtered 50 | liftIO $ assertEqual "The filtered paths should exclude files not actually on disk" onDisk actual 51 | ) 52 | ) 53 | 54 | tests :: Test 55 | tests = TestList [testHiddenFiles, testFilterPaths] 56 | -------------------------------------------------------------------------------- /stack.yaml: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by 'stack init' 2 | # 3 | # Some commonly used options have been documented as comments in this file. 4 | # For advanced use and comprehensive documentation of the format, please see: 5 | # https://docs.haskellstack.org/en/stable/yaml_configuration/ 6 | 7 | # Resolver to choose a 'specific' stackage snapshot or a compiler version. 8 | # A snapshot resolver dictates the compiler version and the set of packages 9 | # to be used for project dependencies. For example: 10 | # 11 | # resolver: lts-3.5 12 | # resolver: nightly-2015-09-21 13 | # resolver: ghc-7.10.2 14 | # resolver: ghcjs-0.1.0_ghc-7.10.2 15 | # 16 | # The location of a snapshot can be provided as a file or url. Stack assumes 17 | # a snapshot provided as a file might change, whereas a url resource does not. 18 | # 19 | # resolver: ./custom-snapshot.yaml 20 | # resolver: https://example.com/snapshots/2018-01-01.yaml 21 | resolver: lts-24.8 22 | 23 | # User packages to be built. 24 | # Various formats can be used as shown in the example below. 25 | # 26 | # packages: 27 | # - some-directory 28 | # - https://example.com/foo/bar/baz-0.0.2.tar.gz 29 | # - location: 30 | # git: https://github.com/commercialhaskell/stack.git 31 | # commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a 32 | # - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a 33 | # subdirs: 34 | # - auto-update 35 | # - wai 36 | packages: 37 | - . 38 | # Dependency packages to be pulled from upstream that are not in the resolver 39 | # using the same syntax as the packages field. 40 | # (e.g., acme-missiles-0.3) 41 | # extra-deps: [] 42 | 43 | # Override default flag values for local packages and extra-deps 44 | # flags: {} 45 | 46 | # Extra package databases containing global packages 47 | # extra-package-dbs: [] 48 | 49 | # Control whether we use the GHC we find on the path 50 | # system-ghc: true 51 | # 52 | # Require a specific version of stack, using version ranges 53 | # require-stack-version: -any # Default 54 | # require-stack-version: ">=1.7" 55 | # 56 | # Override the architecture used by stack, especially useful on Windows 57 | # arch: i386 58 | # arch: x86_64 59 | # 60 | # Extra directories used by stack for building 61 | # extra-include-dirs: [/path/to/dir] 62 | # extra-lib-dirs: [/path/to/dir] 63 | # 64 | # Allow a newer minor version of GHC than the snapshot specifies 65 | # compiler-check: newer-minor 66 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | build: 4 | docker: 5 | - image: dastapov/hledger:latest 6 | steps: 7 | - checkout 8 | - restore_cache: 9 | # Read about caching dependencies: https://circleci.com/docs/2.0/caching/ 10 | name: Restore Cached Dependencies 11 | keys: 12 | - hledger-flow-v1-{{ checksum "package.yaml" }}-{{ checksum "stack.yaml" }} 13 | - run: 14 | name: Install Linux Dependencies 15 | command: sudo apt-get update && sudo apt-get install -y curl python3 16 | - run: 17 | name: Install Stack 18 | command: curl -sSL https://get.haskellstack.org/ | sh 19 | - run: 20 | name: Link statically on Linux 21 | command: sed -i 's/- -dynamic /- -static /g' package.yaml 22 | - run: 23 | name: Resolve/Update Stack Dependencies 24 | command: stack setup --interleaved-output 25 | - run: 26 | name: Stack Build And Test 27 | command: ./bin/build-and-test 28 | - run: 29 | name: hledger-flow --help 30 | command: ~/.local/bin/hledger-flow --help 31 | - run: 32 | name: hledger-flow --version 33 | command: ~/.local/bin/hledger-flow --version 34 | - run: 35 | name: List hledger-flow shared object dependencies - should not be dynamic 36 | command: ldd ~/.local/bin/hledger-flow || true 37 | - run: 38 | name: Build Release Tarball 39 | command: ./bin/release-tarball ~/.local/bin/hledger-flow 40 | - run: 41 | name: Show hledger version 42 | command: hledger --version 43 | - run: 44 | name: Clone hledger-flow-example 45 | command: git clone --recurse-submodules https://github.com/apauley/hledger-flow-example.git $HOME/hledger-flow-example 46 | - run: 47 | name: hledger-flow import 48 | command: ~/.local/bin/hledger-flow --verbose --show-options import $HOME/hledger-flow-example 49 | - run: 50 | name: hledger-flow report 51 | command: ~/.local/bin/hledger-flow --verbose --show-options report $HOME/hledger-flow-example 52 | - run: 53 | name: Undo package.yaml change before cache_save 54 | command: git checkout HEAD package.yaml 55 | - save_cache: 56 | name: Cache Dependencies 57 | key: hledger-flow-v1-{{ checksum "package.yaml" }}-{{ checksum "stack.yaml" }} 58 | paths: 59 | - "/root/.stack" 60 | - ".stack-work" 61 | - store_artifacts: 62 | # Upload test summary for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ 63 | path: ./releases 64 | -------------------------------------------------------------------------------- /test/TestHelpersTurtle.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module TestHelpersTurtle where 4 | 5 | import Hledger.Flow.Common (changePathAndExtension) 6 | import Hledger.Flow.PathHelpers (TurtlePath) 7 | import Turtle 8 | 9 | inputJohnBogart :: [TurtlePath] 10 | inputJohnBogart = 11 | [ "import/john/bogartbank/savings/1-in/2017/2017-11-30.csv", 12 | "import/john/bogartbank/savings/1-in/2017/2017-12-30.csv", 13 | "import/john/bogartbank/savings/1-in/2018/2018-02-30.csv", 14 | "import/john/bogartbank/savings/1-in/2018/2018-01-30.csv", 15 | "import/john/bogartbank/checking/1-in/2018/2018-11-30.csv", 16 | "import/john/bogartbank/checking/1-in/2018/2018-10-30.csv", 17 | "import/john/bogartbank/checking/1-in/2018/2018-12-30.csv", 18 | "import/john/bogartbank/checking/1-in/2019/2019-01-30.csv", 19 | "import/john/bogartbank/checking/1-in/2019/2019-02-30.csv" 20 | ] 21 | 22 | inputJohnOther :: [TurtlePath] 23 | inputJohnOther = 24 | [ "import/john/otherbank/creditcard/1-in/2017/2017-12-30.csv", 25 | "import/john/otherbank/creditcard/1-in/2018/2018-01-30.csv", 26 | "import/john/otherbank/investments/1-in/2018/2018-12-30.csv", 27 | "import/john/otherbank/investments/1-in/2019/2019-01-30.csv" 28 | ] 29 | 30 | inputJaneBogart :: [TurtlePath] 31 | inputJaneBogart = 32 | [ "import/jane/bogartbank/savings/1-in/2017/2017-12-30.csv", 33 | "import/jane/bogartbank/savings/1-in/2018/2018-01-30.csv", 34 | "import/jane/bogartbank/checking/1-in/2018/2018-12-30.csv", 35 | "import/jane/bogartbank/checking/1-in/2019/2019-01-30.csv" 36 | ] 37 | 38 | inputJaneOther :: [TurtlePath] 39 | inputJaneOther = 40 | [ "import/jane/otherbank/creditcard/1-in/2017/2017-12-30.csv", 41 | "import/jane/otherbank/creditcard/1-in/2018/2018-01-30.csv", 42 | "import/jane/otherbank/investments/1-in/2018/2018-12-30.csv", 43 | "import/jane/otherbank/investments/1-in/2019/2019-01-30.csv" 44 | ] 45 | 46 | inputFiles :: [TurtlePath] 47 | inputFiles = inputJohnBogart <> inputJohnOther <> inputJaneBogart <> inputJaneOther 48 | 49 | journalFiles :: [TurtlePath] 50 | journalFiles = toJournals inputFiles ++ ignoredJournalFiles 51 | 52 | ignoredJournalFiles :: [TurtlePath] 53 | ignoredJournalFiles = ["import/john/bogartbank/checking/3-journal/2018/not-a-journal.pdf"] 54 | 55 | extraFiles :: [TurtlePath] 56 | extraFiles = ["import/john/bogartbank/savings/2017-opening.journal"] 57 | 58 | hiddenFiles :: [TurtlePath] 59 | hiddenFiles = 60 | [ ".hiddenfile", 61 | "checking/.DS_Store", 62 | "import/john/bogartbank/savings/1-in/.anotherhiddenfile", 63 | "import/john/bogartbank/checking/1-in/2018/.hidden", 64 | "import/john/bogartbank/checking/3-journal/2018/.hidden" 65 | ] 66 | 67 | toJournals :: [TurtlePath] -> [TurtlePath] 68 | toJournals = map (changePathAndExtension "3-journal/" "journal") 69 | 70 | touchAll :: [TurtlePath] -> Shell () 71 | touchAll = foldl (\acc file -> acc <> superTouch file) (return ()) 72 | 73 | superTouch :: TurtlePath -> Shell () 74 | superTouch file = do 75 | mktree $ directory file 76 | touch file 77 | -------------------------------------------------------------------------------- /test/Common/Unit.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module Common.Unit where 5 | 6 | import Hledger.Flow.BaseDir (relativeToBase') 7 | import Hledger.Flow.Common 8 | import Test.HUnit 9 | 10 | testShowCmdArgs :: Test 11 | testShowCmdArgs = 12 | TestCase 13 | ( do 14 | let opts = ["--number", "/tmp/file with spaces"] 15 | let expected = "--number '/tmp/file with spaces'" 16 | let actual = showCmdArgs opts 17 | assertEqual "Convert command-line arguments to text" expected actual 18 | ) 19 | 20 | testRelativeToBase :: Test 21 | testRelativeToBase = 22 | TestCase 23 | ( do 24 | let expected = "file1.journal" 25 | let relativeWithTrailingSlash = relativeToBase' "./base/dir/" "./base/dir/file1.journal" 26 | assertEqual "relative base dir with trailing slash" expected relativeWithTrailingSlash 27 | 28 | let relativeNoTrailingSlash = relativeToBase' "./base/dir" "./base/dir/file1.journal" 29 | assertEqual "relative base dir without a trailing slash" expected relativeNoTrailingSlash 30 | 31 | let absoluteWithTrailingSlash = relativeToBase' "/base/dir/" "/base/dir/file1.journal" 32 | assertEqual "absolute base dir with trailing slash" expected absoluteWithTrailingSlash 33 | 34 | let absoluteNoTrailingSlash = relativeToBase' "/base/dir" "/base/dir/file1.journal" 35 | assertEqual "absolute base dir without a trailing slash" expected absoluteNoTrailingSlash 36 | 37 | let absoluteTwiceNoTrailingSlash = relativeToBase' "/base/dir" "/base/dir" 38 | assertEqual "absolute base dir without a trailing slash supplied twice" "./" absoluteTwiceNoTrailingSlash 39 | 40 | let absoluteTwiceWithTrailingSlash = relativeToBase' "/base/dir/" "/base/dir/" 41 | assertEqual "absolute base dir with a trailing slash supplied twice" "./" absoluteTwiceWithTrailingSlash 42 | 43 | let absoluteTwiceNoTrailingSlashOnSecondParam = relativeToBase' "/base/dir/" "/base/dir" 44 | assertEqual "absolute base dir supplied twice, but the second param has no slash" "./" absoluteTwiceNoTrailingSlashOnSecondParam 45 | 46 | let mismatch = relativeToBase' "/base/dir" "/unrelated/dir/file1.journal" 47 | assertEqual "A basedir with no shared prefix should return the supplied file unchanged" "/unrelated/dir/file1.journal" mismatch 48 | ) 49 | 50 | testExtractDigits :: Test 51 | testExtractDigits = 52 | TestCase 53 | ( do 54 | let txt1 = "A number: 321\nAnother number is 42, so is 0" 55 | 56 | let expected1 = Right 321420 57 | let actual1 = extractDigits txt1 58 | assertEqual "Extract digits from text 1" expected1 actual1 59 | 60 | let txt2 = "No numbers in this line" 61 | 62 | let expected2 = Left "input does not start with a digit" 63 | let actual2 = extractDigits txt2 64 | assertEqual "Extract digits from text 2" expected2 actual2 65 | ) 66 | 67 | tests :: Test 68 | tests = TestList [testShowCmdArgs, testRelativeToBase, testExtractDigits] 69 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Import/ImportHelpers.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE QuasiQuotes #-} 2 | 3 | module Hledger.Flow.Import.ImportHelpers (findInputFiles, findJournalFiles, groupIncludesUpTo, includeFileName) where 4 | 5 | import Data.Char (isDigit) 6 | import qualified Data.Map.Strict as Map 7 | import Data.Maybe (fromMaybe) 8 | import Hledger.Flow.Common (groupValuesBy) 9 | import Hledger.Flow.Import.Types (InputFileBundle) 10 | import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, findFilesIn, pathSize) 11 | import Path 12 | import System.FilePath (dropTrailingPathSeparator) 13 | 14 | findInputFiles :: Integer -> AbsDir -> IO [AbsFile] 15 | findInputFiles startYear = do 16 | let excludeDirs = [[reldir|2-preprocessed|], [reldir|3-journal|]] ++ commonExcludeDirs 17 | findFilesIn (includeYearFilesForParent [reldir|1-in|] startYear) excludeDirs 18 | 19 | findJournalFiles :: AbsDir -> IO [AbsFile] 20 | findJournalFiles = do 21 | let excludeDirs = [[reldir|1-in|], [reldir|2-preprocessed|]] ++ commonExcludeDirs 22 | findFilesIn (includeYearFilesForParent [reldir|3-journal|] 0) excludeDirs 23 | 24 | -- | Include only files directly underneath parentDir/yearDir, e.g. 1-in/2020/* or 3-journal/2020/* 25 | includeYearFilesForParent :: RelDir -> Integer -> AbsDir -> Bool 26 | includeYearFilesForParent parentDir startYear d = 27 | (dirname . parent) d == parentDir 28 | && length shortDirName == 4 29 | && all isDigit shortDirName 30 | && read shortDirName >= startYear 31 | where 32 | shortDirName = dirToStringNoSlash d 33 | 34 | dirToStringNoSlash :: AbsDir -> String 35 | dirToStringNoSlash = init . Path.toFilePath . Path.dirname 36 | 37 | commonExcludeDirs :: [RelDir] 38 | commonExcludeDirs = [[reldir|_manual_|], [reldir|__pycache__|]] 39 | 40 | groupIncludesUpTo :: RelDir -> [RelFile] -> InputFileBundle 41 | groupIncludesUpTo = groupIncludesUpTo' Map.empty 42 | 43 | groupIncludesUpTo' :: InputFileBundle -> RelDir -> [RelFile] -> InputFileBundle 44 | groupIncludesUpTo' acc _ [] = acc 45 | groupIncludesUpTo' acc stopAt journals = do 46 | let dirs = map parent journals :: [RelDir] 47 | let shouldStop = stopAt `elem` dirs 48 | if shouldStop 49 | then acc 50 | else do 51 | let grouped = groupIncludeFilesPerYear journals 52 | groupIncludesUpTo' (acc <> grouped) stopAt (Map.keys grouped) 53 | 54 | groupIncludeFilesPerYear :: [RelFile] -> InputFileBundle 55 | groupIncludeFilesPerYear [] = Map.empty 56 | groupIncludeFilesPerYear ps@(p : _) = 57 | if pathSize (parent p) == 6 58 | then groupValuesBy initialIncludeFilePath ps 59 | else groupValuesBy parentIncludeFilePath ps 60 | 61 | initialIncludeFilePath :: RelFile -> RelFile 62 | initialIncludeFilePath p = (parent . parent . parent) p includeFileName p 63 | 64 | parentIncludeFilePath :: RelFile -> RelFile 65 | parentIncludeFilePath p = (parent . parent) p filename p 66 | 67 | includeFileName :: RelFile -> RelFile 68 | includeFileName f = do 69 | let includeFile = (dropTrailingPathSeparator . toFilePath . dirname . parent) f ++ "-include.journal" 70 | fromMaybe [relfile|unknown-include.journal|] $ parseRelFile includeFile 71 | -------------------------------------------------------------------------------- /src/Hledger/Flow/PathHelpers.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module Hledger.Flow.PathHelpers where 4 | 5 | import Control.Monad.Catch (Exception, MonadThrow, throwM) 6 | import Control.Monad.IO.Class (MonadIO) 7 | import qualified Data.Text as T 8 | import Hledger.Flow.DocHelpers (docURL) 9 | import Path (()) 10 | import qualified Path 11 | import qualified Path.IO as Path 12 | import qualified Turtle 13 | 14 | type TurtlePath = Turtle.FilePath 15 | 16 | type AbsFile = Path.Path Path.Abs Path.File 17 | 18 | type RelFile = Path.Path Path.Rel Path.File 19 | 20 | type AbsDir = Path.Path Path.Abs Path.Dir 21 | 22 | type RelDir = Path.Path Path.Rel Path.Dir 23 | 24 | data PathException = MissingBaseDir AbsDir | InvalidTurtleDir TurtlePath 25 | deriving (Eq) 26 | 27 | instance Show PathException where 28 | show (MissingBaseDir d) = 29 | "Unable to find an import directory at " 30 | ++ show d 31 | ++ " (or in any of its parent directories).\n\n" 32 | ++ "Have a look at the documentation for more information:\n" 33 | ++ T.unpack (docURL "getting-started") 34 | show (InvalidTurtleDir d) = "Expected a directory but got this instead: " ++ d 35 | 36 | instance Exception PathException 37 | 38 | fromTurtleAbsFile :: (MonadThrow m) => TurtlePath -> m AbsFile 39 | fromTurtleAbsFile turtlePath = Path.parseAbsFile turtlePath 40 | 41 | fromTurtleRelFile :: (MonadThrow m) => TurtlePath -> m RelFile 42 | fromTurtleRelFile turtlePath = Path.parseRelFile turtlePath 43 | 44 | fromTurtleAbsDir :: (MonadThrow m) => TurtlePath -> m AbsDir 45 | fromTurtleAbsDir turtlePath = Path.parseAbsDir turtlePath 46 | 47 | fromTurtleRelDir :: (MonadThrow m) => TurtlePath -> m RelDir 48 | fromTurtleRelDir turtlePath = Path.parseRelDir turtlePath 49 | 50 | turtleToAbsDir :: (MonadIO m, MonadThrow m) => AbsDir -> TurtlePath -> m AbsDir 51 | turtleToAbsDir baseDir p = do 52 | isDir <- Turtle.testdir p 53 | if isDir 54 | then Path.resolveDir baseDir p 55 | else throwM $ InvalidTurtleDir p 56 | 57 | pathToTurtle :: Path.Path b t -> TurtlePath 58 | pathToTurtle = Path.toFilePath 59 | 60 | forceTrailingSlash :: TurtlePath -> TurtlePath 61 | forceTrailingSlash p = Turtle.directory (p Turtle. "temp") 62 | 63 | pathSize :: Path.Path b Path.Dir -> Int 64 | pathSize p = pathSize' p 0 65 | 66 | pathSize' :: Path.Path b Path.Dir -> Int -> Int 67 | pathSize' p count = if Path.parent p == p then count else pathSize' (Path.parent p) (count + 1) 68 | 69 | -- | Do a recursive search starting from the given directory. 70 | -- Return all files contained in each directory which matches the given predicate. 71 | findFilesIn :: 72 | (MonadIO m) => 73 | -- | Do we want the files in this directory? 74 | (AbsDir -> Bool) -> 75 | -- | Exclude these directory names 76 | [RelDir] -> 77 | -- | Top of the search tree 78 | AbsDir -> 79 | -- | Absolute paths to all files in the directories which match the predicate 80 | m [AbsFile] 81 | findFilesIn includePred excludeDirs = Path.walkDirAccum (Just excludeHandler) accumulator 82 | where 83 | excludeHandler currentDir _ _ = return $ Path.WalkExclude (map (currentDir ) excludeDirs) 84 | accumulator currentDir _ files = 85 | if includePred currentDir 86 | then return $ excludeHiddenFiles files 87 | else return [] 88 | 89 | excludeHiddenFiles :: [AbsFile] -> [AbsFile] 90 | excludeHiddenFiles = filter (\f -> head (Path.toFilePath (Path.filename f)) /= '.') 91 | -------------------------------------------------------------------------------- /src/Hledger/Flow/BaseDir.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | {-# LANGUAGE QuasiQuotes #-} 3 | 4 | module Hledger.Flow.BaseDir 5 | ( determineBaseDir, 6 | relativeToBase, 7 | relativeToBase', 8 | turtleBaseDir, 9 | effectiveRunDir, 10 | ) 11 | where 12 | 13 | import Control.Monad (when) 14 | import Control.Monad.Catch (MonadThrow, throwM) 15 | import Control.Monad.IO.Class (MonadIO) 16 | import Data.Maybe 17 | import qualified Data.Text as T 18 | import qualified Data.Text.IO as T 19 | import Hledger.Flow.PathHelpers 20 | import Hledger.Flow.Types (BaseDir, HasBaseDir, RunDir, baseDir) 21 | import Path 22 | import Path.IO 23 | import qualified Turtle (liftIO, repr, stripPrefix) 24 | 25 | determineBaseDir :: Maybe TurtlePath -> IO (BaseDir, RunDir) 26 | determineBaseDir suppliedDir = do 27 | pwd <- getCurrentDir 28 | determineBaseDir' pwd suppliedDir 29 | 30 | determineBaseDir' :: AbsDir -> Maybe TurtlePath -> IO (BaseDir, RunDir) 31 | determineBaseDir' pwd (Just suppliedDir) = do 32 | absDir <- turtleToAbsDir pwd suppliedDir 33 | determineBaseDirFromStartDir absDir 34 | determineBaseDir' pwd Nothing = determineBaseDirFromStartDir pwd 35 | 36 | determineBaseDirFromStartDir :: AbsDir -> IO (BaseDir, RunDir) 37 | determineBaseDirFromStartDir startDir = determineBaseDirFromStartDir' startDir startDir 38 | 39 | determineBaseDirFromStartDir' :: (MonadIO m, MonadThrow m) => AbsDir -> AbsDir -> m (BaseDir, RunDir) 40 | determineBaseDirFromStartDir' startDir possibleBaseDir = do 41 | Control.Monad.when (parent possibleBaseDir == possibleBaseDir) $ throwM (MissingBaseDir startDir) 42 | foundBaseDir <- doesDirExist $ possibleBaseDir [reldir|import|] 43 | if foundBaseDir 44 | then do 45 | runDir <- limitRunDir possibleBaseDir startDir 46 | return (possibleBaseDir, runDir) 47 | else determineBaseDirFromStartDir' startDir $ parent possibleBaseDir 48 | 49 | -- | We have unexpected behaviour when the runDir is deeper than the account directory, 50 | -- e.g. "1-in" or the year directory. Specifically, include files are generated incorrectly 51 | -- and some journals are written entirely outside of the baseDir. 52 | -- limitRunDir can possibly removed if the above is fixed. 53 | limitRunDir :: (MonadIO m, MonadThrow m) => BaseDir -> AbsDir -> m RunDir 54 | limitRunDir bd absRunDir = do 55 | rel <- makeRelative bd absRunDir 56 | let runDirDepth = pathSize rel 57 | let fun = composeN (runDirDepth - 4) parent 58 | let newRunDir = fun rel 59 | when (runDirDepth > 4) $ do 60 | let msg = T.pack $ "Changing runDir from " ++ Turtle.repr rel ++ " to " ++ Turtle.repr newRunDir :: T.Text 61 | Turtle.liftIO $ T.putStrLn msg 62 | return newRunDir 63 | 64 | composeN :: Int -> (a -> a) -> (a -> a) 65 | composeN n f 66 | | n < 1 = id 67 | | n == 1 = f 68 | | otherwise = composeN (n - 1) (f . f) 69 | 70 | relativeToBase :: (HasBaseDir o) => o -> TurtlePath -> TurtlePath 71 | relativeToBase opts = relativeToBase' $ pathToTurtle (baseDir opts) 72 | 73 | relativeToBase' :: TurtlePath -> TurtlePath -> TurtlePath 74 | relativeToBase' bd p = 75 | if forceTrailingSlash bd == forceTrailingSlash p 76 | then "./" 77 | else 78 | fromMaybe p $ Turtle.stripPrefix (forceTrailingSlash bd) p 79 | 80 | turtleBaseDir :: (HasBaseDir o) => o -> TurtlePath 81 | turtleBaseDir opts = pathToTurtle $ baseDir opts 82 | 83 | effectiveRunDir :: BaseDir -> RunDir -> AbsDir 84 | effectiveRunDir bd rd = do 85 | let baseImportDir = bd [Path.reldir|import|] 86 | let absRunDir = bd rd 87 | if absRunDir == bd then baseImportDir else absRunDir 88 | -------------------------------------------------------------------------------- /step-by-step/part3.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | #+TITLE: Hledger Flow: Step-By-Step 3 | #+AUTHOR: 4 | #+REVEAL_TRANS: default 5 | #+REVEAL_THEME: beige 6 | #+OPTIONS: num:nil 7 | #+PROPERTY: header-args:sh :prologue exec 2>&1 :epilogue echo : 8 | 9 | * Part 3 10 | 11 | This is the third part in a a series of step-by-step instructions. 12 | 13 | They are intended to be read in sequence. Head over to the [[file:README.org][docs README]] to see all parts. 14 | 15 | * About This Document 16 | 17 | This document is a [[https://www.offerzen.com/blog/literate-programming-empower-your-writing-with-emacs-org-mode][literate]] [[https://orgmode.org/worg/org-contrib/babel/intro.html][program]]. 18 | You can read it like a normal article, either [[https://github.com/apauley/hledger-flow/blob/master/docs/part3.org][on the web]] or [[https://pauley.org.za/hledger-flow/part3.html][as a slide show]]. 19 | 20 | But you can also [[https://github.com/apauley/hledger-flow][clone the repository]] and open [[https://raw.githubusercontent.com/apauley/hledger-flow/master/docs/part3.org][this org-mode file]] in emacs. 21 | Then you can execute each code snippet by pressing =C-c C-c= with your cursor on the relevant code block. 22 | 23 | * Adding More Statements 24 | 25 | Now that the all the boilerplate for the first statement has been done, 26 | adding some more should be easy: 27 | 28 | #+NAME: more-input-files 29 | #+BEGIN_SRC sh :results org :exports both 30 | cp -f Downloads/Bogart/123456789_2016*.csv \ 31 | my-finances/import/gawie/bogart/cheque/1-in/2016/ 32 | hledger-flow import ./my-finances 33 | hledger -f my-finances/all-years.journal incomestatement \ 34 | --pretty-tables --monthly --average --begin 2016-03-01 35 | #+END_SRC 36 | 37 | #+REVEAL: split 38 | 39 | #+RESULTS: more-input-files 40 | #+begin_src org 41 | Found 3 input files in 0.002017s. Proceeding with import... 42 | Wrote include files for 3 journals in 0.004568s 43 | Imported 3/3 journals in 0.109403s 44 | Income Statement 2016-03-01..2016-05-31 45 | 46 | ║ Mar Apr May Average 47 | ══════════════════╬════════════════════════════════════════════ 48 | Revenues ║ 49 | ──────────────────╫──────────────────────────────────────────── 50 | income:unknown ║ R37256.28 R37256.28 R37256.28 R37256.28 51 | ──────────────────╫──────────────────────────────────────────── 52 | ║ R37256.28 R37256.28 R37256.28 R37256.28 53 | ══════════════════╬════════════════════════════════════════════ 54 | Expenses ║ 55 | ──────────────────╫──────────────────────────────────────────── 56 | expenses:unknown ║ R37284.66 R40333.49 R37234.43 R38284.19 57 | ──────────────────╫──────────────────────────────────────────── 58 | ║ R37284.66 R40333.49 R37234.43 R38284.19 59 | ══════════════════╬════════════════════════════════════════════ 60 | Net: ║ R-28.38 R-3077.21 R21.85 R-1027.91 61 | 62 | #+end_src 63 | 64 | #+REVEAL: split 65 | 66 | Actually this doesn't look so good. 67 | In March and April, Gawie spent more than he earned. 68 | 69 | It is time to classify each transaction so that he can have a better view into 70 | what is going on. 71 | 72 | #+REVEAL: split 73 | 74 | And the new statements gets added to the repository. 75 | #+NAME: git-checkpoint-more-statements 76 | #+BEGIN_SRC sh :results none :exports both 77 | cd my-finances/ 78 | git add . 79 | git commit -m 'Added a few more statements' 80 | cd .. 81 | #+END_SRC 82 | 83 | #+REVEAL: split 84 | 85 | #+NAME: git-push-hledger-flow-example 86 | #+BEGIN_SRC sh :results none :exports results 87 | cd my-finances/ 88 | git remote add origin git@github.com:apauley/hledger-flow-example.git 89 | git push --force -u origin master 90 | cd .. 91 | #+END_SRC 92 | -------------------------------------------------------------------------------- /step-by-step/part2.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | #+TITLE: Hledger Flow: Step-By-Step 3 | #+AUTHOR: 4 | #+REVEAL_TRANS: default 5 | #+REVEAL_THEME: beige 6 | #+OPTIONS: num:nil 7 | #+PROPERTY: header-args:sh :prologue exec 2>&1 :epilogue echo : 8 | 9 | * Part 2 10 | 11 | This is the second part in a a series of step-by-step instructions. 12 | 13 | They are intended to be read in sequence. Head over to the [[file:README.org][docs README]] to see all parts. 14 | 15 | * About This Document 16 | 17 | This document is a [[https://www.offerzen.com/blog/literate-programming-empower-your-writing-with-emacs-org-mode][literate]] [[https://orgmode.org/worg/org-contrib/babel/intro.html][program]]. 18 | You can read it like a normal article, either [[https://github.com/apauley/hledger-flow/blob/master/docs/part2.org][on the web]] or [[https://pauley.org.za/hledger-flow/part2.html][as a slide show]]. 19 | 20 | But you can also [[https://github.com/apauley/hledger-flow][clone the repository]] and open [[https://raw.githubusercontent.com/apauley/hledger-flow/master/docs/part2.org][this org-mode file]] in emacs. 21 | Then you can execute each code snippet by pressing =C-c C-c= with your cursor on the relevant code block. 22 | 23 | * An hledger report 24 | 25 | Now that we have [[file:part1.org][our first successful import]], can hledger show us some useful information? 26 | 27 | #+NAME: hledger-err-balance 28 | #+BEGIN_SRC sh :results none :exports code 29 | hledger -f my-finances/all-years.journal incomestatement 30 | #+END_SRC 31 | 32 | #+REVEAL: split 33 | 34 | #+BEGIN_SRC hledger 35 | hledger: balance assertion error in 36 | "my-finances/import/gawie/bogart/cheque/3-journal/2016/123456789_2016-03-30.journal" 37 | (line 2, column 56): 38 | transaction: 39 | 2016-03-01 * #Monthly Bank Fee// 40 | Assets:Current:Gawie:Bogart:Cheque R-500.00 = R40000.00 41 | expenses:unknown R500.00 42 | 43 | assertion details: 44 | date: 2016-03-01 45 | account: Assets:Current:Gawie:Bogart:Cheque 46 | commodity: R 47 | calculated: -500.00 48 | asserted: 40000.00 49 | difference: 40500.00 50 | #+END_SRC 51 | 52 | #+REVEAL: split 53 | 54 | Not yet - we have a balance assertion error. 55 | =hledger= thinks the balance should be =-R500=, but our import asserted that it should be =R40000=. 56 | 57 | #+REVEAL: split 58 | 59 | Remember the =balance= field we added to the rules file? 60 | #+NAME: balance-field-rules-file 61 | #+BEGIN_SRC hledger 62 | fields _, _, date, desc1, desc2, desc3, amount, balance, _ 63 | #+END_SRC 64 | 65 | It adds a balance assertion to each transaction, using the data helpfully provided by Bogart Bank. 66 | 67 | #+REVEAL: split 68 | 69 | Clearly the cheque account has a pre-existing balance of =R40500=. 70 | To make =hledger= happy, we need to tell it what the opening balance for this account is: 71 | #+NAME: bogart-cheque-opening-balance 72 | #+BEGIN_SRC hledger :tangle my-finances/import/gawie/bogart/cheque/2016-opening.journal 73 | 2016-03-01 Cheque Account Opening Balance 74 | Assets:Current:Gawie:Bogart:Cheque R40500 75 | Equity:Opening Balances:Gawie:Bogart:Cheque 76 | #+END_SRC 77 | 78 | Save this as =my-finances/import/gawie/bogart/cheque/2016-opening.journal=. 79 | 80 | #+NAME: tangle-opening-balances 81 | #+BEGIN_SRC emacs-lisp :results none :exports results 82 | ; Narrator: this just tells emacs to write out the rules file. Carry on. 83 | ; FIXME: This should just tangle the one relevant block, not all tangle blocks 84 | (org-babel-tangle-file (buffer-file-name)) 85 | #+END_SRC 86 | 87 | #+REVEAL: split 88 | 89 | Then run the import again: 90 | #+NAME: part2-import1 91 | #+BEGIN_SRC sh :results org :exports both 92 | hledger-flow import ./my-finances 93 | cd my-finances 94 | git diff 95 | cd .. 96 | #+END_SRC 97 | 98 | #+RESULTS: part2-import1 99 | #+begin_src org 100 | Wrote include files for 1 journals in 0.003676s 101 | Imported 1/1 journals in 0.093608s 102 | diff --git import/gawie/bogart/cheque/2016-include.journal import/gawie/bogart/cheque/2016-include.journal 103 | index 62a0a11..0962f4a 100644 104 | --- import/gawie/bogart/cheque/2016-include.journal 105 | +++ import/gawie/bogart/cheque/2016-include.journal 106 | @@ -1,3 +1,4 @@ 107 | ### Generated by hledger-flow - DO NOT EDIT ### 108 | 109 | +include 2016-opening.journal 110 | include 3-journal/2016/123456789_2016-03-30.journal 111 | 112 | #+end_src 113 | 114 | #+REVEAL: split 115 | 116 | Time for another git checkpoint. 117 | 118 | #+NAME: git-checkpoint-opening-balance 119 | #+BEGIN_SRC sh :results none :exports both 120 | cd my-finances/ 121 | git add . 122 | git commit -m 'Added an opening balance for Bogart Cheque Account' 123 | cd .. 124 | #+END_SRC 125 | 126 | #+REVEAL: split 127 | 128 | Now we can try the income statement again. 129 | 130 | #+NAME: hledger-incomestatement 131 | #+BEGIN_SRC sh :results org :exports both 132 | hledger -f my-finances/all-years.journal incomestatement \ 133 | --pretty-tables 134 | #+END_SRC 135 | 136 | #+REVEAL: split 137 | 138 | #+RESULTS: hledger-incomestatement 139 | #+begin_src org 140 | Income Statement 2016-03-01..2016-03-25 141 | 142 | ║ 2016-03-01..2016-03-25 143 | ══════════════════╬════════════════════════ 144 | Revenues ║ 145 | ──────────────────╫──────────────────────── 146 | income:unknown ║ R37256.28 147 | ──────────────────╫──────────────────────── 148 | ║ R37256.28 149 | ══════════════════╬════════════════════════ 150 | Expenses ║ 151 | ──────────────────╫──────────────────────── 152 | expenses:unknown ║ R36734.43 153 | ──────────────────╫──────────────────────── 154 | ║ R36734.43 155 | ══════════════════╬════════════════════════ 156 | Net: ║ R521.85 157 | 158 | #+end_src 159 | 160 | It worked! 161 | 162 | #+REVEAL: split 163 | 164 | The story continues with [[file:part3.org][part 3]]. 165 | -------------------------------------------------------------------------------- /test/TestHelpers.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | {-# LANGUAGE QuasiQuotes #-} 3 | 4 | module TestHelpers where 5 | 6 | import Data.Maybe (fromMaybe) 7 | import Hledger.Flow.Internals (SystemInfo (..), versionInfo) 8 | import Hledger.Flow.PathHelpers (RelFile) 9 | import Hledger.Flow.RuntimeOptions 10 | import qualified Hledger.Flow.Types as FlowTypes 11 | import Path 12 | import qualified System.Info as Sys 13 | 14 | defaultHlInfo :: FlowTypes.HledgerInfo 15 | defaultHlInfo = FlowTypes.HledgerInfo [absfile|/path/to/hledger|] "1.2.3" 16 | 17 | testSystemInfo :: SystemInfo 18 | testSystemInfo = 19 | SystemInfo 20 | { os = Sys.os, 21 | arch = Sys.arch, 22 | compilerName = Sys.compilerName, 23 | compilerVersion = Sys.compilerVersion, 24 | cores = 1, 25 | availableCores = 1 26 | } 27 | 28 | defaultOpts :: FlowTypes.BaseDir -> RuntimeOptions 29 | defaultOpts bd = 30 | RuntimeOptions 31 | { baseDir = bd, 32 | importRunDir = [reldir|./|], 33 | importStartYear = Nothing, 34 | onlyNewFiles = False, 35 | hfVersion = versionInfo testSystemInfo, 36 | hledgerInfo = defaultHlInfo, 37 | sysInfo = testSystemInfo, 38 | verbose = False, 39 | showOptions = False, 40 | sequential = False, 41 | batchSize = 1, 42 | prettyReports = True 43 | } 44 | 45 | toJournal :: RelFile -> RelFile 46 | toJournal inFile = do 47 | let journalFile = fromMaybe [relfile|oops.err|] $ replaceExtension ".journal" inFile 48 | let journalName = filename journalFile 49 | let yearDir = dirname . parent $ inFile 50 | (parent . parent . parent) inFile [reldir|3-journal|] yearDir journalName 51 | 52 | inputJohnSavings2017 :: [RelFile] 53 | inputJohnSavings2017 = 54 | [ [relfile|import/john/bogartbank/savings/1-in/2017/2017-11-30.csv|], 55 | [relfile|import/john/bogartbank/savings/1-in/2017/2017-12-30.csv|] 56 | ] 57 | 58 | johnSavingsJournals2017 :: [RelFile] 59 | johnSavingsJournals2017 = map toJournal inputJohnSavings2017 60 | 61 | inputJohnSavings2018 :: [RelFile] 62 | inputJohnSavings2018 = 63 | [ [relfile|import/john/bogartbank/savings/1-in/2018/2018-02-30.csv|], 64 | [relfile|import/john/bogartbank/savings/1-in/2018/2018-01-30.csv|] 65 | ] 66 | 67 | johnSavingsJournals2018 :: [RelFile] 68 | johnSavingsJournals2018 = map toJournal inputJohnSavings2018 69 | 70 | inputJohnChecking2018 :: [RelFile] 71 | inputJohnChecking2018 = 72 | [ [relfile|import/john/bogartbank/checking/1-in/2018/2018-11-30.csv|], 73 | [relfile|import/john/bogartbank/checking/1-in/2018/2018-10-30.csv|], 74 | [relfile|import/john/bogartbank/checking/1-in/2018/2018-12-30.csv|] 75 | ] 76 | 77 | johnCheckingJournals2018 :: [RelFile] 78 | johnCheckingJournals2018 = map toJournal inputJohnChecking2018 79 | 80 | inputJohnChecking2019 :: [RelFile] 81 | inputJohnChecking2019 = 82 | [ [relfile|import/john/bogartbank/checking/1-in/2019/2019-01-30.csv|], 83 | [relfile|import/john/bogartbank/checking/1-in/2019/2019-02-30.csv|] 84 | ] 85 | 86 | johnCheckingJournals2019 :: [RelFile] 87 | johnCheckingJournals2019 = map toJournal inputJohnChecking2019 88 | 89 | inputJohnBogart :: [RelFile] 90 | inputJohnBogart = inputJohnSavings2017 <> inputJohnSavings2018 <> inputJohnChecking2018 <> inputJohnChecking2019 91 | 92 | johnCC2017 :: RelFile 93 | johnCC2017 = [relfile|import/john/otherbank/creditcard/1-in/2017/2017-12-30.csv|] 94 | 95 | johnCCJournal2017 :: RelFile 96 | johnCCJournal2017 = toJournal johnCC2017 97 | 98 | johnCC2018 :: RelFile 99 | johnCC2018 = [relfile|import/john/otherbank/creditcard/1-in/2018/2018-01-30.csv|] 100 | 101 | johnCCJournal2018 :: RelFile 102 | johnCCJournal2018 = toJournal johnCC2018 103 | 104 | johnInvest2018 :: RelFile 105 | johnInvest2018 = [relfile|import/john/otherbank/investments/1-in/2018/2018-12-30.csv|] 106 | 107 | johnInvestJournal2018 :: RelFile 108 | johnInvestJournal2018 = toJournal johnInvest2018 109 | 110 | johnInvest2019 :: RelFile 111 | johnInvest2019 = [relfile|import/john/otherbank/investments/1-in/2019/2019-01-30.csv|] 112 | 113 | johnInvestJournal2019 :: RelFile 114 | johnInvestJournal2019 = toJournal johnInvest2019 115 | 116 | inputJohnOther :: [RelFile] 117 | inputJohnOther = [johnCC2017, johnCC2018, johnInvest2018, johnInvest2019] 118 | 119 | janeSavings2017 :: RelFile 120 | janeSavings2017 = [relfile|import/jane/bogartbank/savings/1-in/2017/2017-12-30.csv|] 121 | 122 | janeSavings2018 :: [RelFile] 123 | janeSavings2018 = 124 | [ [relfile|import/jane/bogartbank/savings/1-in/2018/2018-01-30.csv|], 125 | [relfile|import/jane/bogartbank/savings/1-in/2018/2018-12-30.csv|] 126 | ] 127 | 128 | janeSavings2019 :: RelFile 129 | janeSavings2019 = [relfile|import/jane/bogartbank/savings/1-in/2019/2019-01-30.csv|] 130 | 131 | inputJaneBogart :: [RelFile] 132 | inputJaneBogart = 133 | [ janeSavings2017, 134 | [relfile|import/jane/bogartbank/savings/3-journals/2018/2018-01-30.journal|], 135 | [relfile|import/jane/bogartbank/savings/3-journals/2018/2018-12-30.journal|], 136 | janeSavings2019 137 | ] 138 | 139 | janeSavingsJournal2017 :: RelFile 140 | janeSavingsJournal2017 = toJournal janeSavings2017 141 | 142 | janeSavingsJournals2018 :: [RelFile] 143 | janeSavingsJournals2018 = map toJournal janeSavings2018 144 | 145 | janeSavingsJournal2019 :: RelFile 146 | janeSavingsJournal2019 = toJournal janeSavings2019 147 | 148 | janeSavingsJournals :: [RelFile] 149 | janeSavingsJournals = [janeSavingsJournal2017] ++ janeSavingsJournals2018 ++ [janeSavingsJournal2019] 150 | 151 | janeCC2017 :: RelFile 152 | janeCC2017 = [relfile|import/jane/otherbank/creditcard/1-in/2017/2017-12-30.csv|] 153 | 154 | janeCCJournal2017 :: RelFile 155 | janeCCJournal2017 = toJournal janeCC2017 156 | 157 | janeCC2018 :: RelFile 158 | janeCC2018 = [relfile|import/jane/otherbank/creditcard/1-in/2018/2018-01-30.csv|] 159 | 160 | janeCCJournal2018 :: RelFile 161 | janeCCJournal2018 = toJournal janeCC2018 162 | 163 | janeInvest2018 :: RelFile 164 | janeInvest2018 = [relfile|import/jane/otherbank/investments/1-in/2018/2018-12-30.csv|] 165 | 166 | janeInvestJournal2018 :: RelFile 167 | janeInvestJournal2018 = toJournal janeInvest2018 168 | 169 | janeInvest2019 :: RelFile 170 | janeInvest2019 = [relfile|import/jane/otherbank/investments/1-in/2019/2019-01-30.csv|] 171 | 172 | janeInvestJournal2019 :: RelFile 173 | janeInvestJournal2019 = toJournal janeInvest2019 174 | 175 | inputJaneOther :: [RelFile] 176 | inputJaneOther = [janeCC2017, janeCC2018, janeInvest2018, janeInvest2019] 177 | 178 | inputFiles :: [RelFile] 179 | inputFiles = inputJohnBogart <> inputJohnOther <> inputJaneBogart <> inputJaneOther 180 | 181 | journalFiles :: [RelFile] 182 | journalFiles = map toJournal inputFiles 183 | -------------------------------------------------------------------------------- /test/BaseDir/Integration.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | {-# LANGUAGE QuasiQuotes #-} 4 | 5 | module BaseDir.Integration (tests) where 6 | 7 | import Control.Exception (try) 8 | import qualified Data.Text as T 9 | import qualified Data.Text.IO as T 10 | import Hledger.Flow.BaseDir (determineBaseDir) 11 | import Hledger.Flow.Common 12 | import Hledger.Flow.PathHelpers 13 | import Hledger.Flow.Types (BaseDir, RunDir) 14 | import Path 15 | import Path.IO 16 | import Test.HUnit 17 | import qualified Turtle 18 | import Prelude hiding (readFile, writeFile) 19 | 20 | assertSubDirsForDetermineBaseDir :: AbsDir -> BaseDir -> [Path.Path b Dir] -> IO () 21 | assertSubDirsForDetermineBaseDir initialPwd expectedBaseDir importDirs = do 22 | sequence_ $ map (assertDetermineBaseDir initialPwd expectedBaseDir) importDirs 23 | 24 | assertDetermineBaseDir :: AbsDir -> BaseDir -> Path.Path b Dir -> IO () 25 | assertDetermineBaseDir initialPwd expectedBaseDir subDir = do 26 | setCurrentDir initialPwd 27 | (bd1, runDir1) <- determineBaseDir $ Just $ pathToTurtle subDir 28 | assertFindTestFileUsingRundir bd1 runDir1 29 | 30 | setCurrentDir subDir 31 | (bd2, runDir2) <- determineBaseDir Nothing 32 | assertFindTestFileUsingRundir bd2 runDir2 33 | 34 | (bd3, runDir3) <- determineBaseDir $ Just "." 35 | assertFindTestFileUsingRundir bd3 runDir3 36 | 37 | (bd4, runDir4) <- determineBaseDir $ Just "./" 38 | assertFindTestFileUsingRundir bd4 runDir4 39 | 40 | setCurrentDir initialPwd 41 | let msg dir = "determineBaseDir searches from pwd upwards until it finds a dir containing 'import' - " ++ show dir 42 | sequence_ $ map (\dir -> assertEqual (msg dir) expectedBaseDir dir) [bd1, bd2, bd3, bd4] 43 | 44 | assertFindTestFileUsingRundir :: BaseDir -> RunDir -> IO () 45 | assertFindTestFileUsingRundir baseDir runDir = do 46 | let absRunDir = baseDir runDir 47 | 48 | found <- Turtle.single $ fmap head $ shellToList $ Turtle.find (Turtle.has "test-file.txt") $ pathToTurtle absRunDir 49 | fileContents <- T.readFile found 50 | assertEqual "We should find our test file by searching from the returned runDir" (T.pack $ "The expected base dir is " ++ show baseDir) fileContents 51 | 52 | assertCurrentDirVariations :: AbsDir -> RelDir -> IO () 53 | assertCurrentDirVariations absoluteTempDir bdRelativeToTempDir = do 54 | let absBaseDir = absoluteTempDir bdRelativeToTempDir 55 | 56 | setCurrentDir absBaseDir 57 | (bd1, runDir1) <- determineBaseDir Nothing 58 | (bd2, runDir2) <- determineBaseDir $ Just "." 59 | (bd3, runDir3) <- determineBaseDir $ Just "./" 60 | (bd4, runDir4) <- determineBaseDir $ Just $ pathToTurtle absBaseDir 61 | 62 | let msg label dir = "When pwd is the base dir, determineBaseDir returns the same " ++ label ++ ", regardless of the input variation. " ++ show dir 63 | sequence_ $ map (\dir -> assertEqual (msg "baseDir" dir) absBaseDir dir) [bd1, bd2, bd3, bd4] 64 | sequence_ $ map (\dir -> assertEqual (msg "runDir" dir) [reldir|.|] dir) [runDir1, runDir2, runDir3, runDir4] 65 | 66 | testBaseDirWithTempDir :: AbsDir -> AbsDir -> IO () 67 | testBaseDirWithTempDir initialPwd absoluteTempDir = do 68 | error1 <- try $ determineBaseDir $ Just "/path/to/dir" 69 | assertEqual "determineBaseDir produces an error message when given a non-existant dir" (Left $ InvalidTurtleDir "/path/to/dir") error1 70 | 71 | let unrelatedDir = absoluteTempDir [reldir|unrelated|] 72 | createDir unrelatedDir 73 | 74 | bdUnrelated <- try $ determineBaseDir $ Just (pathToTurtle unrelatedDir) 75 | assertEqual "determineBaseDir produces an error message when it cannot find a baseDir" (Left $ MissingBaseDir unrelatedDir) bdUnrelated 76 | 77 | let baseDir = [reldir|bd1|] 78 | let importDir = baseDir [reldir|import|] 79 | let ownerDir = importDir [reldir|john|] 80 | let bankDir = ownerDir [reldir|mybank|] 81 | let accDir = bankDir [reldir|myacc|] 82 | let inDir = accDir [reldir|1-in|] 83 | let yearDir = inDir [reldir|2019|] 84 | let subDirs = [yearDir, inDir, accDir, bankDir, ownerDir, importDir, baseDir] :: [RelDir] 85 | 86 | createDirIfMissing True $ absoluteTempDir yearDir 87 | 88 | let fictionalDir = absoluteTempDir ownerDir [reldir|fictionalDir|] 89 | errorSub <- try $ determineBaseDir $ Just $ pathToTurtle fictionalDir 90 | assertEqual "determineBaseDir produces an error message when given a non-existant subdir of a valid basedir" (Left $ InvalidTurtleDir $ pathToTurtle fictionalDir) errorSub 91 | 92 | assertCurrentDirVariations absoluteTempDir baseDir 93 | 94 | relativeTempDir <- makeRelative initialPwd absoluteTempDir 95 | let subDirsRelativeToTop = map (relativeTempDir ) subDirs 96 | let absoluteSubDirs = map (absoluteTempDir ) subDirs 97 | 98 | let absoluteBaseDir = absoluteTempDir baseDir 99 | 100 | T.writeFile (pathToTurtle $ absoluteTempDir yearDir [relfile|test-file.txt|]) (T.pack $ "The expected base dir is " ++ show absoluteBaseDir) 101 | 102 | assertSubDirsForDetermineBaseDir absoluteTempDir absoluteBaseDir subDirs 103 | assertSubDirsForDetermineBaseDir absoluteTempDir absoluteBaseDir absoluteSubDirs 104 | assertSubDirsForDetermineBaseDir initialPwd absoluteBaseDir subDirsRelativeToTop 105 | return () 106 | 107 | assertRunDirs :: RelDir -> [RelDir] -> [RelDir] -> IO () 108 | assertRunDirs accDir businessAsUsualRundirs specialTreatmentRundirs = do 109 | sequence_ $ map (assertRunDir id "Normal rundirs should not be modified") businessAsUsualRundirs 110 | sequence_ $ map (assertRunDir (\_ -> accDir) "Rundirs deeper than account-level should return the account dir instead") specialTreatmentRundirs 111 | 112 | assertRunDir :: (RelDir -> RelDir) -> String -> RelDir -> IO () 113 | assertRunDir expectedRunDir msg subDir = do 114 | (_, runDir) <- determineBaseDir $ Just $ pathToTurtle subDir 115 | assertEqual msg (expectedRunDir subDir) runDir 116 | 117 | testRunDirsWithTempDir :: AbsDir -> IO () 118 | testRunDirsWithTempDir absoluteTempDir = do 119 | let baseDir = absoluteTempDir [reldir|bd1|] 120 | 121 | let importDir = [reldir|import|] 122 | let ownerDir = importDir [reldir|john|] 123 | let bankDir = ownerDir [reldir|mybank|] 124 | let accDir = bankDir [reldir|myacc|] 125 | let inDir = accDir [reldir|1-in|] 126 | let yearDir = inDir [reldir|2019|] 127 | 128 | createDirIfMissing True $ baseDir yearDir 129 | 130 | withCurrentDir baseDir $ assertRunDirs accDir [accDir, bankDir, ownerDir, importDir] [yearDir, inDir] 131 | 132 | testRunDirs :: Test 133 | testRunDirs = 134 | TestCase 135 | ( do 136 | initialPwd <- getCurrentDir 137 | let tmpbase = initialPwd [reldir|test|] [reldir|tmp|] 138 | withTempDir tmpbase "hlflowtest" testRunDirsWithTempDir 139 | ) 140 | 141 | testDetermineBaseDir :: Test 142 | testDetermineBaseDir = 143 | TestCase 144 | ( do 145 | initialPwd <- getCurrentDir 146 | let tmpbase = initialPwd [reldir|test|] [reldir|tmp|] 147 | createDirIfMissing True tmpbase 148 | withTempDir tmpbase "hlflowtest" $ testBaseDirWithTempDir initialPwd 149 | ) 150 | 151 | tests :: Test 152 | tests = TestList [testDetermineBaseDir, testRunDirs] 153 | -------------------------------------------------------------------------------- /app/Main.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | {-# LANGUAGE QuasiQuotes #-} 3 | 4 | module Main where 5 | 6 | import Parsing ( parseStartYear ) 7 | import Path ( reldir ) 8 | 9 | import qualified Turtle hiding (switch) 10 | import Prelude hiding (putStrLn) 11 | 12 | import Options.Applicative 13 | ( auto, 14 | optional, 15 | Alternative(many, (<|>)), 16 | Parser, 17 | flag', 18 | help, 19 | long, 20 | metavar, 21 | option, 22 | short, 23 | str, 24 | switch ) 25 | 26 | import Hledger.Flow.PathHelpers (TurtlePath) 27 | import Hledger.Flow.Common ( hledgerInfoFromPath ) 28 | import Hledger.Flow.Internals (versionInfo, systemInfo) 29 | import Hledger.Flow.BaseDir ( determineBaseDir ) 30 | import qualified Hledger.Flow.RuntimeOptions as RT 31 | import Hledger.Flow.Reports ( generateReports ) 32 | import Hledger.Flow.Import.CSVImport ( importCSVs ) 33 | 34 | import qualified Data.Text.IO as T 35 | 36 | data ImportParams = ImportParams { maybeImportBaseDir :: Maybe TurtlePath 37 | , importStartYear :: Maybe String 38 | , onlyNewFiles :: Bool 39 | } deriving (Show) 40 | 41 | data ReportParams = ReportParams { maybeReportBaseDir :: Maybe TurtlePath 42 | , asciiReports :: Bool 43 | } deriving (Show) 44 | 45 | data Command = Import ImportParams | Report ReportParams deriving (Show) 46 | 47 | data MainParams = MainParams { verbosity :: Int 48 | , hledgerPathOpt :: Maybe TurtlePath 49 | , showOpts :: Bool 50 | , batchSize :: Maybe Int 51 | , sequential :: Bool 52 | } deriving (Show) 53 | data BaseCommand = Version | Command { mainParams :: MainParams, command :: Command } deriving (Show) 54 | 55 | main :: IO () 56 | main = do 57 | cmd <- Turtle.options "An hledger workflow focusing on automated statement import and classification:\nhttps://github.com/apauley/hledger-flow#readme" baseCommandParser 58 | case cmd of 59 | Version -> do 60 | sysInfo <- systemInfo 61 | T.putStrLn $ versionInfo sysInfo 62 | Command mainParams' (Import subParams) -> toRuntimeOptionsImport mainParams' subParams >>= importCSVs 63 | Command mainParams' (Report subParams) -> toRuntimeOptionsReport mainParams' subParams >>= generateReports 64 | 65 | defaultBatchSize :: Int 66 | defaultBatchSize = 20 67 | 68 | determineBatchSize :: MainParams -> IO Int 69 | determineBatchSize mainParams' = 70 | case (batchSize mainParams') of 71 | Nothing -> return defaultBatchSize 72 | Just size -> return size 73 | 74 | toRuntimeOptionsImport :: MainParams -> ImportParams -> IO RT.RuntimeOptions 75 | toRuntimeOptionsImport mainParams' subParams' = do 76 | startYear <- parseStartYear $ importStartYear subParams' 77 | let maybeBD = maybeImportBaseDir subParams' :: Maybe TurtlePath 78 | (bd, runDir) <- determineBaseDir maybeBD 79 | hli <- hledgerInfoFromPath $ hledgerPathOpt mainParams' 80 | size <- determineBatchSize mainParams' 81 | sysInfo <- systemInfo 82 | return RT.RuntimeOptions { RT.baseDir = bd 83 | , RT.importRunDir = runDir 84 | , RT.importStartYear = startYear 85 | , RT.onlyNewFiles = onlyNewFiles subParams' 86 | , RT.hfVersion = versionInfo sysInfo 87 | , RT.hledgerInfo = hli 88 | , RT.sysInfo = sysInfo 89 | , RT.verbose = verbosity mainParams' > 0 90 | , RT.showOptions = showOpts mainParams' 91 | , RT.sequential = sequential mainParams' 92 | , RT.batchSize = size 93 | , RT.prettyReports = True 94 | } 95 | 96 | toRuntimeOptionsReport :: MainParams -> ReportParams -> IO RT.RuntimeOptions 97 | toRuntimeOptionsReport mainParams' subParams' = do 98 | let maybeBD = maybeReportBaseDir subParams' :: Maybe TurtlePath 99 | (bd, _) <- determineBaseDir maybeBD 100 | hli <- hledgerInfoFromPath $ hledgerPathOpt mainParams' 101 | size <- determineBatchSize mainParams' 102 | sysInfo <- systemInfo 103 | return RT.RuntimeOptions { RT.baseDir = bd 104 | , RT.importRunDir = [reldir|.|] 105 | , RT.importStartYear = Nothing 106 | , RT.onlyNewFiles = False 107 | , RT.hfVersion = versionInfo sysInfo 108 | , RT.hledgerInfo = hli 109 | , RT.sysInfo = sysInfo 110 | , RT.verbose = verbosity mainParams' > 0 111 | , RT.showOptions = showOpts mainParams' 112 | , RT.sequential = sequential mainParams' 113 | , RT.batchSize = size 114 | , RT.prettyReports = not(asciiReports subParams') 115 | } 116 | 117 | baseCommandParser :: Parser BaseCommand 118 | baseCommandParser = (Command <$> verboseParser <*> commandParser) 119 | <|> flag' Version (long "version" <> short 'V' <> help "Display version information") 120 | 121 | commandParser :: Parser Command 122 | commandParser = fmap Import (Turtle.subcommand "import" "Uses hledger with your own rules and/or scripts to convert electronic statements into categorised journal files" subcommandParserImport) 123 | <|> fmap Report (Turtle.subcommand "report" "Generate Reports" subcommandParserReport) 124 | 125 | verboseParser :: Parser MainParams 126 | verboseParser = MainParams 127 | <$> (length <$> many (flag' () (long "verbose" <> short 'v' <> help "Print more verbose output"))) 128 | <*> optional (Turtle.optPath "hledger-path" 'H' "The full path to an hledger executable") 129 | <*> switch (long "show-options" <> help "Print the options this program will run with") 130 | <*> optional (option auto (long "batch-size" <> metavar "SIZE" <> help ("Parallel processing of files are done in batches of the specified size. Default: " <> show defaultBatchSize <> ". Ignored during sequential processing."))) 131 | <*> switch (long "sequential" <> help "Disable parallel processing") 132 | 133 | subcommandParserImport :: Parser ImportParams 134 | subcommandParserImport = ImportParams 135 | <$> optional (Turtle.argPath "dir" "The directory to import. Use the base directory for a full import or a sub-directory for a partial import. Defaults to the current directory.") 136 | <*> optional (option str (long "start-year" <> metavar "YEAR" <> help "Import only from the specified year and onwards, ignoring previous years. By default all available years are imported. Valid values include a 4-digit year or 'current' for the current year")) 137 | <*> switch (long "new-files-only" <> help "Don't regenerate transaction files if they are already present. This applies to hledger journal files as well as files produced by the preprocess and construct scripts.") 138 | 139 | subcommandParserReport :: Parser ReportParams 140 | subcommandParserReport = ReportParams 141 | <$> optional (Turtle.argPath "basedir" "The hledger-flow base directory") 142 | <*> switch (long "ascii-reports" <> help "If to avoid using hledger --pretty-tables flag when generating reports.") 143 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Reports.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module Hledger.Flow.Reports 4 | ( generateReports, 5 | ) 6 | where 7 | 8 | import Control.Concurrent.STM 9 | import Data.Either 10 | import qualified Data.List as List 11 | import Data.Maybe 12 | import qualified Data.Text as T 13 | import qualified Data.Text.IO as T 14 | import Hledger.Flow.BaseDir (relativeToBase, turtleBaseDir) 15 | import Hledger.Flow.Common 16 | import Hledger.Flow.Logging 17 | import Hledger.Flow.PathHelpers (TurtlePath, pathToTurtle) 18 | import Hledger.Flow.RuntimeOptions 19 | import qualified Hledger.Flow.Types as FlowTypes 20 | import Turtle ((%), (<.>), ()) 21 | import qualified Turtle as Turtle hiding (proc, stderr, stdout) 22 | import Prelude hiding (putStrLn, readFile, writeFile) 23 | 24 | data ReportParams = ReportParams 25 | { ledgerFile :: TurtlePath, 26 | reportYears :: [Integer], 27 | outputDir :: TurtlePath 28 | } 29 | deriving (Show) 30 | 31 | type ReportGenerator = RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> TurtlePath -> Integer -> IO (Either TurtlePath TurtlePath) 32 | 33 | generateReports :: RuntimeOptions -> IO () 34 | generateReports opts = 35 | Turtle.sh 36 | ( do 37 | ch <- Turtle.liftIO newTChanIO 38 | logHandle <- Turtle.fork $ consoleChannelLoop ch 39 | Turtle.liftIO $ if (showOptions opts) then channelOutLn ch (Turtle.repr opts) else return () 40 | (reports, diff) <- Turtle.time $ Turtle.liftIO $ generateReports' opts ch 41 | let failedAttempts = lefts reports 42 | let failedText = if List.null failedAttempts then "" else Turtle.format ("(and attempted to write " % Turtle.d % " more) ") $ length failedAttempts 43 | Turtle.liftIO $ channelOutLn ch $ Turtle.format ("Generated " % Turtle.d % " reports " % Turtle.s % "in " % Turtle.s) (length (rights reports)) failedText $ Turtle.repr diff 44 | Turtle.liftIO $ terminateChannelLoop ch 45 | Turtle.wait logHandle 46 | ) 47 | 48 | generateReports' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [Either TurtlePath TurtlePath] 49 | generateReports' opts ch = do 50 | let wipMsg = 51 | "These reports can be used as a starting point for more tailored reports.\n" 52 | <> "The first line of each report contains the command used - change the parameters and use it in your own reports.\n" 53 | channelOutLn ch wipMsg 54 | owners <- Turtle.single $ shellToList $ listOwners opts 55 | ledgerEnvValue <- Turtle.need "LEDGER_FILE" :: IO (Maybe T.Text) 56 | let hledgerJournal = fromMaybe (turtleBaseDir opts allYearsFileName) $ fmap T.unpack ledgerEnvValue 57 | hledgerJournalExists <- Turtle.testfile hledgerJournal 58 | _ <- if not hledgerJournalExists then Turtle.die $ Turtle.format ("Unable to find journal file: " % Turtle.fp % "\nIs your LEDGER_FILE environment variable set correctly?") hledgerJournal else return () 59 | let journalWithYears = journalFile opts [] 60 | let aggregateReportDir = outputReportDir opts ["all"] 61 | aggregateYears <- includeYears ch journalWithYears 62 | let aggregateParams = 63 | ReportParams 64 | { ledgerFile = hledgerJournal, 65 | reportYears = aggregateYears, 66 | outputDir = aggregateReportDir 67 | } 68 | let aggregateOnlyReports = reportActions opts ch [transferBalance] aggregateParams 69 | ownerParams <- ownerParameters opts ch owners 70 | let ownerWithAggregateParams = (if length owners > 1 then [aggregateParams] else []) ++ ownerParams 71 | let sharedOptions = (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--depth", "2"] 72 | let ownerWithAggregateReports = List.concat $ fmap (reportActions opts ch [incomeStatement sharedOptions, incomeMonthlyStatement sharedOptions, balanceSheet sharedOptions]) ownerWithAggregateParams 73 | let ownerOnlyReports = List.concat $ fmap (reportActions opts ch [accountList, unknownTransactions]) ownerParams 74 | parAwareActions opts (aggregateOnlyReports ++ ownerWithAggregateReports ++ ownerOnlyReports) 75 | 76 | reportActions :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [ReportGenerator] -> ReportParams -> [IO (Either TurtlePath TurtlePath)] 77 | reportActions opts ch reports (ReportParams journal years reportsDir) = do 78 | y <- years 79 | map (\r -> r opts ch journal reportsDir y) reports 80 | 81 | accountList :: ReportGenerator 82 | accountList opts ch journal baseOutDir year = do 83 | let reportArgs = ["accounts"] 84 | generateReport opts ch journal year (baseOutDir intPath year) ("accounts" <.> "txt") reportArgs (not . T.null) 85 | 86 | unknownTransactions :: ReportGenerator 87 | unknownTransactions opts ch journal baseOutDir year = do 88 | let reportArgs = ["print", "unknown"] 89 | generateReport opts ch journal year (baseOutDir intPath year) ("unknown-transactions" <.> "txt") reportArgs (not . T.null) 90 | 91 | incomeStatement :: [T.Text] -> ReportGenerator 92 | incomeStatement sharedOptions opts ch journal baseOutDir year = do 93 | let reportArgs = ["incomestatement"] ++ sharedOptions 94 | generateReport opts ch journal year (baseOutDir intPath year) ("income-expenses" <.> "txt") reportArgs (not . T.null) 95 | 96 | incomeMonthlyStatement :: [T.Text] -> ReportGenerator 97 | incomeMonthlyStatement sharedOptions opts ch journal baseOutDir year = do 98 | let reportArgs = ["incomestatement"] ++ sharedOptions ++ ["--monthly"] 99 | generateReport opts ch journal year (baseOutDir intPath year "monthly") ("income-expenses" <.> "txt") reportArgs (not . T.null) 100 | 101 | balanceSheet :: [T.Text] -> ReportGenerator 102 | balanceSheet sharedOptions opts ch journal baseOutDir year = do 103 | let reportArgs = ["balancesheet"] ++ sharedOptions ++ ["--flat"] 104 | generateReport opts ch journal year (baseOutDir intPath year) ("balance-sheet" <.> "txt") reportArgs (not . T.null) 105 | 106 | transferBalance :: ReportGenerator 107 | transferBalance opts ch journal baseOutDir year = do 108 | let reportArgs = ["balance"] ++ (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--quarterly", "--flat", "--no-total", "transfer"] 109 | generateReport opts ch journal year (baseOutDir intPath year) ("transfer-balance" <.> "txt") reportArgs (\txt -> (length $ T.lines txt) > 4) 110 | 111 | generateReport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> Integer -> TurtlePath -> TurtlePath -> [T.Text] -> (T.Text -> Bool) -> IO (Either TurtlePath TurtlePath) 112 | generateReport opts ch journal year reportsDir fileName args successCheck = do 113 | Turtle.mktree reportsDir 114 | let outputFile = reportsDir fileName 115 | let relativeJournal = relativeToBase opts journal 116 | let relativeOutputFile = relativeToBase opts outputFile 117 | let reportArgs = ["--file", Turtle.format Turtle.fp journal, "--period", Turtle.repr year] ++ args 118 | let reportDisplayArgs = ["--file", Turtle.format Turtle.fp relativeJournal, "--period", Turtle.repr year] ++ args 119 | let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text 120 | let cmdLabel = Turtle.format ("hledger " % Turtle.s) $ showCmdArgs reportDisplayArgs 121 | ((exitCode, stdOut, _), _) <- timeAndExitOnErr opts ch cmdLabel dummyLogger channelErr Turtle.procStrictWithErr (hledger, reportArgs, mempty) 122 | if (successCheck stdOut) 123 | then do 124 | T.writeFile outputFile (cmdLabel <> "\n\n" <> stdOut) 125 | logVerbose opts ch $ Turtle.format ("Wrote " % Turtle.fp) $ relativeOutputFile 126 | return $ Right outputFile 127 | else do 128 | channelErrLn ch $ Turtle.format ("Did not write '" % Turtle.fp % "' (" % Turtle.s % ") " % Turtle.s) relativeOutputFile cmdLabel (Turtle.repr exitCode) 129 | exists <- Turtle.testfile outputFile 130 | if exists then Turtle.rm outputFile else return () 131 | return $ Left outputFile 132 | 133 | journalFile :: RuntimeOptions -> [TurtlePath] -> TurtlePath 134 | journalFile opts dirs = (foldl () (turtleBaseDir opts) ("import" : dirs)) allYearsFileName 135 | 136 | outputReportDir :: RuntimeOptions -> [TurtlePath] -> TurtlePath 137 | outputReportDir opts dirs = foldl () (turtleBaseDir opts) ("reports" : dirs) 138 | 139 | ownerParameters :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [TurtlePath] -> IO [ReportParams] 140 | ownerParameters opts ch owners = do 141 | let actions = map (ownerParameters' opts ch) owners 142 | parAwareActions opts actions 143 | 144 | ownerParameters' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> IO ReportParams 145 | ownerParameters' opts ch owner = do 146 | let ownerJournal = journalFile opts [owner] 147 | years <- includeYears ch ownerJournal 148 | return $ ReportParams ownerJournal years (outputReportDir opts [owner]) 149 | -------------------------------------------------------------------------------- /CONTRIBUTING.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | 3 | * Hledger Flow Contributor Guidelines 4 | :PROPERTIES: 5 | :CUSTOM_ID: hledger-flow-contributor-guidelines 6 | :END: 7 | 8 | Please read the [[#hledger-flow-contributor-agreement][Contributor Agreement]] before continuing below. 9 | By submitting a contribution it means that you agree to the terms in the 10 | contributor's agreement. 11 | 12 | * List of Contributors 13 | :PROPERTIES: 14 | :CUSTOM_ID: list-of-contributors 15 | :END: 16 | 17 | The complete list of contributors can be seen in the git logs or 18 | [[https://github.com/apauley/hledger-flow/graphs/contributors][on GitHub]]. 19 | 20 | If you have made a contribution, please consider adding your name (or github 21 | username) to the list below, and include it in a new or existing pull request. 22 | 23 | Hledger Flow is brought to you by: 24 | - [[https://github.com/apauley][Andreas Pauley]] 25 | 26 | Miscellaneous contributors: 27 | - [[https://github.com/apauley/hledger-flow/issues?q=author%3Alestephane][Le Stephane]] (various contributions to issues with suggested solutions and always with thoughtful feedback) 28 | - [[https://github.com/apauley/hledger-flow/issues?q=author%3Ajecaro][Jean-Charles Quillet]] (improved command-line parameters) 29 | - [[https://github.com/apauley/hledger-flow/issues?q=author%3Akain88-de][Max Linke]] (added a monthly version of the income statement in reports) 30 | - [[https://github.com/apauley/hledger-flow/issues?q=author%3Akepi][Kepi]] (doc fixes) 31 | 32 | * Contributing 33 | :PROPERTIES: 34 | :CUSTOM_ID: contributing 35 | :END: 36 | 37 | To begin contributing, please follow these steps: 38 | 39 | 1. [[#get-the-project][Get the Project]] 40 | 2. [[#build-the-project][Build the Project]] 41 | 3. [[#find-an-issue][Find an Issue]] 42 | 4. [[#create-a-pull-request][Create a Pull Request]] 43 | 5. [[#get-your-pull-request-merged][Get Your Pull Request Merged]] 44 | 45 | ** Getting Started 46 | :PROPERTIES: 47 | :CUSTOM_ID: getting-started 48 | :END: 49 | 50 | *** Get The Project 51 | :PROPERTIES: 52 | :CUSTOM_ID: get-the-project 53 | :END: 54 | 55 | If you don't already have one, sign up for a free 56 | [[https://github.com/join][Github Account]]. 57 | 58 | After you [[https://github.com/login][log into]] Github using your 59 | account, go to the [[https://github.com/apauley/hledger-flow][Hledger Flow Project Page]], and click on [[https://github.com/apauley/hledger-flow/fork][Fork]] to fork the 60 | Hledger Flow repository into your own account. 61 | 62 | Once you have forked the repository, you can now clone your forked 63 | repository to your own machine, so you have a complete copy of the 64 | project and can begin safely making your modifications. 65 | 66 | To clone your forked repository, first make sure you have installed 67 | [[https://git-scm.com/downloads][Git]], the version control system used 68 | by Github. Then open a Terminal and type the following commands: 69 | 70 | #+BEGIN_SRC sh 71 | mkdir hledger-flow 72 | cd hledger-flow 73 | git clone git@github.com:apauley/hledger-flow.git . 74 | #+END_SRC 75 | 76 | This repository has some submodules included, mostly related to the 77 | examples in the documentation. 78 | 79 | You need to initialise and update the submodules: 80 | 81 | #+BEGIN_SRC sh 82 | git submodule init 83 | git submodule update 84 | #+END_SRC 85 | 86 | *** Build the Project 87 | :PROPERTIES: 88 | :CUSTOM_ID: build-the-project 89 | :END: 90 | 91 | You need a recent version of [[https://docs.haskellstack.org/en/stable/README/][stack]] installed. 92 | 93 | Then run: 94 | 95 | #+NAME: build-script 96 | #+BEGIN_SRC sh 97 | ./bin/build-and-test 98 | #+END_SRC 99 | 100 | Which should end with this: 101 | 102 | #+BEGIN_SRC org 103 | Copied executables to ~/.local/bin: 104 | - hledger-flow 105 | #+END_SRC 106 | 107 | Ensure that =${HOME}/.local/bin= is in your =PATH=. 108 | 109 | Usually this means adding this to your =~/.bashrc=: 110 | 111 | #+BEGIN_SRC sh 112 | PATH="${HOME}/.local/bin:${PATH}" 113 | #+END_SRC 114 | 115 | *** Find an Issue 116 | :PROPERTIES: 117 | :CUSTOM_ID: find-an-issue 118 | :END: 119 | 120 | You may have your own idea about what contributions to make to Hledger 121 | Flow, which is great! If you want to make sure the Hledger Flow 122 | contributors are open to your idea, you can 123 | [[https://github.com/apauley/hledger-flow/issues/new][open an issue]] 124 | first on the Hledger Flow project site. 125 | 126 | Otherwise, if you have no ideas about what to contribute, you can find a 127 | list of issues on the project's [[https://github.com/apauley/hledger-flow/issues][issue tracker]]. 128 | 129 | Issues are tagged with various labels, such as =good first issue= or 130 | =help wanted=, which can help you find issues that are a fit for you. 131 | 132 | If some issue is confusing or you think you might need help, then just 133 | post a comment on the issue asking for help. 134 | 135 | Once you've decided on an issue and understand what is necessary to 136 | complete the issue, then it's a good idea to post a comment on the issue 137 | saying that you intend to work on it. Otherwise, someone else might work 138 | on it too! 139 | 140 | *** Create a Pull Request 141 | :PROPERTIES: 142 | :CUSTOM_ID: create-a-pull-request 143 | :END: 144 | 145 | To create a pull request, first push all your changes to your fork of 146 | the project repository: 147 | 148 | #+BEGIN_SRC sh 149 | git push 150 | #+END_SRC 151 | 152 | Next, [[https://github.com/apauley/hledger-flow/compare][open a new pull request]] on Github, and select /Compare Across Forks/. 153 | On the right hand side, choose your own fork of the Hledger Flow repository, 154 | in which you've been making your contribution. 155 | 156 | Provide a description for the pull request, which details the issue it 157 | is fixing, and has other information that may be helpful to developers 158 | reviewing the pull request. 159 | 160 | Finally, click /Create Pull Request/! 161 | 162 | *** Get Your Pull Request Merged 163 | :PROPERTIES: 164 | :CUSTOM_ID: get-your-pull-request-merged 165 | :END: 166 | 167 | Once you have a pull request open, it's still your job to get it merged! 168 | To get it merged, a core contributor has to approve the code. 169 | 170 | Code reviews can sometimes take a few days, because open source projects 171 | are largely done outside of work, in people's leisure time. Be patient, 172 | but don't wait forever. If you haven't gotten a review within a few 173 | days, then consider gently reminding people that you need a review. 174 | 175 | Once you receive a review, you will probably have to go back and make 176 | minor changes that improve your contribution and make it follow existing 177 | conventions in the code base. This is normal, even for experienced 178 | contributors, and the rigorous reviews help ensure that the quality of 179 | the code stays high. 180 | 181 | After you make changes, you may need to remind reviewers to check out 182 | the code again. If they give a final approval, it means your code is 183 | ready for merge! 184 | 185 | If you don't get a merge in a day after your review is successful, then 186 | please gently remind folks that your code is ready to be merged. 187 | 188 | 189 | * Hledger Flow Contributor Agreement 190 | :PROPERTIES: 191 | :CUSTOM_ID: hledger-flow-contributor-agreement 192 | :END: 193 | 194 | Thank you for your interest in contributing to the Hledger Flow open 195 | source project. 196 | 197 | This is the official contributor agreement for the Hledger Flow project. 198 | 199 | The purpose of this agreement is to ensure: 200 | 1. that there is a clear legal status and audit trail for the project 201 | 2. that you get proper credit for your work 202 | 3. that we are able to remain license-compatible with related software by 203 | updating to newer versions of our license when appropriate (eg maintaining 204 | compatibility with [[https://hledger.org/][hledger]]) 205 | 206 | By submitting a contribution you declare that all of your contributions to 207 | hledger-flow: 208 | - are free of patent violations or copyright violations, to the best of your knowledge 209 | - are released under the hledger-flow project's license 210 | - are granted legal ownership to both yourself and the project leaders of hledger-flow 211 | - may be relicensed in future at the discretion of the project leader 212 | 213 | This contributor agreement describes the terms and conditions under which you 214 | may submit a contribution to us. By submitting a contribution to us, you accept 215 | the terms and conditions in the agreement. If you do not accept the terms and 216 | conditions in the agreement, you must not submit any contribution to us. 217 | 218 | Although it is not required, we encourage you to add your name to the 219 | [[#list-of-contributors][list of contributors]] if you have made a contribution to the project. 220 | -------------------------------------------------------------------------------- /test/CSVImport/ImportHelperTests.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | {-# LANGUAGE QuasiQuotes #-} 4 | 5 | module CSVImport.ImportHelperTests where 6 | 7 | import Hledger.Flow.Import.ImportHelpers (groupIncludesUpTo, includeFileName) 8 | import Hledger.Flow.Import.Types (InputFileBundle) 9 | import Path 10 | import Test.HUnit 11 | import TestHelpers 12 | 13 | testToJournal :: Test 14 | testToJournal = 15 | TestCase 16 | ( do 17 | let journal = toJournal [relfile|import/jane/bogartbank/investment/1-in/2020/2020-09-30.csv|] 18 | let expected = [relfile|import/jane/bogartbank/investment/3-journal/2020/2020-09-30.journal|] 19 | assertEqual "toJournal" expected journal 20 | ) 21 | 22 | testIncludeFileName :: Test 23 | testIncludeFileName = 24 | TestCase 25 | ( do 26 | let includeFile = includeFileName [relfile|import/jane/bogartbank/investment/3-journals/2020/2020-09-30.journal|] 27 | assertEqual "includeFileName" [relfile|2020-include.journal|] includeFile 28 | ) 29 | 30 | testGroupIncludesUpToTinySet :: Test 31 | testGroupIncludesUpToTinySet = 32 | TestCase 33 | ( do 34 | let expected = 35 | [ ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017]), 36 | ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]]), 37 | ([relfile|import/jane/2017-include.journal|], [[relfile|import/jane/bogartbank/2017-include.journal|]]) 38 | ] :: 39 | InputFileBundle 40 | 41 | let grouped = groupIncludesUpTo [reldir|import/jane|] [janeSavingsJournal2017] 42 | assertEqual "groupIncludesUpTo: A single journal file grouping" expected grouped 43 | ) 44 | 45 | testGroupIncludesUpToSmallSet :: Test 46 | testGroupIncludesUpToSmallSet = 47 | TestCase 48 | ( do 49 | let expected = 50 | [ ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017]), 51 | ([relfile|import/jane/bogartbank/savings/2018-include.journal|], janeSavingsJournals2018), 52 | ([relfile|import/jane/bogartbank/savings/2019-include.journal|], [janeSavingsJournal2019]), 53 | ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]]), 54 | ([relfile|import/jane/bogartbank/2018-include.journal|], [[relfile|import/jane/bogartbank/savings/2018-include.journal|]]), 55 | ([relfile|import/jane/bogartbank/2019-include.journal|], [[relfile|import/jane/bogartbank/savings/2019-include.journal|]]), 56 | ([relfile|import/jane/2017-include.journal|], [[relfile|import/jane/bogartbank/2017-include.journal|]]), 57 | ([relfile|import/jane/2018-include.journal|], [[relfile|import/jane/bogartbank/2018-include.journal|]]), 58 | ([relfile|import/jane/2019-include.journal|], [[relfile|import/jane/bogartbank/2019-include.journal|]]) 59 | ] :: 60 | InputFileBundle 61 | 62 | let grouped = groupIncludesUpTo [reldir|import/jane|] janeSavingsJournals 63 | assertEqual "groupIncludesUpTo: A small set of journal files - same account over 3 years" expected grouped 64 | ) 65 | 66 | testGroupIncludesUpTo :: Test 67 | testGroupIncludesUpTo = 68 | TestCase 69 | ( do 70 | let expected = 71 | [ ([relfile|import/john/bogartbank/savings/2017-include.journal|], johnSavingsJournals2017), 72 | ([relfile|import/john/bogartbank/savings/2018-include.journal|], johnSavingsJournals2018), 73 | ([relfile|import/john/bogartbank/checking/2018-include.journal|], johnCheckingJournals2018), 74 | ([relfile|import/john/bogartbank/checking/2019-include.journal|], johnCheckingJournals2019), 75 | ([relfile|import/john/otherbank/creditcard/2017-include.journal|], [johnCCJournal2017]), 76 | ([relfile|import/john/otherbank/creditcard/2018-include.journal|], [johnCCJournal2018]), 77 | ([relfile|import/john/otherbank/investments/2018-include.journal|], [johnInvestJournal2018]), 78 | ([relfile|import/john/otherbank/investments/2019-include.journal|], [johnInvestJournal2019]), 79 | ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017]), 80 | ([relfile|import/jane/bogartbank/savings/2018-include.journal|], janeSavingsJournals2018), 81 | ([relfile|import/jane/bogartbank/savings/2019-include.journal|], [janeSavingsJournal2019]), 82 | ([relfile|import/jane/otherbank/creditcard/2017-include.journal|], [janeCCJournal2017]), 83 | ([relfile|import/jane/otherbank/creditcard/2018-include.journal|], [janeCCJournal2018]), 84 | ([relfile|import/jane/otherbank/investments/2018-include.journal|], [janeInvestJournal2018]), 85 | ([relfile|import/jane/otherbank/investments/2019-include.journal|], [janeInvestJournal2019]), 86 | ([relfile|import/john/bogartbank/2017-include.journal|], [[relfile|import/john/bogartbank/savings/2017-include.journal|]]), 87 | ([relfile|import/john/bogartbank/2018-include.journal|], [[relfile|import/john/bogartbank/checking/2018-include.journal|], [relfile|import/john/bogartbank/savings/2018-include.journal|]]), 88 | ([relfile|import/john/bogartbank/2019-include.journal|], [[relfile|import/john/bogartbank/checking/2019-include.journal|]]), 89 | ([relfile|import/john/otherbank/2017-include.journal|], [[relfile|import/john/otherbank/creditcard/2017-include.journal|]]), 90 | ([relfile|import/john/otherbank/2018-include.journal|], [[relfile|import/john/otherbank/creditcard/2018-include.journal|], [relfile|import/john/otherbank/investments/2018-include.journal|]]), 91 | ([relfile|import/john/otherbank/2019-include.journal|], [[relfile|import/john/otherbank/investments/2019-include.journal|]]), 92 | ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]]), 93 | ([relfile|import/jane/bogartbank/2018-include.journal|], [[relfile|import/jane/bogartbank/savings/2018-include.journal|]]), 94 | ([relfile|import/jane/bogartbank/2019-include.journal|], [[relfile|import/jane/bogartbank/savings/2019-include.journal|]]), 95 | ([relfile|import/jane/otherbank/2017-include.journal|], [[relfile|import/jane/otherbank/creditcard/2017-include.journal|]]), 96 | ([relfile|import/jane/otherbank/2018-include.journal|], [[relfile|import/jane/otherbank/creditcard/2018-include.journal|], [relfile|import/jane/otherbank/investments/2018-include.journal|]]), 97 | ([relfile|import/jane/otherbank/2019-include.journal|], [[relfile|import/jane/otherbank/investments/2019-include.journal|]]), 98 | ([relfile|import/john/2017-include.journal|], [[relfile|import/john/bogartbank/2017-include.journal|], [relfile|import/john/otherbank/2017-include.journal|]]), 99 | ([relfile|import/john/2018-include.journal|], [[relfile|import/john/bogartbank/2018-include.journal|], [relfile|import/john/otherbank/2018-include.journal|]]), 100 | ([relfile|import/john/2019-include.journal|], [[relfile|import/john/bogartbank/2019-include.journal|], [relfile|import/john/otherbank/2019-include.journal|]]), 101 | ([relfile|import/jane/2017-include.journal|], [[relfile|import/jane/bogartbank/2017-include.journal|], [relfile|import/jane/otherbank/2017-include.journal|]]), 102 | ([relfile|import/jane/2018-include.journal|], [[relfile|import/jane/bogartbank/2018-include.journal|], [relfile|import/jane/otherbank/2018-include.journal|]]), 103 | ([relfile|import/jane/2019-include.journal|], [[relfile|import/jane/bogartbank/2019-include.journal|], [relfile|import/jane/otherbank/2019-include.journal|]]), 104 | ([relfile|import/2017-include.journal|], [[relfile|import/jane/2017-include.journal|], [relfile|import/john/2017-include.journal|]]), 105 | ([relfile|import/2018-include.journal|], [[relfile|import/jane/2018-include.journal|], [relfile|import/john/2018-include.journal|]]), 106 | ([relfile|import/2019-include.journal|], [[relfile|import/jane/2019-include.journal|], [relfile|import/john/2019-include.journal|]]) 107 | ] :: 108 | InputFileBundle 109 | 110 | let grouped = groupIncludesUpTo [reldir|import|] journalFiles 111 | assertEqual "groupIncludesUpTo: A full set of journal files" expected grouped 112 | ) 113 | 114 | tests :: Test 115 | tests = TestList [testToJournal, testIncludeFileName, testGroupIncludesUpToTinySet, testGroupIncludesUpToSmallSet, testGroupIncludesUpTo] 116 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Import/ImportHelpersTurtle.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | -- | Functions which currently uses TurtlePath and will be replaced with Path eventually 4 | module Hledger.Flow.Import.ImportHelpersTurtle 5 | ( allYearIncludeFiles, 6 | extractImportDirs, 7 | extraIncludesForFile, 8 | groupIncludeFiles, 9 | groupAndWriteIncludeFiles, 10 | includePreamble, 11 | toIncludeFiles, 12 | toIncludeLine, 13 | writeIncludesUpTo, 14 | writeToplevelAllYearsInclude, 15 | yearsIncludeMap, 16 | ) 17 | where 18 | 19 | import Control.Concurrent.STM (TChan) 20 | import qualified Data.List as List (nub, sort) 21 | import qualified Data.Map.Strict as Map 22 | import Data.Maybe (fromMaybe) 23 | import qualified Data.Text as T 24 | import Hledger.Flow.BaseDir (relativeToBase, relativeToBase', turtleBaseDir) 25 | import Hledger.Flow.Common (allYearsFileName, directivesFile, filterPaths, groupValuesBy, writeFiles, writeFiles') 26 | import Hledger.Flow.DocHelpers (docURL) 27 | import Hledger.Flow.Import.Types 28 | import Hledger.Flow.Logging (logVerbose) 29 | import Hledger.Flow.PathHelpers (TurtlePath) 30 | import Hledger.Flow.Types 31 | import Turtle ((%), (<.>), ()) 32 | import qualified Turtle 33 | 34 | extractImportDirs :: TurtlePath -> Either T.Text ImportDirs 35 | extractImportDirs inputFile = do 36 | case importDirBreakdown inputFile of 37 | [bd, owner, bank, account, filestate, year] -> Right $ ImportDirs bd owner bank account filestate year 38 | _ -> do 39 | Left $ 40 | Turtle.format 41 | ( "I couldn't find the right number of directories between \"import\" and the input file:\n" 42 | % Turtle.fp 43 | % "\n\nhledger-flow expects to find input files in this structure:\n" 44 | % "import/owner/bank/account/filestate/year/trxfile\n\n" 45 | % "Have a look at the documentation for a detailed explanation:\n" 46 | % Turtle.s 47 | ) 48 | inputFile 49 | (docURL "input-files") 50 | 51 | importDirBreakdown :: TurtlePath -> [TurtlePath] 52 | importDirBreakdown = importDirBreakdown' [] 53 | 54 | importDirBreakdown' :: [TurtlePath] -> TurtlePath -> [TurtlePath] 55 | importDirBreakdown' acc path = do 56 | let dir = Turtle.directory path 57 | if Turtle.dirname dir == "import" || (Turtle.dirname dir == "") 58 | then dir : acc 59 | else importDirBreakdown' (dir : acc) $ Turtle.parent dir 60 | 61 | groupIncludeFiles :: [TurtlePath] -> (TurtleFileBundle, TurtleFileBundle) 62 | groupIncludeFiles = allYearIncludeFiles . groupIncludeFilesPerYear . filter isJournalFile 63 | 64 | isJournalFile :: TurtlePath -> Bool 65 | isJournalFile f = Turtle.extension f == Just "journal" 66 | 67 | allYearIncludeFiles :: TurtleFileBundle -> (TurtleFileBundle, TurtleFileBundle) 68 | allYearIncludeFiles m = (m, yearsIncludeMap $ Map.keys m) 69 | 70 | yearsIncludeMap :: [TurtlePath] -> TurtleFileBundle 71 | yearsIncludeMap = groupValuesBy allYearsPath 72 | 73 | allYearsPath :: TurtlePath -> TurtlePath 74 | allYearsPath = allYearsPath' Turtle.directory 75 | 76 | allYearsPath' :: (TurtlePath -> TurtlePath) -> TurtlePath -> TurtlePath 77 | allYearsPath' dir p = dir p allYearsFileName 78 | 79 | includeFileName :: TurtlePath -> TurtlePath 80 | includeFileName = (<.> "journal") . T.unpack . (Turtle.format (Turtle.fp % "-include")) . Turtle.dirname 81 | 82 | groupIncludeFilesPerYear :: [TurtlePath] -> TurtleFileBundle 83 | groupIncludeFilesPerYear [] = Map.empty 84 | groupIncludeFilesPerYear ps@(p : _) = case extractImportDirs p of 85 | Right _ -> groupValuesBy initialIncludeFilePath ps 86 | Left _ -> groupValuesBy parentIncludeFilePath ps 87 | 88 | initialIncludeFilePath :: TurtlePath -> TurtlePath 89 | initialIncludeFilePath p = (Turtle.parent . Turtle.parent . Turtle.parent) p includeFileName p 90 | 91 | parentIncludeFilePath :: TurtlePath -> TurtlePath 92 | parentIncludeFilePath p = (Turtle.parent . Turtle.parent) p Turtle.filename p 93 | 94 | toIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtleFileBundle -> IO (Map.Map TurtlePath T.Text) 95 | toIncludeFiles opts ch m = do 96 | preMap <- extraIncludes opts ch (Map.keys m) ["opening.journal"] ["pre-import.journal"] [] 97 | postMap <- extraIncludes opts ch (Map.keys m) ["closing.journal"] ["post-import.journal"] ["prices.journal"] 98 | return $ (addPreamble . toIncludeFiles' preMap postMap) m 99 | 100 | toIncludeFiles' :: TurtleFileBundle -> TurtleFileBundle -> TurtleFileBundle -> Map.Map TurtlePath T.Text 101 | toIncludeFiles' preMap postMap = Map.mapWithKey $ generatedIncludeText preMap postMap 102 | 103 | addPreamble :: Map.Map TurtlePath T.Text -> Map.Map TurtlePath T.Text 104 | addPreamble = Map.map (\txt -> includePreamble <> "\n" <> txt) 105 | 106 | toIncludeLine :: TurtlePath -> TurtlePath -> T.Text 107 | toIncludeLine base file = Turtle.format ("include " % Turtle.fp) $ relativeToBase' base file 108 | 109 | generatedIncludeText :: TurtleFileBundle -> TurtleFileBundle -> TurtlePath -> [TurtlePath] -> T.Text 110 | generatedIncludeText preMap postMap outputFile fs = do 111 | let preFiles = fromMaybe [] $ Map.lookup outputFile preMap 112 | let files = List.nub . List.sort $ fs 113 | let postFiles = fromMaybe [] $ Map.lookup outputFile postMap 114 | let lns = map (toIncludeLine $ Turtle.directory outputFile) $ preFiles ++ files ++ postFiles 115 | T.intercalate "\n" $ lns ++ [""] 116 | 117 | includePreamble :: T.Text 118 | includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n" 119 | 120 | groupAndWriteIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> IO [TurtlePath] 121 | groupAndWriteIncludeFiles opts ch = writeFileMap opts ch . groupIncludeFiles 122 | 123 | writeFileMap :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> (TurtleFileBundle, TurtleFileBundle) -> IO [TurtlePath] 124 | writeFileMap opts ch (m, allYears) = do 125 | _ <- writeFiles' $ (addPreamble . toIncludeFiles' Map.empty Map.empty) allYears 126 | writeFiles . (toIncludeFiles opts ch) $ m 127 | 128 | writeIncludesUpTo :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtlePath -> [TurtlePath] -> IO [TurtlePath] 129 | writeIncludesUpTo _ _ _ [] = return [] 130 | writeIncludesUpTo opts ch stopAt journalFiles = do 131 | let shouldStop = any (\dir -> dir == stopAt) $ map Turtle.parent journalFiles 132 | if shouldStop 133 | then return journalFiles 134 | else do 135 | newJournalFiles <- groupAndWriteIncludeFiles opts ch journalFiles 136 | writeIncludesUpTo opts ch stopAt newJournalFiles 137 | 138 | writeToplevelAllYearsInclude :: (HasBaseDir o, HasVerbosity o) => o -> IO [TurtlePath] 139 | writeToplevelAllYearsInclude opts = do 140 | directivesExists <- Turtle.testfile (directivesFile opts) 141 | let preMap = if directivesExists then Map.singleton (turtleBaseDir opts allYearsFileName) [directivesFile opts] else Map.empty 142 | let allTop = Map.singleton (turtleBaseDir opts allYearsFileName) ["import" allYearsFileName] 143 | writeFiles' $ (addPreamble . toIncludeFiles' preMap Map.empty) allTop 144 | 145 | extraIncludes :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle 146 | extraIncludes opts ch = extraIncludes' opts ch Map.empty 147 | 148 | extraIncludes' :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtleFileBundle -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle 149 | extraIncludes' _ _ acc [] _ _ _ = return acc 150 | extraIncludes' opts ch acc (file : files) extraSuffixes manualFiles prices = do 151 | extra <- extraIncludesForFile opts ch file extraSuffixes manualFiles prices 152 | extraIncludes' opts ch (Map.unionWith (++) acc extra) files extraSuffixes manualFiles prices 153 | 154 | extraIncludesForFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> TurtlePath -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle 155 | extraIncludesForFile opts ch file extraSuffixes manualFiles prices = do 156 | let dirprefix = T.unpack $ fst $ T.breakOn "-" $ Turtle.format Turtle.fp $ Turtle.basename file 157 | let fileNames = map (T.unpack . Turtle.format (Turtle.fp % "-" % Turtle.s) dirprefix) extraSuffixes 158 | let suffixFiles = map (Turtle.directory file ) fileNames 159 | let suffixDirFiles = map (((Turtle.directory file "_manual_") dirprefix) ) manualFiles 160 | let priceFiles = map ((((Turtle.directory file "..") "prices") dirprefix) ) prices 161 | let extraFiles = suffixFiles ++ suffixDirFiles ++ priceFiles 162 | filtered <- Turtle.single $ filterPaths Turtle.testfile extraFiles 163 | let logMsg = 164 | Turtle.format 165 | ("Looking for possible extra include files for '" % Turtle.fp % "' among these " % Turtle.d % " options: " % Turtle.s % ". Found " % Turtle.d % ": " % Turtle.s) 166 | (relativeToBase opts file) 167 | (length extraFiles) 168 | (Turtle.repr $ relativeFilesAsText opts extraFiles) 169 | (length filtered) 170 | (Turtle.repr $ relativeFilesAsText opts filtered) 171 | logVerbose opts ch logMsg 172 | return $ Map.fromList [(file, filtered)] 173 | 174 | relativeFilesAsText :: (HasBaseDir o) => o -> [TurtlePath] -> [T.Text] 175 | relativeFilesAsText opts = map (Turtle.format Turtle.fp . relativeToBase opts) 176 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | 3 | * hledger-flow 4 | :PROPERTIES: 5 | :CUSTOM_ID: hledger-flow 6 | :END: 7 | 8 | ** Project Status 9 | 10 | I haven't been able to spend much time on this project in the last few years, 11 | and looking at my current responsibilities this is likely to remain the case. 12 | 13 | I do still try to respond to bug reports as they happen. 14 | 15 | ** What is it? 16 | :PROPERTIES: 17 | :CUSTOM_ID: what-is-it 18 | :END: 19 | 20 | =hledger-flow= is a command-line program that gives you a guided [[https://hledger.org/][Hledger]] 21 | workflow. It is important to note that most of the heavy lifting is done by the 22 | upstream =hledger= project. For example, =hledger-flow= cares about where you 23 | put your files for long-term maintainability, but the actual conversion to 24 | classified accounting journals is done by =hledger=. 25 | 26 | =hledger-flow= focuses on automated processing of electronic statements as much as possible, 27 | as opposed to manually adding your own hledger journal entries. Manual entries 28 | are still possible, it just saves time in the long run to automatically process 29 | a statement whenever one is available. 30 | 31 | Within =hledger-flow= you will keep your original bank statements around 32 | permanently as input, and generate (h)ledger journals each time 33 | you run the program. The classification is done with [[https://hledger.org/csv.html][hledger's rules files]], 34 | and/or your own script hooks. 35 | 36 | Keeping the original statements means that you never have to worry too 37 | much about "am I doing this accounting thing right?" or "what happens if 38 | I make a mistake?". If you want to change your mind about some 39 | classification, or if you made a mistake, you just change your 40 | classification rules, and run the program again. 41 | 42 | It started when I realized that the scripts I wrote while playing around with 43 | [[https://github.com/adept/full-fledged-hledger/wiki][adept's Full-fledged Hledger]] aren't really specific to my own finances, and can 44 | be shared. 45 | 46 | ** Overview of the Basic Workflow 47 | :PROPERTIES: 48 | :CUSTOM_ID: overview-of-the-basic-workflow 49 | :END: 50 | 51 | 1. Save an input transaction file (typically CSV) to a [[#input-files][specific directory]]. 52 | 2. Add an hledger [[#rules-files][rules file]]. 53 | Include some classification rules if you want. 54 | 3. Run =hledger-flow import= 55 | 56 | Add all your files to your favourite version control system. 57 | 58 | The generated journal that you most likely want to use as your 59 | =LEDGER_FILE= is called =all-years.journal=. This has include directives 60 | to all the automatically imported journals, as well as includes for your 61 | own manually managed journal entries. 62 | 63 | In a typical software project we don't add generated files to version 64 | control, but in this case I think it is a good idea to add all the 65 | generated files to version control as well - when you inevitably change 66 | something, e.g. how you classify transactions in your rules file, then 67 | you can easily see if your change had the desired effect by looking at a 68 | diff. 69 | 70 | ** Who should use this? 71 | :PROPERTIES: 72 | :CUSTOM_ID: who-should-use-this 73 | :END: 74 | 75 | =hledger-flow= is intended for you if: 76 | 77 | - You want a way to organise your finances into a structure that will be 78 | maintainable over the long term. 79 | - You like the idea of treating your source transactions (typically CSV files) 80 | as input, and having your hledger journals (mostly) being generated as output. 81 | - You want to automate as much as possible when dealing with your 82 | financial life. 83 | - You don't mind writing some scripts when needed, as long as it saves 84 | you time over the long term. 85 | 86 | ** How do I install it? 87 | :PROPERTIES: 88 | :CUSTOM_ID: how-do-i-install-it 89 | :END: 90 | 91 | If you can compile it yourself, please do so by following the [[https://github.com/apauley/hledger-flow/blob/master/CONTRIBUTING.org#build-the-project][build instructions]]. 92 | 93 | Otherwise download [[https://github.com/apauley/hledger-flow/releases][the latest release]] for your OS (Linux or Mac OS X), and copy the =hledger-flow= executable to a directory in your PATH. 94 | 95 | On Linux you should just be able to run the executable. 96 | On Mac you may see a warning. 97 | 98 | ** Windows Support 99 | 100 | Currently =hledger-flow= does not work on Windows. 101 | 102 | This [[https://github.com/apauley/hledger-flow/issues?q=is%3Aissue+is%3Aopen+label%3Awindows][list of issues]] describes some of the details of what doesn't work. 103 | 104 | I believe it wouldn't take too much effort to fix those issues, but I'm going to leave Windows support 105 | for other contributors. 106 | 107 | Please send me some pull requests if you would like =hledger-flow= to work on Windows. 108 | 109 | ** Getting Started 110 | :PROPERTIES: 111 | :CUSTOM_ID: getting-started 112 | :END: 113 | 114 | Have a look at the [[file:step-by-step/README.org][detailed step-by-step instructions]] and the [[file:docs/README.org][feature reference]]. 115 | 116 | You can see the example imported financial transactions as it was 117 | generated by the step-by-step instructions here: 118 | 119 | [[https://github.com/apauley/hledger-flow-example][https://github.com/apauley/hledger-flow-example]] 120 | 121 | ** Compatibility with hledger 122 | 123 | =hledger-flow= should work with any recent version of =hledger=. 124 | 125 | The most recent version that it was tested with is [[https://hledger.org/relnotes.html#2025-09-03-hledger-150][hledger-1.50]] (September 2025). 126 | 127 | Note to future readers: if you are using =hledger-flow= with a more recent version of =hledger= than mentioned above, 128 | please submit a small pull request with the updated version of =hledger=. 129 | 130 | ** Compatibility with ledger-cli 131 | :PROPERTIES: 132 | :CUSTOM_ID: compatibility-with-ledger 133 | :END: 134 | 135 | =hledger-flow= uses =hledger= to produce journal's, so this page about [[https://hledger.org/ledger.html][hledger and Ledger]] should be relevant for all =hledger-flow= users. 136 | 137 | That said, here are some observations that are specifically relevant to =hledger-flow=: 138 | 139 | When writing out the journal include files, =hledger-flow= sorts the 140 | include statements by filename. 141 | 142 | [[https://www.ledger-cli.org/][Ledger]] fails any balance assertions 143 | when the transactions aren't included in chronological order. 144 | 145 | An easy way around this is to name your input files so that March's 146 | statement is listed before December's statement. 147 | 148 | Another option is to add =--permissive= to any 149 | [[https://www.ledger-cli.org/][ledger]] command. 150 | 151 | So you should easily be able to use both =ledger= and =hledger= on these 152 | journals if you take care to [[https://hledger.org/faq.html#how-is-hledger-different-from-ledger-][avoid the few incompatibilities]] which exists 153 | (eg in your rules files or manual journals). 154 | 155 | ** Project Goals 156 | :PROPERTIES: 157 | :CUSTOM_ID: project-goals 158 | :END: 159 | 160 | My =hledger= files started to collect a bunch of supporting code that 161 | weren't really specific to my financial situation. 162 | 163 | I want to extract and share as much as possible of that supporting code. 164 | 165 | [[https://github.com/adept/full-fledged-hledger/wiki][Adept's]] goals 166 | also resonated with me: 167 | 168 | - Tracking expenses should take as little time, effort and manual work 169 | as possible 170 | - Eventual consistency should be achievable: even if I can't record 171 | something precisely right now, maybe I would be able to do it later, 172 | so I should be able to leave things half-done and pick them up later 173 | - Ability to refactor is a must. I want to be able to go back and change 174 | the way I am doing things, with as little effort as possible and 175 | without fear of irrevocably breaking things. 176 | 177 | 178 | * Contributing to hledger-flow 179 | 180 | Have a look at the [[file:CONTRIBUTING.org][contribution guidelines]]. 181 | 182 | * FAQ 183 | :PROPERTIES: 184 | :CUSTOM_ID: faq 185 | :END: 186 | 187 | ** How do you balance transfers between 2 accounts when you have statements for both accounts? 188 | :PROPERTIES: 189 | :CUSTOM_ID: transfer-2-accounts 190 | :END: 191 | 192 | *** The Problem 193 | 194 | In your primary bank account you've happily been classifying transfers to a 195 | secondary account as just =Expenses:OtherAccount=. 196 | 197 | But you've recently started processing the statements from the second account as 198 | well so that you can classify those expenses more accurately. 199 | 200 | And now the balances of these two accounts are all wrong when the statements of 201 | each account deals with money transferred between these two accounts. 202 | 203 | In =bank1.journal=, imported from =bank1.csv=: 204 | #+BEGIN_EXAMPLE 205 | 2018/11/09 Transfer from primary account to secondary account 206 | Assets:Bank1:Primary $-200 207 | Assets:Bank2:Secondary 208 | #+END_EXAMPLE 209 | 210 | In =bank2.journal=, imported from =bank2.csv=: 211 | #+BEGIN_EXAMPLE 212 | 2018/11/09 Transfer from primary account to secondary account 213 | Assets:Bank2:Secondary $200 214 | Assets:Bank1:Primary 215 | #+END_EXAMPLE 216 | 217 | *** The Solution 218 | 219 | As soon as you start importing statements for both accounts you will have to 220 | introduce an intermediate account for classification between these two accounts. 221 | 222 | I use =Assets:Transfers:*=. 223 | 224 | And we may have reports looking at these transfers accounts at some point, you 225 | should consider using the same names. 226 | 227 | The above example then becomes as follows. 228 | 229 | In =bank1.journal=, imported from =bank1.csv=: 230 | #+BEGIN_EXAMPLE 231 | 2019-05-18 Transfer from primary account to secondary account 232 | Assets:Bank1:Primary $-200 233 | Assets:Transfers:Bank1Bank2 234 | #+END_EXAMPLE 235 | 236 | In =bank2.journal=, imported from =bank2.csv=: 237 | #+BEGIN_EXAMPLE 238 | 2019-05-18 Transfer from primary account to secondary account 239 | Assets:Bank2:Secondary $200 240 | Assets:Transfers:Bank1Bank2 241 | #+END_EXAMPLE 242 | 243 | Any posting to =Assets:Transfers:*= indicates an in "in-flight" amount. 244 | You would expect the balance of =Assets:Transfers= to be zero most of the time. 245 | Whenever it isn't zero it means that you either don't yet have the other side of 246 | the transfer, or that something is wrong in your rules. 247 | 248 | You could theoretically just use =Assets:Transfers= without any subaccounts, but 249 | I found it useful to use subaccounts. Because then the subaccounts can show me 250 | where I should look for any missing transfer transaction. 251 | 252 | I typically use sorted names as the subaccount (Python code sample): 253 | 254 | #+BEGIN_SRC python 255 | "Assets:Transfers:" + "".join(sorted(["Bank2", "Bank1"])) 256 | #+END_SRC 257 | 258 | *** External references 259 | 260 | This approach is based on what is described in Full-fledged hledger: 261 | [[https://github.com/adept/full-fledged-hledger/wiki/Adding-more-accounts#lets-make-sure-that-transfers-are-not-double-counted]] 262 | 263 | The question was first asked in [[https://github.com/apauley/hledger-flow/issues/51][issue #51]]. 264 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # Changelog for [hledger-flow](https://github.com/apauley/hledger-flow) 2 | 3 | ## 0.16.1 4 | 5 | - Fix documentation URL: https://github.com/apauley/hledger-flow/tree/master/docs#feature-reference 6 | 7 | ## 0.16.0 8 | 9 | - Switched back to `hledger print` during statement import [#126](https://github.com/apauley/hledger-flow/issues/126) 10 | - Move feature reference to https://github.com/apauley/hledger-flow/docs 11 | - Fix preprocessing logic for CSV files [#123](https://github.com/apauley/hledger-flow/issues/123) 12 | - Add --ascii-reports flag [#115](https://github.com/apauley/hledger-flow/pull/115) 13 | 14 | Other changes: 15 | - Switched to Stackage lts-24.8 (GHC 9.10.2) 16 | 17 | ## 0.15.0 18 | 19 | Made some changes that will result in formatting changes of generated files: 20 | 21 | - Removed the obsolete exclamation mark from the `include` directive 22 | - Switched from `hledger print` to `hledger import` during statement import. 23 | 24 | `hledger import` uses your preferred commodity styles from your 25 | [directives.journal](https://github.com/apauley/hledger-flow#hledger-directives) to generate journals. 26 | 27 | Other changes: 28 | 29 | - Switched to GHC 9.0.1 30 | 31 | ## 0.14.4 32 | 33 | Add an option to process files in batches. 34 | 35 | If the number of input files processed by `hledger-flow` grows large, and you have resource-intensive processing scripts, your system resources can be overwhelmed. 36 | 37 | With this change the input files will be processed in batches, by default 200 at a time. The batch size can be set from the command-line. 38 | 39 | https://github.com/apauley/hledger-flow/issues/93 40 | 41 | https://github.com/apauley/hledger-flow/pull/94 42 | 43 | ## 0.14.3 44 | 45 | Ensure that generated include files only contain files ending with .journal 46 | 47 | Fixes [#92](https://github.com/apauley/hledger-flow/issues/92) 48 | 49 | ## 0.14.2 50 | 51 | Add an optional `--start-year` command-line option for imports: 52 | 53 | Import only from the specified year and onwards, 54 | ignoring previous years. Valid values include a 4-digit 55 | year or 'current' for the current year. 56 | 57 | An implementation for [this feature request](https://github.com/apauley/hledger-flow/issues/81) 58 | 59 | ## 0.14.1 60 | 61 | - Make `--enable-future-rundir` the default, and deprecate the command-line option. To be removed in a future release. 62 | - Ensure that the deepest rundir is the account directory, because the program doesn't generate include files correctly in directories below the account level. 63 | 64 | ## 0.14.0 65 | 66 | - Add a new performance-related command-line option to import: `--new-files-only`. [PR #89](https://github.com/apauley/hledger-flow/pull/89) 67 | 68 | Don't regenerate transaction files if they are 69 | already present. This applies to hledger journal 70 | files as well as files produced by the preprocess and 71 | construct scripts. 72 | 73 | - Generate monthly versions of the income statement in reports. A contribution by [Max Linke](https://github.com/apauley/hledger-flow/pull/88) 74 | 75 | - Switch some usages of system-filepath over to [path](https://github.com/apauley/hledger-flow/pull/87) 76 | 77 | hledger-flow started as a collection of bash scripts that I translated into Haskell with the help of [Turtle](https://hackage.haskell.org/package/turtle). 78 | 79 | Turtle uses the now deprecated [system-filepath](https://hackage.haskell.org/package/system-filepath) to represent all paths. 80 | 81 | I've had many filepath-related issues in hledger-flow. 82 | They were related to issues such as that 2 instances of the same directory would not be treated as equal, because one could have a trailing slash and the other not. 83 | Another issue that popped up was knowing wether a path is a file or a directory, and if it is absolute or relative. 84 | 85 | All of these issues are articulated in the `path` library: 86 | https://github.com/commercialhaskell/path 87 | 88 | 89 | ## 0.13.2 90 | 91 | Improve support for importing a subset of journals: start importing only from the directory given as argument, 92 | or the current directory, and generate only the relevant include files. 93 | 94 | This is a behavioural change and (for now) it needs to be enabled with the --enable-future-rundir switch. 95 | This will become the default behaviour in 0.14.x, at which time the switch will be removed. 96 | 97 | Reports: 98 | Use the LEDGER_FILE env var (if set) when generating reports. 99 | Default to the top-level all-years.journal if not set. 100 | 101 | Build with Stackage Nightly 2020-03-10 (ghc-8.8.3) 102 | 103 | ## 0.13.1 104 | 105 | - Automatically add [include lines for yearly price files](https://github.com/apauley/hledger-flow/#price-files) if they are present on disk. 106 | - Minor report changes - do not assume too many extra options for default reports. 107 | 108 | ## 0.13.0 109 | 110 | - Add an experimental rundir option for imports 111 | 112 | The experimental rundir is an attempt to restrict hledger-flow into processing just a subset of files, primarily to quickly get feedback/failures while adding new accounts to an existing set of accounts. 113 | 114 | The use case has been described in [issue 64](https://github.com/apauley/hledger-flow/issues/64). 115 | 116 | It is experimental, because the only problem it currently solves is getting hledger-flow to fail fast. 117 | One of the current side effects of doing so is that the generated include files are then written to only 118 | include the subset of files that were processed. 119 | 120 | But as soon as you do a full run again, the include files will be correctly re-generated as before. 121 | 122 | ## 0.12.4.0 123 | 124 | - Update usage of hledger to reflect updated command-line flags of hledger version 1.15 125 | https://github.com/apauley/hledger-flow/issues/73 126 | - Compile with stackage lts-14.9 127 | 128 | ## 0.12.3.1 129 | 130 | Fixed a bug where: 131 | 132 | Given: 133 | - A run of `hledger-flow import` 134 | 135 | When: 136 | - specifying a relative import base directory 137 | - but specifically without any relative prefixes such as `./` or `../` 138 | 139 | Then: 140 | - the account-level include files pointing to the real journal entries would have incorrect paths 141 | 142 | https://github.com/apauley/hledger-flow/issues/65 143 | 144 | ## 0.12.3 145 | 146 | Add more reports: 147 | 148 | - Balance Sheet per owner per year, and for all owners per year 149 | - Unknown transactions per owner per year 150 | - A transfer balance overview per year 151 | 152 | ## 0.12.2.1 153 | 154 | Fix resolver extraction and hledger-flow --version in release-tarball script 155 | 156 | ## 0.12.2 157 | 158 | Slightly smarter reporting. 159 | 160 | - Get the available report years for each individual owner. Only generate reports for those years. 161 | - Create uniform output directories. 162 | - Add system info to version output 163 | 164 | ## 0.12.1 165 | 166 | Generate some reports per owner. 167 | 168 | Report generation is still a work-in-progress. 169 | 170 | https://github.com/apauley/hledger-flow/pull/57 171 | 172 | ## 0.12.0 173 | 174 | - Re-organised the command-line interface: 175 | moved various command-line options out of subcommands, into the top-level. 176 | - Added a [contributor's agreement](https://github.com/apauley/hledger-flow/blob/master/CONTRIBUTING.org) 177 | after receiving some more valued contributions from 178 | [jecaro](https://github.com/apauley/hledger-flow/pull/42) 179 | 180 | ## 0.11.3 181 | 182 | - Detect the hledger-flow base directory correctly, even when in a subdirectory. Similar to how git behaves. 183 | - Change the version subcommand into a flag - thanks to [jecaro](https://github.com/apauley/hledger-flow/pull/38) for the contribution. 184 | 185 | ## 0.11.2 186 | 187 | - Improved display of external process output 188 | 189 | ## 0.11.1.2 190 | 191 | - Exit with an error code when any external script fails - https://github.com/apauley/hledger-flow/issues/28 192 | - Capture external process output when doing parallel processing, in order to better prevent mangled concurrent output. 193 | - Allow users to specify a path to an hledger executable 194 | - Display a user-friendly error message if hledger cannot be found - https://github.com/apauley/hledger-flow/issues/22 195 | 196 | ## 0.11.1.1 197 | 198 | - Support input files from the year 2011 - https://github.com/apauley/hledger-flow/issues/27 199 | Use a more specific input-file pattern, so as not to match 2011-include.journal 200 | - Print command-line options if requested - https://github.com/apauley/hledger-flow/issues/11 201 | - Use the channel output functions consistently to avoid concurrency issues. 202 | 203 | ## 0.11.1 204 | 205 | - Create statically linked executables on Linux - https://github.com/apauley/hledger-flow/releases 206 | - Add an option to disable parallel processing 207 | - Log the exit status of shell commands. 208 | - Upgrade to LTS 13.16 for GHC 8.6.4. 209 | 210 | ## 0.11 211 | 212 | - Change the name from `hledger-makeitso` to `hledger-flow`. 213 | 214 | ## 0.10 215 | 216 | - Add a `version` subcommand. 217 | Create [issue #15](https://github.com/apauley/hledger-flow/issues/15) 218 | to change it into a `--version` flag later. 219 | - Fix a minor issue where yearly include files were generated at the top-level 220 | of the directtory structure, even though the same content was available in the 221 | `import` directory. 222 | The top-level `all-years.journal` now just includes the years within the 223 | `import` directory. 224 | - Upgrade to LTS 13.15 for GHC 8.6.4 225 | - Add CircleCI and TravisCI build instructions. Switch the README.org to a 226 | README.md in order to better support CI status badges. 227 | 228 | 229 | ## 0.9.0.1 230 | 231 | First hackage release. Minor changes to fix `stack sdist` warnings and errors, in preperation of 232 | the hackage upload. 233 | 234 | 1bf817c "Merge pull request #9 from apauley/hackage-upload" Mar 31 21:51:38 2019 +0200 235 | 236 | ## 0.9 237 | 238 | Process all statements in parallel. 239 | 240 | This has a significant speed improvement on multi-processor machines when dealing with lots of input files. 241 | a906bb5 "Merge pull request #8 from apauley/parallel-import" 2019-03-27 22:45:23 +0200 242 | 243 | ## 0.8 244 | 245 | Generate an all-years.journal on each level which includes all the available years for that level. Replace the old makeitso.journal with the top-level version of this: 246 | 247 | 06f2127 "Merge pull request #5 from apauley/all-years-includes" 2019-03-22 00:09:27 +0200 248 | 249 | ## 0.7 250 | 251 | Change the way include files are aggregated. 252 | 253 | It used to be by owner/bank/account, now each of those levels (owner/bank/account) are aggregated per year: 254 | eb17fed "Merge pull request #3 from apauley/annual-includes" 2019-03-12 23:09:17 +0200 255 | 256 | ## 0.6 257 | 258 | 61c71d6 "Upgrade to lts-13.9 (GHC 8.6.3)" 2019-03-01 11:31:23 +0200 259 | 260 | ## 0.5 261 | 262 | 3a7a39e "Upgrade to lts-13.6 (GHC 8.6.3)" 2019-02-16 09:54:49 +0200 263 | 264 | ## 0.4 265 | 266 | 213552d "Upgrade to lts-12.16 (GHC 8.4.4)" 2018-11-03 20:00:21 +0200 267 | 268 | ## 0.3 269 | 270 | 5e2d45f "Update from lts-12.1 to lts-12.11" 2018-10-01 23:07:21 +0200 271 | 272 | ## 0.2 273 | 274 | First support for the construct script, when it was confusingly named an import script: 275 | 276 | 24ac4c7 "Support a fully custom import script" 2018-09-16 16:11:53 +0200 277 | 278 | ## 0.1 279 | 280 | The first semi-useful version, replacing a previous bash script: 281 | 282 | 131f8af "Write journal" 2018-07-23 16:08:44 +0200 283 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Import/CSVImport.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | 3 | module Hledger.Flow.Import.CSVImport 4 | ( importCSVs, 5 | ) 6 | where 7 | 8 | import Control.Concurrent.STM 9 | import Control.Monad 10 | import qualified Data.List.NonEmpty as NonEmpty 11 | import Data.Maybe (fromMaybe, isNothing) 12 | import qualified Data.Text as T 13 | import qualified Data.Text.IO as T 14 | import Hledger.Flow.BaseDir (effectiveRunDir, relativeToBase) 15 | import Hledger.Flow.Common 16 | import Hledger.Flow.DocHelpers (docURL) 17 | import Hledger.Flow.Import.ImportHelpers 18 | import Hledger.Flow.Import.ImportHelpersTurtle (extractImportDirs, writeIncludesUpTo, writeToplevelAllYearsInclude) 19 | import Hledger.Flow.Import.Types 20 | import Hledger.Flow.Logging 21 | import Hledger.Flow.PathHelpers (TurtlePath, pathToTurtle) 22 | import Hledger.Flow.RuntimeOptions 23 | import qualified Hledger.Flow.Types as FlowTypes 24 | import Turtle ((%), (<.>), ()) 25 | import qualified Turtle hiding (proc, procStrictWithErr, stderr, stdout) 26 | import Prelude hiding (putStrLn, take, writeFile) 27 | 28 | type FileWasGenerated = Bool 29 | 30 | importCSVs :: RuntimeOptions -> IO () 31 | importCSVs opts = 32 | Turtle.sh 33 | ( do 34 | ch <- Turtle.liftIO newTChanIO 35 | logHandle <- Turtle.fork $ consoleChannelLoop ch 36 | Turtle.liftIO $ when (showOptions opts) (channelOutLn ch (Turtle.repr opts)) 37 | Turtle.liftIO $ logVerbose opts ch "Starting import" 38 | (journals, diff) <- Turtle.time $ Turtle.liftIO $ importCSVs' opts ch 39 | let generatedJournals = filter snd journals 40 | Turtle.liftIO $ channelOutLn ch $ Turtle.format ("Imported " % Turtle.d % "/" % Turtle.d % " journals in " % Turtle.s) (length generatedJournals) (length journals) $ Turtle.repr diff 41 | Turtle.liftIO $ terminateChannelLoop ch 42 | Turtle.wait logHandle 43 | ) 44 | 45 | importCSVs' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [(TurtlePath, FileWasGenerated)] 46 | importCSVs' opts ch = do 47 | let effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts) 48 | let startYearMsg = maybe " " (Turtle.format (" (for the year " % Turtle.d % " and onwards) ")) (importStartYear opts) 49 | channelOutLn ch $ Turtle.format ("Collecting input files" % Turtle.s % "from " % Turtle.fp) startYearMsg (pathToTurtle effectiveDir) 50 | (inputFiles, diff) <- Turtle.time $ findInputFiles (fromMaybe 0 $ importStartYear opts) effectiveDir 51 | 52 | let fileCount = length inputFiles 53 | if fileCount == 0 && isNothing (importStartYear opts) 54 | then do 55 | let msg = 56 | Turtle.format 57 | ( "I couldn't find any input files underneath " 58 | % Turtle.fp 59 | % "\n\nhledger-flow expects to find its input files in specifically\nnamed directories.\n\n" 60 | % "Have a look at the documentation for a detailed explanation:\n" 61 | % Turtle.s 62 | ) 63 | (pathToTurtle effectiveDir) 64 | (docURL "input-files") 65 | errExit 1 ch msg [] 66 | else do 67 | channelOutLn ch $ Turtle.format ("Found " % Turtle.d % " input files" % Turtle.s % "in " % Turtle.s % ". Proceeding with import...") fileCount startYearMsg (Turtle.repr diff) 68 | let actions = map (extractAndImport opts ch . pathToTurtle) inputFiles :: [IO (TurtlePath, FileWasGenerated)] 69 | importedJournals <- parAwareActions opts actions 70 | (journalsOnDisk, journalFindTime) <- Turtle.time $ findJournalFiles effectiveDir 71 | (_, writeIncludeTime1) <- Turtle.time $ writeIncludesUpTo opts ch (pathToTurtle effectiveDir) $ fmap pathToTurtle journalsOnDisk 72 | (_, writeIncludeTime2) <- Turtle.time $ writeToplevelAllYearsInclude opts 73 | let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2 74 | channelOutLn ch $ Turtle.format ("Wrote include files for " % Turtle.d % " journals in " % Turtle.s) (length journalsOnDisk) (Turtle.repr includeGenTime) 75 | return importedJournals 76 | 77 | extractAndImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> IO (TurtlePath, FileWasGenerated) 78 | extractAndImport opts ch inputFile = do 79 | case extractImportDirs inputFile of 80 | Right importDirs -> importCSV opts ch importDirs inputFile 81 | Left errorMessage -> do 82 | errExit 1 ch errorMessage (inputFile, False) 83 | 84 | importCSV :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs -> TurtlePath -> IO (TurtlePath, FileWasGenerated) 85 | importCSV opts ch importDirs srcFile = do 86 | let preprocessScript = accountDir importDirs "preprocess" 87 | let constructScript = accountDir importDirs "construct" 88 | let bankName = importDirLine bankDir importDirs 89 | let accountName = importDirLine accountDir importDirs 90 | let ownerName = importDirLine ownerDir importDirs 91 | (csvFile, preprocessHappened) <- preprocessIfNeeded opts ch preprocessScript bankName accountName ownerName srcFile 92 | let journalOut = changePathAndExtension "3-journal/" "journal" csvFile 93 | shouldImport <- 94 | if onlyNewFiles opts && not preprocessHappened 95 | then not <$> verboseTestFile opts ch journalOut 96 | else return True 97 | 98 | importFun <- 99 | if shouldImport 100 | then constructOrImport opts ch constructScript bankName accountName ownerName 101 | else do 102 | _ <- logNewFileSkip opts ch "import" journalOut 103 | return $ \_p1 _p2 -> return journalOut 104 | Turtle.mktree $ Turtle.directory journalOut 105 | out <- importFun csvFile journalOut 106 | return (out, shouldImport) 107 | 108 | constructOrImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> Turtle.Line -> Turtle.Line -> Turtle.Line -> IO (TurtlePath -> TurtlePath -> IO TurtlePath) 109 | constructOrImport opts ch constructScript bankName accountName ownerName = do 110 | constructScriptExists <- verboseTestFile opts ch constructScript 111 | if constructScriptExists 112 | then return $ customConstruct opts ch constructScript bankName accountName ownerName 113 | else return $ hledgerImport opts ch 114 | 115 | preprocessIfNeeded :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> Turtle.Line -> Turtle.Line -> Turtle.Line -> TurtlePath -> IO (TurtlePath, Bool) 116 | preprocessIfNeeded opts ch script bank account owner src = do 117 | let csvOut = changePathAndExtension "2-preprocessed/" "csv" src 118 | scriptExists <- verboseTestFile opts ch script 119 | targetExists <- verboseTestFile opts ch csvOut 120 | shouldProceed <- 121 | if onlyNewFiles opts 122 | then return $ scriptExists && not targetExists 123 | else return scriptExists 124 | if shouldProceed 125 | then do 126 | out <- preprocess opts ch script bank account owner src csvOut 127 | return (out, True) 128 | else do 129 | _ <- logNewFileSkip opts ch "preprocess" csvOut 130 | if targetExists 131 | then return (csvOut, False) 132 | else return (src, False) 133 | 134 | logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> TurtlePath -> IO () 135 | logNewFileSkip opts ch logIdentifier absTarget = 136 | Control.Monad.when (onlyNewFiles opts) $ do 137 | let relativeTarget = relativeToBase opts absTarget 138 | logVerbose opts ch $ 139 | Turtle.format 140 | ( "Skipping " 141 | % Turtle.s 142 | % " - only creating new files and this output file already exists: '" 143 | % Turtle.fp 144 | % "'" 145 | ) 146 | logIdentifier 147 | relativeTarget 148 | 149 | preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> Turtle.Line -> Turtle.Line -> Turtle.Line -> TurtlePath -> TurtlePath -> IO TurtlePath 150 | preprocess opts ch script bank account owner src csvOut = do 151 | Turtle.mktree $ Turtle.directory csvOut 152 | let args = [Turtle.format Turtle.fp src, Turtle.format Turtle.fp csvOut, Turtle.lineToText bank, Turtle.lineToText account, Turtle.lineToText owner] 153 | let relScript = relativeToBase opts script 154 | let relSrc = relativeToBase opts src 155 | let cmdLabel = Turtle.format ("executing '" % Turtle.fp % "' on '" % Turtle.fp % "'") relScript relSrc 156 | _ <- timeAndExitOnErr opts ch cmdLabel channelOut channelErr (parAwareProc opts) (Turtle.format Turtle.fp script, args, Turtle.empty) 157 | return csvOut 158 | 159 | hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> TurtlePath -> IO TurtlePath 160 | hledgerImport opts ch csvSrc journalOut = do 161 | case extractImportDirs csvSrc of 162 | Right importDirs -> hledgerImport' opts ch importDirs csvSrc journalOut 163 | Left errorMessage -> do 164 | errExit 1 ch errorMessage csvSrc 165 | 166 | hledgerImport' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs -> TurtlePath -> TurtlePath -> IO TurtlePath 167 | hledgerImport' opts ch importDirs csvSrc journalOut = do 168 | let candidates = rulesFileCandidates csvSrc importDirs 169 | maybeRulesFile <- firstExistingFile candidates 170 | let relCSV = relativeToBase opts csvSrc 171 | case maybeRulesFile of 172 | Just rf -> do 173 | let relRules = relativeToBase opts rf 174 | let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text 175 | directivesExist <- Turtle.testfile $ directivesFile opts 176 | let directivesArgs = if directivesExist then ["--file", Turtle.format Turtle.fp (directivesFile opts)] else [] 177 | let csvArgs = ["--file", Turtle.format Turtle.fp csvSrc, "--rules-file", Turtle.format Turtle.fp rf] 178 | let args = ["print", "--explicit"] ++ directivesArgs ++ csvArgs 179 | 180 | let cmdLabel = Turtle.format ("importing '" % Turtle.fp % "' using rules file '" % Turtle.fp % "'") relCSV relRules 181 | ((_, stdOut, _), _) <- timeAndExitOnErr opts ch cmdLabel dummyLogger channelErr (parAwareProc opts) (hledger, args, Turtle.empty) 182 | _ <- T.writeFile journalOut stdOut 183 | return journalOut 184 | Nothing -> 185 | do 186 | let relativeCandidates = map (relativeToBase opts) candidates 187 | let candidatesTxt = T.intercalate "\n" $ map (Turtle.format Turtle.fp) relativeCandidates 188 | let msg = 189 | Turtle.format 190 | ( "I couldn't find an hledger rules file while trying to import\n" 191 | % Turtle.fp 192 | % "\n\nI will happily use the first rules file I can find from any one of these " 193 | % Turtle.d 194 | % " files:\n" 195 | % Turtle.s 196 | % "\n\nHere is a bit of documentation about rules files that you may find helpful:\n" 197 | % Turtle.s 198 | ) 199 | relCSV 200 | (length candidates) 201 | candidatesTxt 202 | (docURL "rules-files") 203 | errExit 1 ch msg csvSrc 204 | 205 | rulesFileCandidates :: TurtlePath -> ImportDirs -> [TurtlePath] 206 | rulesFileCandidates csvSrc importDirs = statementSpecificRulesFiles csvSrc importDirs ++ generalRulesFiles importDirs 207 | 208 | importDirLines :: (ImportDirs -> TurtlePath) -> ImportDirs -> [Turtle.Line] 209 | importDirLines dirFun importDirs = NonEmpty.toList $ Turtle.textToLines $ Turtle.format Turtle.fp $ Turtle.dirname $ dirFun importDirs 210 | 211 | importDirLine :: (ImportDirs -> TurtlePath) -> ImportDirs -> Turtle.Line 212 | importDirLine dirFun importDirs = foldl (<>) "" $ importDirLines dirFun importDirs 213 | 214 | generalRulesFiles :: ImportDirs -> [TurtlePath] 215 | generalRulesFiles importDirs = do 216 | let bank = importDirLines bankDir importDirs 217 | let account = importDirLines accountDir importDirs 218 | let accountRulesFile = accountDir importDirs buildFilename (bank ++ account) "rules" 219 | 220 | let bankRulesFile = importDir importDirs buildFilename bank "rules" 221 | [accountRulesFile, bankRulesFile] 222 | 223 | statementSpecificRulesFiles :: TurtlePath -> ImportDirs -> [TurtlePath] 224 | statementSpecificRulesFiles csvSrc importDirs = do 225 | let srcSuffix = snd $ T.breakOnEnd "_" (Turtle.format Turtle.fp (Turtle.basename csvSrc)) 226 | 227 | if ((T.take 3 srcSuffix) == "rfo") 228 | then do 229 | let srcSpecificFilename = T.unpack srcSuffix <.> "rules" 230 | map ( srcSpecificFilename) [accountDir importDirs, bankDir importDirs, importDir importDirs] 231 | else [] 232 | 233 | customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> TurtlePath -> Turtle.Line -> Turtle.Line -> Turtle.Line -> TurtlePath -> TurtlePath -> IO TurtlePath 234 | customConstruct opts ch constructScript bank account owner csvSrc journalOut = do 235 | let script = Turtle.format Turtle.fp constructScript :: T.Text 236 | let relScript = relativeToBase opts constructScript 237 | let constructArgs = [Turtle.format Turtle.fp csvSrc, "-", Turtle.lineToText bank, Turtle.lineToText account, Turtle.lineToText owner] 238 | let constructCmdText = Turtle.format ("Running: " % Turtle.fp % " " % Turtle.s) relScript (showCmdArgs constructArgs) 239 | let stdLines = inprocWithErrFun (channelErrLn ch) (script, constructArgs, Turtle.empty) 240 | let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text 241 | let args = 242 | [ "print", 243 | "--ignore-assertions", 244 | "--file", 245 | "-", 246 | "--output-file", 247 | Turtle.format Turtle.fp journalOut 248 | ] 249 | 250 | let relSrc = relativeToBase opts csvSrc 251 | let cmdLabel = Turtle.format ("executing '" % Turtle.fp % "' on '" % Turtle.fp % "'") relScript relSrc 252 | _ <- timeAndExitOnErr' opts ch cmdLabel [constructCmdText] channelOut channelErr (parAwareProc opts) (hledger, args, stdLines) 253 | return journalOut 254 | -------------------------------------------------------------------------------- /src/Hledger/Flow/Common.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedStrings #-} 2 | {-# LANGUAGE QuasiQuotes #-} 3 | 4 | module Hledger.Flow.Common where 5 | 6 | import Control.Concurrent.STM 7 | import qualified Control.Foldl as Fold 8 | import Data.Char (isDigit) 9 | import Data.Either 10 | import Data.Function (on) 11 | import qualified Data.List as List (groupBy, null, sortBy) 12 | import qualified Data.Map.Strict as Map 13 | import Data.Ord (comparing) 14 | import qualified Data.Text as T 15 | import qualified Data.Text.IO as T 16 | import qualified Data.Text.Read as T 17 | import qualified GHC.IO.Handle.FD as H 18 | import Hledger.Flow.BaseDir (relativeToBase, turtleBaseDir) 19 | import Hledger.Flow.Logging 20 | import Hledger.Flow.PathHelpers (AbsFile, TurtlePath, fromTurtleAbsFile, pathToTurtle) 21 | import Hledger.Flow.Types 22 | import Path (absfile, relfile) 23 | import qualified Path.IO as Path 24 | import Turtle ((%), (<.>), ()) 25 | import qualified Turtle 26 | import Prelude hiding (putStrLn, readFile, writeFile) 27 | 28 | hledgerPathFromOption :: Maybe TurtlePath -> IO AbsFile 29 | hledgerPathFromOption pathOption = do 30 | case pathOption of 31 | Just h -> do 32 | hlAbs <- fromTurtleAbsFile h 33 | isOnDisk <- Path.doesFileExist hlAbs 34 | if isOnDisk 35 | then return hlAbs 36 | else do 37 | let msg = Turtle.format ("Unable to find hledger at " % Turtle.fp) h 38 | errExit' 1 (T.hPutStrLn H.stderr) msg hlAbs 39 | Nothing -> do 40 | maybeH <- Path.findExecutable [relfile|hledger|] 41 | case maybeH of 42 | Just h -> return h 43 | Nothing -> do 44 | let msg = 45 | "Unable to find hledger in your path.\n" 46 | <> "You need to either install hledger, or add it to your PATH, or provide the path to an hledger executable.\n\n" 47 | <> "There are a number of installation options on the hledger website: https://hledger.org/download.html" 48 | errExit' 1 (T.hPutStrLn H.stderr) msg [absfile|/hledger|] 49 | 50 | hledgerVersionFromPath :: TurtlePath -> IO T.Text 51 | hledgerVersionFromPath hlp = fmap (T.strip . Turtle.linesToText) (Turtle.single $ shellToList $ Turtle.inproc (Turtle.format Turtle.fp hlp) ["--version"] Turtle.empty) 52 | 53 | hledgerInfoFromPath :: Maybe TurtlePath -> IO HledgerInfo 54 | hledgerInfoFromPath pathOption = do 55 | hlp <- hledgerPathFromOption pathOption 56 | hlv <- hledgerVersionFromPath $ pathToTurtle hlp 57 | return $ HledgerInfo hlp hlv 58 | 59 | showCmdArgs :: [T.Text] -> T.Text 60 | showCmdArgs args = T.intercalate " " (map escapeArg args) 61 | 62 | escapeArg :: T.Text -> T.Text 63 | escapeArg a = if T.count " " a > 0 then "'" <> a <> "'" else a 64 | 65 | errExit :: Int -> TChan LogMessage -> T.Text -> a -> IO a 66 | errExit exitStatus ch = errExit' exitStatus (channelErrLn ch) 67 | 68 | errExit' :: Int -> (T.Text -> IO ()) -> T.Text -> a -> IO a 69 | errExit' exitStatus logFun errorMessage dummyReturnValue = do 70 | logFun errorMessage 71 | Turtle.sleep 0.1 72 | _ <- Turtle.exit $ Turtle.ExitFailure exitStatus 73 | return dummyReturnValue 74 | 75 | descriptiveOutput :: T.Text -> T.Text -> T.Text 76 | descriptiveOutput outputLabel outTxt = do 77 | if not (T.null outTxt) 78 | then Turtle.format (Turtle.s % ":\n" % Turtle.s % "\n") outputLabel outTxt 79 | else "" 80 | 81 | logTimedAction :: 82 | (HasVerbosity o) => 83 | o -> 84 | TChan LogMessage -> 85 | T.Text -> 86 | [T.Text] -> 87 | T.Text -> 88 | [T.Text] -> 89 | (TChan LogMessage -> T.Text -> IO ()) -> 90 | (TChan LogMessage -> T.Text -> IO ()) -> 91 | IO FullOutput -> 92 | IO FullTimedOutput 93 | logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger stderrLogger action = do 94 | logVerbose opts ch $ Turtle.format ("Begin: " % Turtle.s) cmdLabel 95 | logVerbose opts ch $ Turtle.format (Turtle.s % " " % Turtle.s) cmd (T.intercalate " " args) 96 | if (List.null extraCmdLabels) then return () else logVerbose opts ch $ T.intercalate "\n" extraCmdLabels 97 | timed@((ec, stdOut, stdErr), diff) <- Turtle.time action 98 | stdoutLogger ch stdOut 99 | stderrLogger ch stdErr 100 | logVerbose opts ch $ Turtle.format ("End: " % Turtle.s % " " % Turtle.s % " (" % Turtle.s % ")") cmdLabel (Turtle.repr ec) (Turtle.repr diff) 101 | return timed 102 | 103 | timeAndExitOnErr :: 104 | (HasSequential o, HasVerbosity o) => 105 | o -> 106 | TChan LogMessage -> 107 | T.Text -> 108 | (TChan LogMessage -> T.Text -> IO ()) -> 109 | (TChan LogMessage -> T.Text -> IO ()) -> 110 | ProcFun -> 111 | ProcInput -> 112 | IO FullTimedOutput 113 | timeAndExitOnErr opts ch cmdLabel = timeAndExitOnErr' opts ch cmdLabel [] 114 | 115 | timeAndExitOnErr' :: 116 | (HasSequential o, HasVerbosity o) => 117 | o -> 118 | TChan LogMessage -> 119 | T.Text -> 120 | [T.Text] -> 121 | (TChan LogMessage -> T.Text -> IO ()) -> 122 | (TChan LogMessage -> T.Text -> IO ()) -> 123 | ProcFun -> 124 | ProcInput -> 125 | IO FullTimedOutput 126 | timeAndExitOnErr' opts ch cmdLabel extraCmdLabels stdoutLogger stderrLogger procFun (cmd, args, stdInput) = do 127 | let action = procFun cmd args stdInput 128 | timed@((ec, stdOut, stdErr), _) <- logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger stderrLogger action 129 | case ec of 130 | Turtle.ExitFailure i -> do 131 | let cmdText = Turtle.format (Turtle.s % " " % Turtle.s) cmd $ showCmdArgs args 132 | let msgOut = descriptiveOutput "Standard output" stdOut 133 | let msgErr = descriptiveOutput "Error output" stdErr 134 | 135 | let exitMsg = 136 | Turtle.format 137 | ( "\n=== Begin Error: " 138 | % Turtle.s 139 | % " ===\nExternal command:\n" 140 | % Turtle.s 141 | % "\nExit code " 142 | % Turtle.d 143 | % "\n" 144 | % Turtle.s 145 | % Turtle.s 146 | % "=== End Error: " 147 | % Turtle.s 148 | % " ===\n" 149 | ) 150 | cmdLabel 151 | cmdText 152 | i 153 | msgOut 154 | msgErr 155 | cmdLabel 156 | errExit i ch exitMsg timed 157 | Turtle.ExitSuccess -> return timed 158 | 159 | procWithEmptyOutput :: ProcFun 160 | procWithEmptyOutput cmd args stdinput = do 161 | ec <- Turtle.proc cmd args stdinput 162 | return (ec, T.empty, T.empty) 163 | 164 | parAwareProc :: (HasSequential o) => o -> ProcFun 165 | parAwareProc opts = if (sequential opts) then procWithEmptyOutput else Turtle.procStrictWithErr 166 | 167 | parAwareActions :: (HasSequential o, HasBatchSize o) => o -> [IO a] -> IO [a] 168 | parAwareActions opts = if (sequential opts) then sequence else parBatchedActions (batchSize opts) [] 169 | 170 | parBatchedActions :: Int -> [a] -> [IO a] -> IO [a] 171 | parBatchedActions _ done [] = return done 172 | parBatchedActions batch done todo = do 173 | let doNow = take batch todo 174 | let remaining = drop batch todo 175 | doneNow <- (Turtle.single . shellToList . Turtle.parallel) doNow 176 | parBatchedActions batch (done ++ doneNow) remaining 177 | 178 | inprocWithErrFun :: (T.Text -> IO ()) -> ProcInput -> Turtle.Shell Turtle.Line 179 | inprocWithErrFun errFun (cmd, args, standardInput) = do 180 | result <- Turtle.inprocWithErr cmd args standardInput 181 | case result of 182 | Right ln -> return ln 183 | Left ln -> do 184 | (Turtle.liftIO . errFun . Turtle.lineToText) ln 185 | Turtle.empty 186 | 187 | verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> TurtlePath -> IO Bool 188 | verboseTestFile opts ch p = do 189 | fileExists <- Turtle.testfile p 190 | let rel = relativeToBase opts p 191 | if fileExists 192 | then logVerbose opts ch $ Turtle.format ("Found '" % Turtle.fp % "'") rel 193 | else logVerbose opts ch $ Turtle.format ("Looked for but did not find '" % Turtle.fp % "'") rel 194 | return fileExists 195 | 196 | groupPairs' :: (Eq a, Ord a) => [(a, b)] -> [(a, [b])] 197 | groupPairs' = 198 | map (\ll -> (fst . head $ ll, map snd ll)) 199 | . List.groupBy ((==) `on` fst) 200 | . List.sortBy (comparing fst) 201 | 202 | groupPairs :: (Eq a, Ord a) => [(a, b)] -> Map.Map a [b] 203 | groupPairs = Map.fromList . groupPairs' 204 | 205 | pairBy :: (a -> b) -> [a] -> [(b, a)] 206 | pairBy keyFun = map (\v -> (keyFun v, v)) 207 | 208 | groupValuesBy :: (Ord k, Ord v) => (v -> k) -> [v] -> Map.Map k [v] 209 | groupValuesBy keyFun = groupPairs . pairBy keyFun 210 | 211 | allYearsFileName :: TurtlePath 212 | allYearsFileName = "all-years" <.> "journal" 213 | 214 | directivesFile :: (HasBaseDir o) => o -> TurtlePath 215 | directivesFile opts = turtleBaseDir opts "directives" <.> "journal" 216 | 217 | lsDirs :: TurtlePath -> Turtle.Shell TurtlePath 218 | lsDirs = onlyDirs . Turtle.ls 219 | 220 | onlyDirs :: Turtle.Shell TurtlePath -> Turtle.Shell TurtlePath 221 | onlyDirs = excludeHiddenFiles . excludeWeirdPaths . filterPathsByFileStatus Turtle.isDirectory 222 | 223 | onlyFiles :: Turtle.Shell TurtlePath -> Turtle.Shell TurtlePath 224 | onlyFiles = excludeHiddenFiles . filterPathsByFileStatus Turtle.isRegularFile 225 | 226 | filterPathsByFileStatus :: (Turtle.FileStatus -> Bool) -> Turtle.Shell TurtlePath -> Turtle.Shell TurtlePath 227 | filterPathsByFileStatus filepred files = do 228 | files' <- shellToList files 229 | filtered <- filterPathsByFileStatus' filepred [] files' 230 | Turtle.select filtered 231 | 232 | filterPathsByFileStatus' :: (Turtle.FileStatus -> Bool) -> [TurtlePath] -> [TurtlePath] -> Turtle.Shell [TurtlePath] 233 | filterPathsByFileStatus' _ acc [] = return acc 234 | filterPathsByFileStatus' filepred acc (file : files) = do 235 | filestat <- Turtle.stat file 236 | let filtered = if (filepred filestat) then file : acc else acc 237 | filterPathsByFileStatus' filepred filtered files 238 | 239 | filterPaths :: (TurtlePath -> IO Bool) -> [TurtlePath] -> Turtle.Shell [TurtlePath] 240 | filterPaths = filterPaths' [] 241 | 242 | filterPaths' :: [TurtlePath] -> (TurtlePath -> IO Bool) -> [TurtlePath] -> Turtle.Shell [TurtlePath] 243 | filterPaths' acc _ [] = return acc 244 | filterPaths' acc filepred (file : files) = do 245 | shouldInclude <- Turtle.liftIO $ filepred file 246 | let filtered = if shouldInclude then file : acc else acc 247 | filterPaths' filtered filepred files 248 | 249 | excludeHiddenFiles :: Turtle.Shell TurtlePath -> Turtle.Shell TurtlePath 250 | excludeHiddenFiles paths = do 251 | p <- paths 252 | case (Turtle.match (Turtle.prefix ".") $ Turtle.format Turtle.fp $ Turtle.filename p) of 253 | [] -> Turtle.select [p] 254 | _ -> Turtle.select [] 255 | 256 | excludeWeirdPaths :: Turtle.Shell TurtlePath -> Turtle.Shell TurtlePath 257 | excludeWeirdPaths = Turtle.findtree (Turtle.suffix $ Turtle.noneOf "_") 258 | 259 | firstExistingFile :: [TurtlePath] -> IO (Maybe TurtlePath) 260 | firstExistingFile files = do 261 | case files of 262 | [] -> return Nothing 263 | file : fs -> do 264 | exists <- Turtle.testfile file 265 | if exists then return (Just file) else firstExistingFile fs 266 | 267 | basenameLine :: TurtlePath -> Turtle.Shell Turtle.Line 268 | basenameLine path = case (Turtle.textToLine $ Turtle.format Turtle.fp $ Turtle.basename path) of 269 | Nothing -> Turtle.die $ Turtle.format ("Unable to determine basename from path: " % Turtle.fp % "\n") path 270 | Just bn -> return bn 271 | 272 | buildFilename :: [Turtle.Line] -> T.Text -> TurtlePath 273 | buildFilename identifiers ext = T.unpack (T.intercalate "-" (map Turtle.lineToText identifiers)) Turtle.<.> (T.unpack ext) 274 | 275 | shellToList :: Turtle.Shell a -> Turtle.Shell [a] 276 | shellToList files = Turtle.fold files Fold.list 277 | 278 | writeFiles :: IO (Map.Map TurtlePath T.Text) -> IO [TurtlePath] 279 | writeFiles fileMap = do 280 | m <- fileMap 281 | writeFiles' m 282 | 283 | writeFiles' :: Map.Map TurtlePath T.Text -> IO [TurtlePath] 284 | writeFiles' fileMap = do 285 | writeTextMap fileMap 286 | return $ Map.keys fileMap 287 | 288 | writeTextMap :: Map.Map TurtlePath T.Text -> IO () 289 | writeTextMap = Map.foldlWithKey (\a k v -> a <> T.writeFile k v) (return ()) 290 | 291 | changeExtension :: T.Text -> TurtlePath -> TurtlePath 292 | changeExtension ext path = (Turtle.dropExtension path) Turtle.<.> (T.unpack ext) 293 | 294 | changePathAndExtension :: TurtlePath -> T.Text -> TurtlePath -> TurtlePath 295 | changePathAndExtension newOutputLocation newExt = (changeOutputPath newOutputLocation) . (changeExtension newExt) 296 | 297 | changeOutputPath :: TurtlePath -> TurtlePath -> TurtlePath 298 | changeOutputPath newOutputLocation srcFile = mconcat $ map changeSrcDir $ Turtle.splitDirectories srcFile 299 | where 300 | changeSrcDir file = if file == "1-in/" || file == "2-preprocessed/" then newOutputLocation else file 301 | 302 | listOwners :: (HasBaseDir o) => o -> Turtle.Shell TurtlePath 303 | listOwners opts = fmap Turtle.basename $ lsDirs $ (turtleBaseDir opts) "import" 304 | 305 | intPath :: Integer -> TurtlePath 306 | intPath = T.unpack . (Turtle.format Turtle.d) 307 | 308 | includeYears :: TChan LogMessage -> TurtlePath -> IO [Integer] 309 | includeYears ch includeFile = do 310 | txt <- T.readFile includeFile 311 | case includeYears' txt of 312 | Left msg -> do 313 | channelErrLn ch msg 314 | return [] 315 | Right years -> return years 316 | 317 | includeYears' :: T.Text -> Either T.Text [Integer] 318 | includeYears' txt = case partitionEithers (includeYears'' txt) of 319 | (errors, []) -> do 320 | let msg = Turtle.format ("Unable to extract years from the following text:\n" % Turtle.s % "\nErrors:\n" % Turtle.s) txt (T.intercalate "\n" $ map T.pack errors) 321 | Left msg 322 | (_, years) -> Right years 323 | 324 | includeYears'' :: T.Text -> [Either String Integer] 325 | includeYears'' txt = map extractDigits (T.lines txt) 326 | 327 | extractDigits :: T.Text -> Either String Integer 328 | extractDigits txt = fmap fst $ (T.decimal . (T.filter isDigit)) txt 329 | -------------------------------------------------------------------------------- /test/CSVImport/Integration.hs: -------------------------------------------------------------------------------- 1 | {-# LANGUAGE OverloadedLists #-} 2 | {-# LANGUAGE OverloadedStrings #-} 3 | 4 | module CSVImport.Integration (tests) where 5 | 6 | import Control.Concurrent.STM 7 | import qualified Data.List as List (sort) 8 | import qualified Data.Map.Strict as Map 9 | import qualified Data.Text.IO as T 10 | import Hledger.Flow.Common 11 | import Hledger.Flow.Import.ImportHelpersTurtle (extraIncludesForFile, groupAndWriteIncludeFiles, includePreamble, toIncludeFiles) 12 | import Hledger.Flow.PathHelpers 13 | import Test.HUnit 14 | import TestHelpers (defaultOpts) 15 | import TestHelpersTurtle (extraFiles, hiddenFiles, journalFiles, touchAll) 16 | import Turtle 17 | import Prelude hiding (readFile, writeFile) 18 | 19 | testExtraIncludesForFile :: Test 20 | testExtraIncludesForFile = 21 | TestCase 22 | ( sh 23 | ( do 24 | currentDir <- pwd 25 | tmpdir <- using (mktempdir currentDir "hlflow") 26 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 27 | 28 | let importedJournals = map (tmpdir ) journalFiles :: [TurtlePath] 29 | let accountDir = "import/john/bogartbank/savings" 30 | let opening = tmpdir accountDir "2017-opening.journal" 31 | let closing = tmpdir accountDir "2017-closing.journal" 32 | let hidden = map (tmpdir ) hiddenFiles :: [TurtlePath] 33 | touchAll $ importedJournals ++ hidden 34 | 35 | let accountInclude = tmpdir accountDir "2017-include.journal" 36 | let expectedEmpty = [(accountInclude, [])] 37 | 38 | ch <- liftIO newTChanIO 39 | 40 | extraOpening1 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch accountInclude ["opening.journal"] [] [] 41 | liftIO $ assertEqual "The opening journal should not be included when it is not on disk" expectedEmpty extraOpening1 42 | 43 | extraClosing1 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch accountInclude ["closing.journal"] [] [] 44 | liftIO $ assertEqual "The closing journal should not be included when it is not on disk" expectedEmpty extraClosing1 45 | 46 | touchAll [opening, closing] 47 | 48 | extraOpening2 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch accountInclude ["opening.journal"] [] [] 49 | liftIO $ assertEqual "The opening journal should be included when it is on disk" [(accountInclude, [opening])] extraOpening2 50 | 51 | extraClosing2 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch accountInclude ["closing.journal"] [] [] 52 | liftIO $ assertEqual "The closing journal should be included when it is on disk" [(accountInclude, [closing])] extraClosing2 53 | ) 54 | ) 55 | 56 | testExtraIncludesPrices :: Test 57 | testExtraIncludesPrices = 58 | TestCase 59 | ( sh 60 | ( do 61 | currentDir <- pwd 62 | tmpdir <- using (mktempdir currentDir "hlflow") 63 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 64 | 65 | let importedJournals = map (tmpdir ) journalFiles :: [TurtlePath] 66 | touchAll $ importedJournals 67 | 68 | let priceFile = "prices" "2020" "prices.journal" 69 | 70 | let includeFile = tmpdir "import" "2020-include.journal" 71 | let expectedEmpty = [(includeFile, [])] 72 | 73 | ch <- liftIO newTChanIO 74 | 75 | price1 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch includeFile [] [] ["prices.journal"] 76 | liftIO $ assertEqual "The price file should not be included when it is not on disk" expectedEmpty price1 77 | 78 | touchAll [tmpdir priceFile] 79 | let expectedPricePath = tmpdir "import" ".." priceFile 80 | 81 | price2 <- liftIO $ extraIncludesForFile (defaultOpts tmpdirAbsPath) ch includeFile [] [] ["prices.journal"] 82 | liftIO $ assertEqual "The price file should be included when it is on disk" [(includeFile, [expectedPricePath])] price2 83 | ) 84 | ) 85 | 86 | testIncludesPrePost :: Test 87 | testIncludesPrePost = 88 | TestCase 89 | ( sh 90 | ( do 91 | currentDir <- pwd 92 | tmpdir <- using (mktempdir currentDir "hlflow") 93 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 94 | 95 | let ownerDir = tmpdir "import" "john" 96 | let includeFile = ownerDir "2019-include.journal" 97 | let pre = ownerDir "_manual_" "2019" "pre-import.journal" 98 | let post = ownerDir "_manual_" "2019" "post-import.journal" 99 | touchAll [pre, post] 100 | 101 | let includeMap = 102 | Map.singleton 103 | includeFile 104 | [ ownerDir "bank1" "2019-include.journal", 105 | ownerDir "bank2" "2019-include.journal" 106 | ] 107 | 108 | ch <- liftIO newTChanIO 109 | fileMap <- liftIO $ toIncludeFiles (defaultOpts tmpdirAbsPath) ch includeMap 110 | let expectedText = 111 | includePreamble 112 | <> "\n" 113 | <> "include _manual_/2019/pre-import.journal\n" 114 | <> "include bank1/2019-include.journal\n" 115 | <> "include bank2/2019-include.journal\n" 116 | <> "include _manual_/2019/post-import.journal\n" 117 | let expectedMap = Map.singleton includeFile expectedText 118 | liftIO $ assertEqual "All pre/post files on disk should be included" expectedMap fileMap 119 | ) 120 | ) 121 | 122 | testIncludesOpeningClosing :: Test 123 | testIncludesOpeningClosing = 124 | TestCase 125 | ( sh 126 | ( do 127 | currentDir <- pwd 128 | tmpdir <- using (mktempdir currentDir "hlflow") 129 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 130 | 131 | let ownerDir = tmpdir "import/john" 132 | let accountDir = ownerDir "bank1" "savings" 133 | let includeFile = accountDir "2019-include.journal" 134 | let opening = accountDir "2019-opening.journal" 135 | let closing = accountDir "2019-closing.journal" 136 | touchAll [opening, closing] 137 | 138 | let includeMap = Map.singleton includeFile [accountDir "3-journal" "2019" "2019-01-30.journal"] 139 | 140 | ch <- liftIO newTChanIO 141 | fileMap <- liftIO $ toIncludeFiles (defaultOpts tmpdirAbsPath) ch includeMap 142 | let expectedText = 143 | includePreamble 144 | <> "\n" 145 | <> "include 2019-opening.journal\n" 146 | <> "include 3-journal/2019/2019-01-30.journal\n" 147 | <> "include 2019-closing.journal\n" 148 | let expectedMap = Map.singleton includeFile expectedText 149 | liftIO $ assertEqual "All opening/closing files on disk should be included" expectedMap fileMap 150 | ) 151 | ) 152 | 153 | testIncludesPrices :: Test 154 | testIncludesPrices = 155 | TestCase 156 | ( sh 157 | ( do 158 | currentDir <- pwd 159 | tmpdir <- using (mktempdir currentDir "hlflow") 160 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 161 | 162 | let importDir = tmpdir "import" 163 | let includeFile = importDir "2020-include.journal" 164 | let prices = tmpdir "prices" "2020" "prices.journal" 165 | let pre = importDir "_manual_" "2020" "pre-import.journal" 166 | let post = importDir "_manual_" "2020" "post-import.journal" 167 | touchAll [prices, pre, post] 168 | 169 | let includeMap = Map.singleton includeFile [importDir "john" "2020-include.journal"] 170 | 171 | ch <- liftIO newTChanIO 172 | fileMap <- liftIO $ toIncludeFiles (defaultOpts tmpdirAbsPath) ch includeMap 173 | let expectedText = 174 | includePreamble 175 | <> "\n" 176 | <> "include _manual_/2020/pre-import.journal\n" 177 | <> "include john/2020-include.journal\n" 178 | <> "include ../prices/2020/prices.journal\n" 179 | <> "include _manual_/2020/post-import.journal\n" 180 | let expectedMap = Map.singleton includeFile expectedText 181 | liftIO $ assertEqual "The price file should be included together with any pre/post files" expectedMap fileMap 182 | ) 183 | ) 184 | 185 | testWriteIncludeFiles :: Test 186 | testWriteIncludeFiles = 187 | TestCase 188 | ( sh 189 | ( do 190 | currentDir <- pwd 191 | tmpdir <- using (mktempdir currentDir "hlflow") 192 | tmpdirAbsPath <- fromTurtleAbsDir tmpdir 193 | 194 | let importedJournals = map (tmpdir ) journalFiles :: [TurtlePath] 195 | let extras = map (tmpdir ) extraFiles :: [TurtlePath] 196 | let hidden = map (tmpdir ) hiddenFiles :: [TurtlePath] 197 | touchAll $ importedJournals ++ extras ++ hidden 198 | 199 | let jane1 = tmpdir "import/jane/bogartbank/checking/2018-include.journal" 200 | let jane2 = tmpdir "import/jane/bogartbank/checking/2019-include.journal" 201 | let jane3 = tmpdir "import/jane/bogartbank/savings/2017-include.journal" 202 | let jane4 = tmpdir "import/jane/bogartbank/savings/2018-include.journal" 203 | let jane5 = tmpdir "import/jane/otherbank/creditcard/2017-include.journal" 204 | let jane6 = tmpdir "import/jane/otherbank/creditcard/2018-include.journal" 205 | let jane7 = tmpdir "import/jane/otherbank/investments/2018-include.journal" 206 | let jane8 = tmpdir "import/jane/otherbank/investments/2019-include.journal" 207 | 208 | let john1 = tmpdir "import/john/bogartbank/checking/2018-include.journal" 209 | let john2 = tmpdir "import/john/bogartbank/checking/2019-include.journal" 210 | let john3 = tmpdir "import/john/bogartbank/savings/2017-include.journal" 211 | let john4 = tmpdir "import/john/bogartbank/savings/2018-include.journal" 212 | let john5 = tmpdir "import/john/otherbank/creditcard/2017-include.journal" 213 | let john6 = tmpdir "import/john/otherbank/creditcard/2018-include.journal" 214 | let john7 = tmpdir "import/john/otherbank/investments/2018-include.journal" 215 | let john8 = tmpdir "import/john/otherbank/investments/2019-include.journal" 216 | let expectedIncludes = 217 | [ jane1, 218 | jane2, 219 | jane3, 220 | jane4, 221 | jane5, 222 | jane6, 223 | jane7, 224 | jane8, 225 | john1, 226 | john2, 227 | john3, 228 | john4, 229 | john5, 230 | john6, 231 | john7, 232 | john8 233 | ] 234 | 235 | ch <- liftIO newTChanIO 236 | reportedAsWritten <- liftIO $ groupAndWriteIncludeFiles (defaultOpts tmpdirAbsPath) ch importedJournals 237 | liftIO $ assertEqual "groupAndWriteIncludeFiles should return which files it wrote" expectedIncludes reportedAsWritten 238 | 239 | let allYears = 240 | [ tmpdir "import/jane/bogartbank/checking/all-years.journal", 241 | tmpdir "import/jane/bogartbank/savings/all-years.journal", 242 | tmpdir "import/jane/otherbank/creditcard/all-years.journal", 243 | tmpdir "import/jane/otherbank/investments/all-years.journal", 244 | tmpdir "import/john/bogartbank/checking/all-years.journal", 245 | tmpdir "import/john/bogartbank/savings/all-years.journal", 246 | tmpdir "import/john/otherbank/creditcard/all-years.journal", 247 | tmpdir "import/john/otherbank/investments/all-years.journal" 248 | ] 249 | let expectedOnDisk = List.sort $ reportedAsWritten ++ extras ++ importedJournals ++ allYears 250 | allFilesOnDisk <- single $ sort $ onlyFiles $ lstree tmpdir 251 | liftIO $ assertEqual "The actual files on disk should match what groupAndWriteIncludeFiles reported" expectedOnDisk allFilesOnDisk 252 | 253 | let expectedJohn1Contents = 254 | includePreamble 255 | <> "\n" 256 | <> "include 3-journal/2018/2018-10-30.journal\n" 257 | <> "include 3-journal/2018/2018-11-30.journal\n" 258 | <> "include 3-journal/2018/2018-12-30.journal\n" 259 | actualJohn1Contents <- liftIO $ T.readFile john1 260 | liftIO $ assertEqual "John1: The include file contents should be the journal files" expectedJohn1Contents actualJohn1Contents 261 | 262 | let expectedJohn2Contents = 263 | includePreamble 264 | <> "\n" 265 | <> "include 3-journal/2019/2019-01-30.journal\n" 266 | <> "include 3-journal/2019/2019-02-30.journal\n" 267 | actualJohn2Contents <- liftIO $ T.readFile john2 268 | liftIO $ assertEqual "John2: The include file contents should be the journal files" expectedJohn2Contents actualJohn2Contents 269 | 270 | let expectedJohn3Contents = 271 | includePreamble 272 | <> "\n" 273 | <> "include 2017-opening.journal\n" 274 | <> "include 3-journal/2017/2017-11-30.journal\n" 275 | <> "include 3-journal/2017/2017-12-30.journal\n" 276 | actualJohn3Contents <- liftIO $ T.readFile john3 277 | liftIO $ assertEqual "John3: The include file contents should be the journal files" expectedJohn3Contents actualJohn3Contents 278 | 279 | let expectedJohn4Contents = 280 | includePreamble 281 | <> "\n" 282 | <> "include 3-journal/2018/2018-01-30.journal\n" 283 | <> "include 3-journal/2018/2018-02-30.journal\n" 284 | actualJohn4Contents <- liftIO $ T.readFile john4 285 | liftIO $ assertEqual "John4: The include file contents should be the journal files" expectedJohn4Contents actualJohn4Contents 286 | 287 | let expectedJane7Contents = 288 | includePreamble 289 | <> "\n" 290 | <> "include 3-journal/2018/2018-12-30.journal\n" 291 | actualJane7Contents <- liftIO $ T.readFile jane7 292 | liftIO $ assertEqual "Jane7: The include file contents should be the journal files" expectedJane7Contents actualJane7Contents 293 | ) 294 | ) 295 | 296 | tests :: Test 297 | tests = TestList [testExtraIncludesForFile, testExtraIncludesPrices, testIncludesPrePost, testIncludesOpeningClosing, testIncludesPrices, testWriteIncludeFiles] 298 | -------------------------------------------------------------------------------- /step-by-step/part1.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | #+TITLE: Hledger Flow: Step-By-Step 3 | #+AUTHOR: 4 | #+REVEAL_TRANS: default 5 | #+REVEAL_THEME: beige 6 | #+OPTIONS: num:nil 7 | #+PROPERTY: header-args:sh :prologue exec 2>&1 :epilogue echo : 8 | 9 | * Part 1 10 | 11 | This is the first part in a a series of step-by-step instructions. 12 | 13 | They are intended to be read in sequence. Head over to the [[file:README.org][docs README]] to see all parts. 14 | 15 | * About This Document 16 | 17 | This document is a [[https://www.offerzen.com/blog/literate-programming-empower-your-writing-with-emacs-org-mode][literate]] [[https://orgmode.org/worg/org-contrib/babel/intro.html][program]]. 18 | You can read it like a normal article, either [[https://github.com/apauley/hledger-flow/blob/master/docs/part1.org][on the web]] or [[https://pauley.org.za/hledger-flow/][as a slide show]]. 19 | 20 | But you can also [[https://github.com/apauley/hledger-flow][clone the repository]] and open [[https://raw.githubusercontent.com/apauley/hledger-flow/master/docs/part1.org][this org-mode file]] in emacs. 21 | Then you can execute each code snippet by pressing =C-c C-c= with your cursor on the relevant code block. 22 | 23 | * The Story of an African Chef 24 | 25 | Meet Gawie de Groot. Gawie is a successful [[https://en.wikipedia.org/wiki/Gonimbrasia_belina#As_food][mopane worm]] chef, based in the northernmost section of the [[https://en.wikipedia.org/wiki/Kruger_National_Park][Kruger National Park]]. 26 | 27 | [[./img/mopane-worm-meal.jpg]] 28 | 29 | 30 | #+BEGIN_SRC org :results none :exports none 31 | Image downloaded from https://commons.wikimedia.org/wiki/File:Mopane-worm-meal.jpg 32 | Author: Ling Symon 33 | #+END_SRC 34 | 35 | #+REVEAL: split 36 | 37 | In times gone by he used to work in a big city as an IT professional. 38 | But after a few years he decided to choose a quieter life in the bushveld. 39 | 40 | He now works at a renowned bush restaurant aptly named the "=Grillerige Groen Goggatjie=". 41 | 42 | #+REVEAL: split 43 | 44 | Demand for his traditional African cuisine has sky-rocketed, with visitors from all over Zimbabwe, Mozambique and South Africa 45 | coming to enjoy his dishes. 46 | 47 | Business has been good the last few years, but Gawie wants to make sure that he'll also be OK when business is down. 48 | 49 | It is time for Gawie to take charge of his finances. 50 | 51 | * The First CSV Statement 52 | 53 | Gawie followed the [[https://github.com/apauley/hledger-flow#build-instructions][build instructions]] and ended up with his very own =hledger-flow= executable. 54 | 55 | #+REVEAL: split 56 | 57 | Let's just run this =hledger-flow= command and see what happens... 58 | 59 | #+NAME: hm-noargs 60 | #+BEGIN_SRC sh :results output :exports both 61 | hledger-flow 62 | #+END_SRC 63 | 64 | #+RESULTS: hm-noargs 65 | #+begin_example 66 | An hledger workflow focusing on automated statement import and classification: 67 | https://github.com/apauley/hledger-flow#readme 68 | 69 | Usage: hledger-flow ([-v|--verbose] [-H|--hledger-path HLEDGER-PATH] 70 | [--show-options] [--batch-size SIZE] [--sequential] 71 | (import | report) | 72 | (-V|--version)) 73 | 74 | Available options: 75 | -h,--help Show this help text 76 | -v,--verbose Print more verbose output 77 | -H,--hledger-path HLEDGER-PATH 78 | The full path to an hledger executable 79 | --show-options Print the options this program will run with 80 | --batch-size SIZE Parallel processing of files are done in batches of 81 | the specified size. Default: 200. Ignored during 82 | sequential processing. 83 | --sequential Disable parallel processing 84 | -V,--version Display version information 85 | 86 | Available commands: 87 | import Uses hledger with your own rules and/or scripts to convert electronic statements into categorised journal files 88 | report Generate Reports 89 | 90 | #+end_example 91 | 92 | Well OK, at least it prints some helpful output. 93 | 94 | #+REVEAL: split 95 | 96 | Gawie wants to import his CSV statements. 97 | 98 | Luckily the =import= subcommand has a bit of help text as well: 99 | 100 | #+NAME: hm-import-help 101 | #+BEGIN_SRC sh :results org :exports both 102 | hledger-flow import --help 103 | #+END_SRC 104 | 105 | #+RESULTS: hm-import-help 106 | #+begin_src org 107 | Usage: hledger-flow import [DIR] [--start-year YEAR] [--new-files-only] 108 | Uses hledger with your own rules and/or scripts to convert electronic statements into categorised journal files 109 | 110 | Available options: 111 | DIR The directory to import. Use the base directory for a 112 | full import or a sub-directory for a partial import. 113 | Defaults to the current directory. This behaviour is 114 | changing: see --enable-future-rundir 115 | --start-year YEAR Import only from the specified year and onwards, 116 | ignoring previous years. By default all available 117 | years are imported. Valid values include a 4-digit 118 | year or 'current' for the current year 119 | --new-files-only Don't regenerate transaction files if they are 120 | already present. This applies to hledger journal 121 | files as well as files produced by the preprocess and 122 | construct scripts. 123 | -h,--help Show this help text 124 | 125 | #+end_src 126 | 127 | #+REVEAL: split 128 | 129 | Gawie starts by creating a new directory specifically for his hledger journals: 130 | 131 | #+NAME: rm-fin-dir 132 | #+BEGIN_SRC sh :results none :exports results 133 | rm -rf my-finances 134 | #+END_SRC 135 | 136 | #+NAME: new-fin-dir 137 | #+BEGIN_SRC sh :results none :exports both 138 | mkdir my-finances 139 | #+END_SRC 140 | 141 | Now he can point the import to his new finances directory: 142 | #+NAME: import1 143 | #+BEGIN_SRC sh :results org :exports both 144 | hledger-flow import ./my-finances 145 | #+END_SRC 146 | 147 | #+REVEAL: split 148 | 149 | Hmmm, an error: 150 | #+RESULTS: import1 151 | #+begin_src org 152 | hledger-flow: Unable to find an import directory at "./my-finances/" (or in any of its parent directories). 153 | 154 | Have a look at the documentation for more information: 155 | https://github.com/apauley/hledger-flow#getting-started 156 | 157 | #+end_src 158 | 159 | Gawie carefully interprets the error message using the skills he obtained during his years as an IT professional. 160 | 161 | He concludes that =hledger-flow= expects to find its input files in specifically named directories. 162 | 163 | #+REVEAL: split 164 | 165 | Looking at the [[https://github.com/apauley/hledger-flow#input-files][documentation]] he sees there should be several account and bank-specific directories 166 | under the =import= directory. 167 | 168 | #+REVEAL: split 169 | 170 | Gawie's salary is deposited into his cheque account at =Bogart Bank=, so this seems like a good account to start with: 171 | 172 | #+NAME: first-input-file 173 | #+BEGIN_SRC sh :results none :exports both 174 | mkdir -p my-finances/import/gawie/bogart/cheque/1-in/2016/ 175 | cp Downloads/Bogart/123456789_2016-03-30.csv \ 176 | my-finances/import/gawie/bogart/cheque/1-in/2016/ 177 | #+END_SRC 178 | 179 | #+REVEAL: split 180 | 181 | Let's see what our tree structure looks like now: 182 | #+NAME: tree-after-1st-file 183 | #+BEGIN_SRC sh :results org :exports both 184 | tree my-finances/ 185 | #+END_SRC 186 | 187 | #+RESULTS: tree-after-1st-file 188 | #+begin_src org 189 | my-finances/ 190 | `-- import 191 | `-- gawie 192 | `-- bogart 193 | `-- cheque 194 | `-- 1-in 195 | `-- 2016 196 | `-- 123456789_2016-03-30.csv 197 | 198 | 6 directories, 1 file 199 | 200 | #+end_src 201 | 202 | #+REVEAL: split 203 | 204 | It is time to add what we have to source control. 205 | 206 | #+NAME: git-init 207 | #+BEGIN_SRC sh :results none :exports both 208 | cd my-finances/ 209 | git init . 210 | git add . 211 | git commit -m 'Initial commit' 212 | cd .. 213 | #+END_SRC 214 | 215 | #+REVEAL: split 216 | 217 | Let's try the import again: 218 | #+NAME: import2 219 | #+BEGIN_SRC sh :results org :exports both 220 | hledger-flow import ./my-finances 221 | #+END_SRC 222 | 223 | #+RESULTS: import2 224 | #+begin_src org 225 | I couldn't find an hledger rules file while trying to import 226 | import/gawie/bogart/cheque/1-in/2016/123456789_2016-03-30.csv 227 | 228 | I will happily use the first rules file I can find from any one of these 2 files: 229 | import/gawie/bogart/cheque/bogart-cheque.rules 230 | import/bogart.rules 231 | 232 | Here is a bit of documentation about rules files that you may find helpful: 233 | https://github.com/apauley/hledger-flow#rules-files 234 | 235 | #+end_src 236 | 237 | #+REVEAL: split 238 | 239 | Another cryptic error. 240 | 241 | This one is caused by a missing [[https://github.com/apauley/hledger-flow#the-rules-file][rules file]]. 242 | 243 | #+REVEAL: split 244 | 245 | After looking through the [[http://hledger.org/csv.html][hledger documentation on CSV rules files]], 246 | Gawie concludes that the dates in Bogart Bank's CSV statement is incompatible with basic logic, reason and decency. 247 | 248 | Luckily he isn't the only one suffering at the hands of bureaucratic incompetence: someone else has already written [[https://github.com/apauley/fnb-csv-demoronizer][a script]] to 249 | fix stupid dates like those used by Bogart Bank. 250 | 251 | #+REVEAL: split 252 | 253 | This looks like a job for a [[https://github.com/apauley/hledger-flow#the-preprocess-script][preprocess script]]. 254 | 255 | #+REVEAL: split 256 | 257 | Gawie adds the CSV transformation script as a submodule to his repository: 258 | 259 | #+NAME: git-submodule-demoronizer 260 | #+BEGIN_SRC sh :results none :exports both 261 | cd my-finances/ 262 | git submodule add https://github.com/apauley/fnb-csv-demoronizer.git 263 | git commit -m 'Added submodule: fnb-csv-demoronizer' 264 | cd .. 265 | #+END_SRC 266 | 267 | #+REVEAL: split 268 | 269 | =hledger-flow= looks for a file named [[https://github.com/apauley/hledger-flow#the-preprocess-script][preprocess]] in the account directory. 270 | 271 | #+REVEAL: split 272 | 273 | Gawie just creates a symbolic link named =preprocess=. 274 | This works because the downloaded script takes an input file and an output file as the first two positional arguments, 275 | very much as the =preprocess= script would expect. 276 | And luckily it ignores the other parameters that =hledger-flow= sends through. 277 | 278 | #+REVEAL: split 279 | 280 | #+NAME: symlink-demoronizer 281 | #+BEGIN_SRC sh :results none :exports both 282 | cd my-finances/import/gawie/bogart/cheque 283 | ln -s ../../../../fnb-csv-demoronizer/fnb-csv-demoronizer preprocess 284 | #+END_SRC 285 | 286 | Now when we try the import again, it still displays an error due to our missing rules file: 287 | 288 | #+REVEAL: split 289 | 290 | #+NAME: import3 291 | #+BEGIN_SRC sh :results org :exports both 292 | hledger-flow import ./my-finances 293 | #+END_SRC 294 | 295 | #+RESULTS: import3 296 | #+begin_src org 297 | I couldn't find an hledger rules file while trying to import 298 | import/gawie/bogart/cheque/2-preprocessed/2016/123456789_2016-03-30.csv 299 | 300 | I will happily use the first rules file I can find from any one of these 2 files: 301 | import/gawie/bogart/cheque/bogart-cheque.rules 302 | import/bogart.rules 303 | 304 | Here is a bit of documentation about rules files that you may find helpful: 305 | https://github.com/apauley/hledger-flow#rules-files 306 | 307 | #+end_src 308 | 309 | This time we can see that our statement was preprocessed despite the rules file error: 310 | 311 | #+NAME: head-preprocess 312 | #+BEGIN_SRC sh :results org :exports both 313 | head -n 2 my-finances/import/gawie/bogart/cheque/2-preprocessed/2016/123456789_2016-03-30.csv 314 | #+END_SRC 315 | 316 | #+RESULTS: head-preprocess 317 | #+begin_src org 318 | "5","'Nommer'","'Datum'","'Beskrywing1'","'Beskrywing2'","'Beskrywing3'","'Bedrag'","'Saldo'","'Opgeloopte Koste'" 319 | "5","1","2016-03-01","#Monthly Bank Fee","","","-500.00","40000.00","" 320 | 321 | #+end_src 322 | 323 | #+REVEAL: split 324 | 325 | Time for another git checkpoint. 326 | 327 | #+NAME: git-checkpoint-preprocess 328 | #+BEGIN_SRC sh :results none :exports both 329 | cd my-finances/ 330 | git add . 331 | git commit -m 'The preprocessed CSV now has dates we can work with!' 332 | cd .. 333 | #+END_SRC 334 | 335 | #+REVEAL: split 336 | 337 | Now that we have sane dates in a CSV file, let's try to create a [[http://hledger.org/manual.html#csv-rules][rules file]]: 338 | #+NAME: bogart-cheque-rules-file 339 | #+BEGIN_SRC hledger :tangle my-finances/import/gawie/bogart/cheque/bogart-cheque.rules 340 | skip 1 341 | 342 | fields _, _, date, desc1, desc2, desc3, amount, balance, _ 343 | 344 | currency R 345 | status * 346 | 347 | account1 Assets:Current:Gawie:Bogart:Cheque 348 | description %desc1/%desc2/%desc3 349 | #+END_SRC 350 | 351 | Gawie saves this file as =my-finances/import/gawie/bogart/cheque/bogart-cheque.rules=. 352 | 353 | #+REVEAL: split 354 | 355 | #+NAME: tangle-rules 356 | #+BEGIN_SRC emacs-lisp :results none :exports results 357 | ; Narrator: this just tells emacs to write out the rules file. Carry on. 358 | ; FIXME: This should just tangle the one relevant block, not all tangle blocks 359 | (org-babel-tangle-file (buffer-file-name)) 360 | #+END_SRC 361 | 362 | Time for another git checkpoint. 363 | 364 | #+NAME: git-checkpoint-rules 365 | #+BEGIN_SRC sh :results none :exports both 366 | cd my-finances/ 367 | git add . 368 | git commit -m 'A CSV rules file' 369 | cd .. 370 | #+END_SRC 371 | 372 | #+REVEAL: split 373 | 374 | This time the import is successful, and we see a number of newly generated files: 375 | #+NAME: import4 376 | #+BEGIN_SRC sh :results org :exports both 377 | hledger-flow import ./my-finances 378 | tree my-finances 379 | #+END_SRC 380 | 381 | #+REVEAL: split 382 | 383 | #+RESULTS: import4 384 | #+begin_src org 385 | Wrote include files for 1 journals in 0.003716s 386 | Imported 1/1 journals in 0.101596s 387 | my-finances 388 | |-- all-years.journal 389 | |-- fnb-csv-demoronizer 390 | | |-- README.org 391 | | `-- fnb-csv-demoronizer 392 | `-- import 393 | |-- 2016-include.journal 394 | |-- all-years.journal 395 | `-- gawie 396 | |-- 2016-include.journal 397 | |-- all-years.journal 398 | `-- bogart 399 | |-- 2016-include.journal 400 | |-- all-years.journal 401 | `-- cheque 402 | |-- 1-in 403 | | `-- 2016 404 | | `-- 123456789_2016-03-30.csv 405 | |-- 2-preprocessed 406 | | `-- 2016 407 | | `-- 123456789_2016-03-30.csv 408 | |-- 2016-include.journal 409 | |-- 3-journal 410 | | `-- 2016 411 | | `-- 123456789_2016-03-30.journal 412 | |-- all-years.journal 413 | |-- bogart-cheque.rules 414 | `-- preprocess -> ../../../../fnb-csv-demoronizer/fnb-csv-demoronizer 415 | 416 | 11 directories, 16 files 417 | 418 | #+end_src 419 | 420 | #+REVEAL: split 421 | 422 | Bogart Bank's CSV file has been transformed into an =hledger= journal file. 423 | 424 | This is the first transaction in the file: 425 | #+NAME: head-1st-journal 426 | #+BEGIN_SRC sh :results org :exports both 427 | head -n 3 my-finances/import/gawie/bogart/cheque/3-journal/2016/123456789_2016-03-30.journal 428 | #+END_SRC 429 | 430 | #+RESULTS: head-1st-journal 431 | #+begin_src org 432 | 2016-03-01 * #Monthly Bank Fee// 433 | Assets:Current:Gawie:Bogart:Cheque R-500.00 = R40000.00 434 | expenses:unknown R500.00 435 | 436 | #+end_src 437 | 438 | #+REVEAL: split 439 | 440 | A final checkpoint and we're done with part 1. 441 | 442 | #+NAME: git-checkpoint-1st-journal 443 | #+BEGIN_SRC sh :results none :exports both 444 | cd my-finances/ 445 | git add . 446 | git commit -m 'My first imported journal' 447 | cd .. 448 | #+END_SRC 449 | 450 | #+REVEAL: split 451 | 452 | The story continues with [[file:part2.org][part 2]]. 453 | -------------------------------------------------------------------------------- /docs/README.org: -------------------------------------------------------------------------------- 1 | #+STARTUP: showall 2 | 3 | * Getting Started 4 | :PROPERTIES: 5 | :CUSTOM_ID: getting-started 6 | :END: 7 | 8 | Have a look at the [[file:step-by-step/README.org][detailed step-by-step instructions]] and the feature reference below. 9 | 10 | You can see the example imported financial transactions as it was 11 | generated by the step-by-step instructions here: 12 | 13 | [[https://github.com/apauley/hledger-flow-example][https://github.com/apauley/hledger-flow-example]] 14 | 15 | * Feature Reference 16 | :PROPERTIES: 17 | :CUSTOM_ID: feature-reference 18 | :END: 19 | 20 | ** Input Files 21 | :PROPERTIES: 22 | :CUSTOM_ID: input-files 23 | :END: 24 | 25 | Your input files will probably be CSV files with a line for each 26 | transaction, although other file types will work fine if you use a 27 | =preprocess= or a =construct= script that can read them. These scripts 28 | are explained later. 29 | 30 | We mostly use conventions based on a predefined directory structure for 31 | your input statements. 32 | 33 | For example, assuming you have a =savings= account at =mybank=, you'll 34 | put your first CSV statement here: 35 | =import/john/mybank/savings/1-in/2018/123456789_2018-06-30.csv=. 36 | 37 | Some people may want to include accounts belonging to their spouse as 38 | part of the household finances: 39 | =import/spouse/otherbank/checking/1-in/2018/987654321_2018-06-30.csv=. 40 | 41 | *** More About Input Files 42 | :PROPERTIES: 43 | :CUSTOM_ID: more-about-input-files 44 | :END: 45 | 46 | All files and directories under the =import= directory are related to the 47 | automatic importing and classification of transactions. 48 | 49 | The directory directly under =import= is meant to indicate the owner or 50 | custodian of the accounts below it. It mostly has an impact on 51 | reporting. You may want to have separate reports for =import/mycompany= 52 | and =import/personal=. 53 | 54 | Below the directory for the owner we can indicate where an account is 55 | held. For a bank account you may choose to name it =import/john/mybank=. 56 | 57 | If your underground bunker filled with gold has CSV statements linked to 58 | it, then you can absolutely create =import/john/secret-treasure-room=. 59 | 60 | Under the directory for the financial institution, you'll have a 61 | directory for each account at that institution, e.g. 62 | =import/mycompany/bigbankinc/customer-deposits= and 63 | =import/mycompany/bigbankinc/expense-account=. 64 | 65 | Next you'll create a directory named =1-in=. This is to distinguish it 66 | from =2-preprocessed= and =3-journal= which will be auto-generated 67 | later. 68 | 69 | Under =1-in= you'll create a directory for the year, e.g. =2018=, and 70 | within that you can copy the statements for that year: 71 | =import/john/mybank/savings/1-in/2018/123456789_2018-06-30.csv= 72 | 73 | *** Stability of this Feature 74 | :PROPERTIES: 75 | :CUSTOM_ID: stability-of-this-feature 76 | :END: 77 | 78 | The basic owner/bank/account/year structure has been used and tested 79 | fairly extensively, I don't expect a need for it to change. 80 | 81 | I'm open to suggestions for improvement though. 82 | 83 | ** Rules Files 84 | :PROPERTIES: 85 | :CUSTOM_ID: rules-files 86 | :END: 87 | 88 | If your input file is in CSV format, or converted to CSV by your 89 | =preprocess= script, then you'll need an 90 | [[https://hledger.org/hledger.html#csv-format][hledger rules file]]. 91 | 92 | =hledger-flow= will try to find a rules file for each statement in a few 93 | places. The same rules file is typically used for all statements of a 94 | specific account, or even for all accounts of the same specific bank. 95 | 96 | - A global rules file for any =mybank= statement can be saved here: 97 | =import/mybank.rules= 98 | - A rules file for all statements of a specific account: 99 | =import/spouse/bigbankinc/savings/bigbankinc-savings.rules= 100 | 101 | *** Statement-specific Rules Files 102 | :PROPERTIES: 103 | :CUSTOM_ID: statement-specific-rules-files 104 | :END: 105 | 106 | What happens if some of the statements for an account has a different 107 | format than the others? 108 | 109 | This can happen if you normally get your statements directly from your 110 | bank, but some statements you had to download from somewhere else, like 111 | Mint, because your bank is being daft with older statements. 112 | 113 | In order to tell =hledger-flow= that you want to override the rules file 114 | for a specific statement, you need to add a suffix, separated by an 115 | underscore (=_=) and starting with the letters =rfo= (rules file 116 | override) to the filename of that statement. 117 | 118 | For example: assuming you've named your statement 119 | =99966633_20171223_1844_rfo-mint.csv=. 120 | 121 | =hledger-flow= will look for a rules file named =rfo-mint.rules= in the 122 | following places: 123 | 124 | - in the import directory, e.g. =import/rfo-mint.rules= 125 | - in the bank directory, e.g. =import/john/mybank/rfo-mint.rules= 126 | - in the account directory, e.g. 127 | =import/john/mybank/savings/rfo-mint.rules= 128 | 129 | *** Example rules file usage 130 | :PROPERTIES: 131 | :CUSTOM_ID: example-rules-file-usage 132 | :END: 133 | 134 | A common scenario is multiple accounts that share the same file format, 135 | but have different =account1= directives. 136 | 137 | One possible approach would be to include a shared rules file in your 138 | account-specific rules file. 139 | 140 | If you are lucky enough that all statements at =mybank= share a common 141 | format across all accounts, then you can =include= a rules file that 142 | just defines the parts that are shared across accounts. 143 | 144 | Two accounts at =mybank= may have rules files similar to these. 145 | 146 | A checking account at mybank: 147 | 148 | #+BEGIN_EXAMPLE 149 | # Saved as: import/john/mybank/checking/mybank-checking.rules 150 | include ../../../mybank-shared.rules 151 | account1 Assets:Current:John:MyBank:Checking 152 | #+END_EXAMPLE 153 | 154 | Another account at mybank: 155 | 156 | #+BEGIN_EXAMPLE 157 | # Saved as: import/alice/mybank/savings/mybank-savings.rules 158 | include ../../../mybank-shared.rules 159 | account1 Assets:Current:Alice:MyBank:Savings 160 | #+END_EXAMPLE 161 | 162 | Where =import/mybank-shared.rules= may define some shared attributes: 163 | 164 | #+BEGIN_EXAMPLE 165 | skip 1 166 | 167 | fields date, description, amount, balance 168 | 169 | date-format %Y-%m-%d 170 | currency $ 171 | #+END_EXAMPLE 172 | 173 | Another possible approach could be to use your =preprocess= script to 174 | write out a CSV file that has extra fields for =account1= and 175 | =account2=. 176 | 177 | You could then create the above mentioned global =import/mybank.rules= 178 | with the fields defined more or less like this: 179 | 180 | #+BEGIN_EXAMPLE 181 | fields date, description, amount, balance, account1, account2 182 | #+END_EXAMPLE 183 | 184 | *** Stability of this Feature 185 | :PROPERTIES: 186 | :CUSTOM_ID: stability-of-this-feature-1 187 | :END: 188 | 189 | Rules files are a stable feature within 190 | [[http://hledger.org/][hledger]], and we're just using the normal 191 | hledger rules files. The account, bank and statement-specific rules 192 | files have been used and tested fairly extensively, I don't expect this 193 | to change. 194 | 195 | Let me know if you think it should change. 196 | 197 | ** Opening and Closing Balances 198 | :PROPERTIES: 199 | :CUSTOM_ID: opening-and-closing-balances 200 | :END: 201 | 202 | *** Opening Balances 203 | :PROPERTIES: 204 | :CUSTOM_ID: opening-balances 205 | :END: 206 | 207 | =hledger-flow= looks for a file named =YEAR-opening.journal= in each 208 | account directory, where =YEAR= corresponds to an actual year directory, 209 | eg. *1983* (if you have electronic statements 210 | [[https://en.wikipedia.org/wiki/Online_banking#First_online_banking_services_in_the_United_States][dating 211 | back to 1983]]). Example: 212 | =import/john/mybank/savings/1983-opening.journal= 213 | 214 | If it exists the file will automatically be included at the beginning of 215 | the generated journal include file for that year. 216 | 217 | You need to edit this file for each account to specify the opening 218 | balance at the date of the first available transaction. 219 | 220 | An opening balance may look something like this: 221 | 222 | #+BEGIN_EXAMPLE 223 | 2018-06-01 Savings Account Opening Balance 224 | assets:Current:MyBank:Savings $102.01 225 | equity:Opening Balances:MyBank:Savings 226 | #+END_EXAMPLE 227 | 228 | 229 | *** A Note of Caution Regarding Closing Balances 230 | 231 | When closing your balances it may result in some =hledger= queries showing zero-values, or there could be issues with balance assertions. 232 | 233 | Please have a look at the upstream =hledger= documentation on closing balances, e.g here: 234 | https://hledger.org/hledger.html#close-usage 235 | 236 | Some of the gotchas you may run into are also described in [[https://github.com/apauley/hledger-flow/issues/79][this hledger-flow issue]]. 237 | 238 | *** Closing Balances 239 | :PROPERTIES: 240 | :CUSTOM_ID: closing-balances 241 | :END: 242 | 243 | Similar to opening balances, =hledger-flow= looks for an optional file 244 | named =YEAR-closing.journal= in each account directory. Example: 245 | =import/john/mybank/savings/1983-closing.journal= 246 | 247 | If it exists the file will automatically be included at the end of the 248 | generated journal include file for that year. 249 | 250 | A closing balance may look something like this: 251 | 252 | #+BEGIN_EXAMPLE 253 | 2018-06-01 Savings Account Closing Balance 254 | assets:Current:MyBank:Savings $-234.56 = $0.00 255 | equity:Closing Balances:MyBank:Savings 256 | #+END_EXAMPLE 257 | 258 | *** Example Opening and Closing Journal Files 259 | :PROPERTIES: 260 | :CUSTOM_ID: example-opening-and-closing-journal-files 261 | :END: 262 | 263 | As an example, assuming that the relevant year is =2019= and 264 | =hledger-flow= is about to generate 265 | =import/john/mybank/savings/2019-include.journal=, then one or both of 266 | the following files will be added to the include file if they exist: 267 | 268 | 1. =import/john/mybank/savings/2019-opening.journal= 269 | 2. =import/john/mybank/savings/2019-closing.journal= 270 | 271 | The =opening.journal= will be included just before the other included 272 | entries, while the =closing.journal= will be included just after the 273 | other entries in that include file. 274 | 275 | An include file may look like this: 276 | 277 | #+BEGIN_SRC sh 278 | cat import/john/mybank/savings/2019-include.journal 279 | #+END_SRC 280 | 281 | #+BEGIN_EXAMPLE 282 | ### Generated by hledger-flow - DO NOT EDIT ### 283 | 284 | include 2019-opening.journal 285 | include 3-journal/2019/123456789_2019-01-30 286 | include 2019-closing.journal 287 | #+END_EXAMPLE 288 | 289 | *** Stability of this Feature 290 | :PROPERTIES: 291 | :CUSTOM_ID: stability-of-this-feature-2 292 | :END: 293 | 294 | Closing balances sometimes result in [[https://github.com/apauley/hledger-flow/issues/79][unexpected query results]]. 295 | In future we may change how/where the generated files include the closing journal. 296 | 297 | We may also need to suggest some naming conventions for opening and closing balances so that reports can exclude 298 | some of these transactions. 299 | 300 | It is also possible that we might want to change the name/location of the closing journal, 301 | but we'll try to avoid this if possible, because that would require users to rename their existing files. 302 | 303 | ** Price Files 304 | :PROPERTIES: 305 | :CUSTOM_ID: price-files 306 | :END: 307 | 308 | =hledger-flow= looks for [[https://hledger.org/journal.html#market-prices][price files]] to include in each yearly include file. 309 | 310 | For example, the presence of a file named =${BASE}/prices/2020/prices.journal= will result in some extra include file magic. 311 | 312 | The rest of this section assumes you'll have a file named =prices/2020/prices.journal= which contains price data for the year 2020. 313 | The =prices= directory should be right at the top of your =hledger-flow= base directory, next to the =import= directory. 314 | 315 | =hledger-flow= does not care how the price files got there, it only cares that you should have a separate file per year, 316 | and that it follows the above naming convention. 317 | 318 | Here is an example script which downloads prices and follows the naming convention: 319 | https://gist.github.com/apauley/398fa031c202733959af76b3b8ce8197 320 | 321 | After running an import with available price files you'll see a line has been added to =import/2020-include.journal=: 322 | 323 | #+BEGIN_EXAMPLE 324 | include ../prices/2020/prices.journal 325 | #+END_EXAMPLE 326 | 327 | ** Hledger Directives 328 | :PROPERTIES: 329 | :CUSTOM_ID: hledger-directives 330 | :END: 331 | 332 | Hledger allows you to specify some useful [[https://hledger.org/hledger.html#directives][directives]] which affect things such as number formatting. 333 | 334 | A convenient place to put these directives within =hledger-flow= is a file named =directives.journal= (in your hledger-flow base directory). 335 | 336 | If it exists =hledger-flow= will include it within the =all-years.journal=: 337 | 338 | #+BEGIN_SRC sh 339 | cat all-years.journal 340 | #+END_SRC 341 | 342 | #+BEGIN_EXAMPLE 343 | ### Generated by hledger-flow - DO NOT EDIT ### 344 | 345 | include directives.journal 346 | include import/all-years.journal 347 | #+END_EXAMPLE 348 | 349 | ** The =preprocess= Script 350 | :PROPERTIES: 351 | :CUSTOM_ID: the-preprocess-script 352 | :END: 353 | 354 | Sometimes the statements you get from your bank is 355 | [[https://github.com/apauley/fnb-csv-demoronizer][less than suitable]] 356 | for automatic processing. Or maybe you just want to make it easier for 357 | the hledger rules file to do its thing by adding some useful columns. 358 | 359 | If you put a script called =preprocess= in the account directory, e.g. 360 | =import/john/mybank/savings/preprocess=, then =hledger-flow= will call 361 | that script for each input statement. 362 | 363 | The =preprocess= script will be called with 4 positional parameters: 364 | 365 | 1. The path to the input statement, e.g. 366 | =import/john/mybank/savings/1-in/2018/123456789_2018-06-30.csv= 367 | 2. The path to an output file that can be sent to =hledger=, e.g. 368 | =import/john/mybank/savings/2-preprocessed/2018/123456789_2018-06-30.csv= 369 | 3. The name of the bank, e.g. =mybank= 370 | 4. The name of the account, e.g. =savings= 371 | 5. The name of the owner, e.g. =john= 372 | 373 | Your =preprocess= script is expected to: 374 | 375 | - read the input file 376 | - write a new output file at the supplied path that works with your 377 | rules file 378 | - be idempotent. Running =preprocess= multiple times on the same files 379 | will produce the same result. 380 | 381 | *** Stability of this Feature 382 | :PROPERTIES: 383 | :CUSTOM_ID: stability-of-this-feature-3 384 | :END: 385 | 386 | Stable and tested. 387 | 388 | ** The =construct= Script 389 | :PROPERTIES: 390 | :CUSTOM_ID: the-construct-script 391 | :END: 392 | 393 | If you need even more power and flexibility than what you can get from 394 | the =preprocess= script and =hledger='s [[https://hledger.org/csv.html][CSV import functionality]], then 395 | you can create your own custom script to =construct= transactions 396 | exactly as you need them. 397 | 398 | At the expense of more construction work for you, of course. 399 | 400 | The =construct= script can be used in addition to the =preprocess= 401 | script, or on it's own. But since the =construct= script is more 402 | powerful than the =preprocess= script, you could tell your =construct= 403 | script to do anything that the =preprocess= script would have done. 404 | 405 | Save your =construct= script in the account directory, e.g. 406 | =import/john/mybank/savings/construct=. 407 | 408 | =hledger-flow= will call your =construct= script with 5 positional 409 | parameters: 410 | 411 | 1. The path to the input statement, e.g. 412 | =import/john/mybank/savings/1-in/2018/123456789_2018-06-30.csv= 413 | 2. A "-" (indicating that output should be sent to =stdout=) 414 | 3. The name of the bank, e.g. =mybank= 415 | 4. The name of the account, e.g. =savings= 416 | 5. The name of the owner, e.g. =john= 417 | 418 | Your =construct= script is expected to: 419 | 420 | - read the input file 421 | - generate your own =hledger= journal transactions 422 | - be idempotent. Running =construct= multiple times on the same files 423 | should produce the same result. 424 | - send all journals to =stdout=. =hledger-flow= will pipe your standard output into 425 | =hledger= which will format it and save it to an output file. 426 | 427 | You can still use =stderr= in your construct script for any other output that you may want to see. 428 | 429 | *** Stability of this Feature 430 | :PROPERTIES: 431 | :CUSTOM_ID: stability-of-this-feature-4 432 | :END: 433 | 434 | Stable and tested. 435 | 436 | ** Manually Managed Journals 437 | :PROPERTIES: 438 | :CUSTOM_ID: manually-managed-journals 439 | :END: 440 | 441 | Not every transaction in your life comes with CSV statements. 442 | 443 | Sometimes you just need to add a transaction for that time you loaned a 444 | friend some money. 445 | 446 | =hledger-flow= looks for =pre-import= and =post-import= files related to 447 | each generated include file as part of the import. 448 | 449 | You can enter your own transactions manually into these files. 450 | 451 | You can run =hledger-flow import --verbose= to see exactly which files 452 | are being looked for. 453 | 454 | As an example, assuming that the relevant year is =2019= and 455 | =hledger-flow= is about to generate =import/john/2019-include.journal=, 456 | then one or both of the following files will be added to the include 457 | file if they exist: 458 | 459 | 1. =import/john/_manual_/2019/pre-import.journal= 460 | 2. =import/john/_manual_/2019/post-import.journal= 461 | 462 | The =pre-import.journal= will be included just before the other included 463 | entries, while the =post-import.journal= will be included just after the 464 | other entries in that include file. 465 | 466 | An include file may look like this: 467 | 468 | #+BEGIN_SRC sh 469 | cat import/john/2019-include.journal 470 | #+END_SRC 471 | 472 | #+BEGIN_EXAMPLE 473 | ### Generated by hledger-flow - DO NOT EDIT ### 474 | 475 | include _manual_/2019/pre-import.journal 476 | include mybank/2019-include.journal 477 | include otherbank/2019-include.journal 478 | include _manual_/2019/post-import.journal 479 | #+END_EXAMPLE 480 | 481 | *** Stability of this Feature 482 | :PROPERTIES: 483 | :CUSTOM_ID: stability-of-this-feature-5 484 | :END: 485 | 486 | It works, but the naming of =_manual_= looks a bit weird. Should it be 487 | changed? 488 | --------------------------------------------------------------------------------