├── estimate-extract-transform-load.sh
├── dividend-calendar-extract-transform-load.sh
├── earnings-calendar-extract-transform-load.sh
├── financial-statement-transform-load-yesterday.sh
├── dividend-calendar-extract.rkt
├── earnings-calendar-extract.rkt
├── estimate-extract.rkt
├── dump-dolt-earnings-calendar.rkt
├── dump-dat.rkt
├── dividend-calendar-transform-load.rkt
├── financial-statement-extract.rkt
├── earnings-calendar-transform-load.rkt
├── README.md
├── dump-dolt-estimates.rkt
├── schema.sql
├── dump-dolt-statements.rkt
├── cash-flow-statement-transform-load.rkt
├── LICENSE
├── income-statement-transform-load.rkt
├── cash-flow-statement-transform-load.2024-02-01.rkt
├── estimate-transform-load.rkt
└── balance-sheet-transform-load.rkt


/estimate-extract-transform-load.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | today=$(date "+%F")
 4 | dir=$(dirname "$0")
 5 | current_year=$(date "+%Y")
 6 | 
 7 | racket -y ${dir}/estimate-extract.rkt -p "$1"
 8 | racket -y ${dir}/estimate-transform-load.rkt -p "$1"
 9 | 
10 | 7zr a /var/local/zacks/estimates/${current_year}/${today}.7z /var/local/zacks/estimates/${today}/*.html
11 | 
12 | racket -y ${dir}/dump-dolt-estimates.rkt -p "$1"
13 | 


--------------------------------------------------------------------------------
/dividend-calendar-extract-transform-load.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | today=$(date "+%F")
 4 | dir=$(dirname "$0")
 5 | current_year=$(date "+%Y")
 6 | 
 7 | racket -y ${dir}/dividend-calendar-extract.rkt
 8 | racket -y ${dir}/dividend-calendar-transform-load.rkt -p "$1"
 9 | 
10 | 7zr a /var/local/zacks/dividend-calendar/${current_year}.7z /var/local/zacks/dividend-calendar/${today}
11 | 
12 | # racket -y ${dir}/dump-dolt-dividend-calendar.rkt -p "$1"
13 | 


--------------------------------------------------------------------------------
/earnings-calendar-extract-transform-load.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | today=$(date "+%F")
 4 | dir=$(dirname "$0")
 5 | current_year=$(date "+%Y")
 6 | 
 7 | racket -y ${dir}/earnings-calendar-extract.rkt
 8 | racket -y ${dir}/earnings-calendar-transform-load.rkt -p "$1"
 9 | 
10 | 7zr a /var/local/zacks/earnings-calendar/${current_year}.7z /var/local/zacks/earnings-calendar/${today}
11 | 
12 | racket -y ${dir}/dump-dolt-earnings-calendar.rkt -p "$1"
13 | 


--------------------------------------------------------------------------------
/financial-statement-transform-load-yesterday.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | yesterday=$(date -d "-1 day" "+%F")
 4 | dir=$(dirname "$0")
 5 | 
 6 | racket -y ${dir}/balance-sheet-transform-load.rkt -d ${yesterday} -p "$1"
 7 | racket -y ${dir}/cash-flow-statement-transform-load.2024-02-01.rkt -d ${yesterday} -p "$1"
 8 | racket -y ${dir}/income-statement-transform-load.rkt -d ${yesterday} -p "$1"
 9 | 
10 | 7zr a /var/local/zacks/balance-sheet/${yesterday}.7z /var/local/zacks/balance-sheet/${yesterday}/*.html
11 | 7zr a /var/local/zacks/cash-flow-statement/${yesterday}.7z /var/local/zacks/cash-flow-statement/${yesterday}/*.html
12 | 7zr a /var/local/zacks/income-statement/${yesterday}.7z /var/local/zacks/income-statement/${yesterday}/*.html
13 | 
14 | racket -y ${dir}/dump-dolt-statements.rkt -p "$1"
15 | 


--------------------------------------------------------------------------------
/dividend-calendar-extract.rkt:
--------------------------------------------------------------------------------
 1 | #lang racket/base
 2 | 
 3 | (require gregor
 4 |          gregor/period
 5 |          gregor/time
 6 |          net/http-easy
 7 |          racket/cmdline
 8 |          racket/file
 9 |          racket/list
10 |          racket/port
11 |          tasks
12 |          threading)
13 | 
14 | (define (download-day date)
15 |   (make-directory* (string-append "/var/local/zacks/dividend-calendar/" (~t (today) "yyyy-MM-dd")))
16 |   (call-with-output-file* (string-append "/var/local/zacks/dividend-calendar/" (~t (today) "yyyy-MM-dd") "/"
17 |                                         (~t date "yyyy-MM-dd") ".json")
18 |     (λ (out)
19 |       (with-handlers ([exn:fail?
20 |                        (λ (error)
21 |                          (displayln (string-append "Encountered error for " (~t date "yyyy-MM-dd")))
22 |                          (displayln error))])
23 |         (~> (string-append "https://www.zacks.com/includes/classes/z2_class_calendarfunctions_data.php"
24 |                            "?calltype=eventscal&type=5&date="
25 |                            (number->string (->posix (at-time date (time 6)))))
26 |             (get _)
27 |             (response-body _)
28 |             (write-bytes _ out))))
29 |     #:exists 'replace))
30 | 
31 | (define end-date (make-parameter (+days (today) (* 7 6))))
32 | 
33 | (define start-date (make-parameter (today)))
34 | 
35 | (command-line
36 |  #:program "racket dividend-calendar-extract.rkt"
37 |  #:once-each
38 |  [("-e" "--end-date") ed
39 |                       "End date. Defaults to today + 6 weeks"
40 |                       (end-date (iso8601->date ed))]
41 |  [("-s" "--start-date") sd
42 |                         "Start date. Defaults to today"
43 |                         (start-date (iso8601->date sd))])
44 | 
45 | (define delay-interval 10)
46 | 
47 | (with-task-server (for-each (λ (i) (schedule-delayed-task (λ () (download-day (+days (start-date) i)))
48 |                                                           (* i delay-interval)))
49 |                             (range 0 (period-ref (period-between (start-date) (end-date) '(days)) 'days)))
50 |   ; add a final task that will halt the task server
51 |   (schedule-delayed-task
52 |    (λ () (schedule-stop-task)) (* delay-interval (period-ref (period-between (start-date) (end-date) '(days)) 'days)))
53 |   (run-tasks))
54 | 


--------------------------------------------------------------------------------
/earnings-calendar-extract.rkt:
--------------------------------------------------------------------------------
 1 | #lang racket/base
 2 | 
 3 | (require gregor
 4 |          gregor/period
 5 |          gregor/time
 6 |          net/http-easy
 7 |          racket/cmdline
 8 |          racket/file
 9 |          racket/list
10 |          racket/port
11 |          tasks
12 |          threading)
13 | 
14 | (define (download-day date)
15 |   (make-directory* (string-append "/var/local/zacks/earnings-calendar/" (~t (today) "yyyy-MM-dd")))
16 |   (call-with-output-file* (string-append "/var/local/zacks/earnings-calendar/" (~t (today) "yyyy-MM-dd") "/"
17 |                                         (~t date "yyyy-MM-dd") ".json")
18 |     (λ (out)
19 |       (with-handlers ([exn:fail?
20 |                        (λ (error)
21 |                          (displayln (string-append "Encountered error for " (~t date "yyyy-MM-dd")))
22 |                          (displayln error))])
23 |         (~> (string-append "https://www.zacks.com/includes/classes/z2_class_calendarfunctions_data.php"
24 |                            "?calltype=eventscal&type=1&date="
25 |                            (number->string (->posix (at-time date (time 6)))))
26 |             (get _)
27 |             (response-body _)
28 |             (write-bytes _ out))))
29 |     #:exists 'replace))
30 | 
31 | (define end-date (make-parameter (+days (today) (* 7 6))))
32 | 
33 | (define start-date (make-parameter (today)))
34 | 
35 | (command-line
36 |  #:program "racket earnings-calendar-extract.rkt"
37 |  #:once-each
38 |  [("-e" "--end-date") ed
39 |                       "End date. Defaults to today + 6 weeks"
40 |                       (end-date (iso8601->date ed))]
41 |  [("-s" "--start-date") sd
42 |                         "Start date. Defaults to today"
43 |                         (start-date (iso8601->date sd))])
44 | 
45 | (define delay-interval 10)
46 | 
47 | (with-task-server (for-each (λ (i) (schedule-delayed-task (λ () (download-day (+days (start-date) i)))
48 |                                                           (* i delay-interval)))
49 |                             (range 0 (period-ref (period-between (start-date) (end-date) '(days)) 'days)))
50 |   ; add a final task that will halt the task server
51 |   (schedule-delayed-task
52 |    (λ () (schedule-stop-task)) (* delay-interval (period-ref (period-between (start-date) (end-date) '(days)) 'days)))
53 |   (run-tasks))
54 | 


--------------------------------------------------------------------------------
/estimate-extract.rkt:
--------------------------------------------------------------------------------
 1 | #lang racket/base
 2 | 
 3 | (require db
 4 |          gregor
 5 |          net/http-easy
 6 |          racket/cmdline
 7 |          racket/file
 8 |          racket/list
 9 |          racket/port
10 |          tasks
11 |          threading)
12 | 
13 | (define (download-estimates symbol)
14 |   (make-directory* (string-append "/var/local/zacks/estimates/" (~t (today) "yyyy-MM-dd")))
15 |   (call-with-output-file (string-append "/var/local/zacks/estimates/" (~t (today) "yyyy-MM-dd") "/" symbol ".detailed-estimates.html")
16 |     (λ (out) (with-handlers ([exn:fail?
17 |                               (λ (error)
18 |                                 (displayln (string-append "Encountered error for " symbol))
19 |                                 (displayln error))])
20 |                (~> (string-append "https://www.zacks.com/stock/quote/" symbol "/detailed-earning-estimates")
21 |                    (get _)
22 |                    (response-body _)
23 |                    (write-bytes _ out))))
24 |     #:exists 'replace))
25 | 
26 | (define db-user (make-parameter "user"))
27 | 
28 | (define db-name (make-parameter "local"))
29 | 
30 | (define db-pass (make-parameter ""))
31 | 
32 | (define first-symbol (make-parameter ""))
33 | 
34 | (define last-symbol (make-parameter ""))
35 | 
36 | (command-line
37 |  #:program "racket estimate-extract.2023-02-27.rkt"
38 |  #:once-each
39 |  [("-f" "--first-symbol") first
40 |                           "First symbol to query. Defaults to nothing"
41 |                           (first-symbol first)]
42 |  [("-l" "--last-symbol") last
43 |                          "Last symbol to query. Defaults to nothing"
44 |                          (last-symbol last)]
45 |  [("-n" "--db-name") name
46 |                      "Database name. Defaults to 'local'"
47 |                      (db-name name)]
48 |  [("-p" "--db-pass") password
49 |                      "Database password"
50 |                      (db-pass password)]
51 |  [("-u" "--db-user") user
52 |                      "Database user name. Defaults to 'user'"
53 |                      (db-user user)])
54 | 
55 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
56 | 
57 | (define symbols (query-list dbc "
58 | select
59 |   act_symbol
60 | from
61 |   nasdaq.symbol
62 | where
63 |   is_etf = false and
64 |   is_test_issue = false and
65 |   is_next_shares = false and
66 |   security_name !~ 'ETN' and
67 |   nasdaq_symbol !~ '[-\\$\\+\\*#!@%\\^=~]' and
68 |   case when nasdaq_symbol ~ '[A-Z]{4}[L-Z]'
69 |     then security_name !~ '(Note|Preferred|Right|Unit|Warrant)'
70 |     else true
71 |   end and
72 |   last_seen = (select max(last_seen) from nasdaq.symbol) and
73 |   case when $1 != ''
74 |     then act_symbol >= $1
75 |     else true
76 |   end and
77 |   case when $2 != ''
78 |     then act_symbol <= $2
79 |     else true
80 |   end
81 | order by
82 |   act_symbol;
83 | "
84 |                             (first-symbol)
85 |                             (last-symbol)))
86 | 
87 | (disconnect dbc)
88 | 
89 | (define delay-interval 10)
90 | 
91 | (define delays (map (λ (x) (* delay-interval x)) (range 0 (length symbols))))
92 | 
93 | (with-task-server (for-each (λ (l) (schedule-delayed-task (λ () (thread (λ () (download-estimates (first l)))))
94 |                                                           (second l)))
95 |                             (map list symbols delays))
96 |   ; add a final task that will halt the task server
97 |   (schedule-delayed-task (λ () (schedule-stop-task)) (* delay-interval (length delays)))
98 |   (run-tasks))
99 | 


--------------------------------------------------------------------------------
/dump-dolt-earnings-calendar.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          racket/cmdline
  6 |          racket/string
  7 |          racket/system)
  8 | 
  9 | (define base-folder (make-parameter "/var/local/dolt/earnings"))
 10 | 
 11 | (define as-of-date (make-parameter (~t (today) "yyyy-MM-dd")))
 12 | 
 13 | (define db-user (make-parameter "user"))
 14 | 
 15 | (define db-name (make-parameter "local"))
 16 | 
 17 | (define db-pass (make-parameter ""))
 18 | 
 19 | (command-line
 20 |  #:program "racket dump-dolt-calendar.rkt"
 21 |  #:once-each
 22 |  [("-b" "--base-folder") folder
 23 |                          "Base dolt folder. Defaults to /var/local/dolt/earnings"
 24 |                          (base-folder folder)]
 25 |  [("-d" "--date") date
 26 |                   "Final date for history retrieval. Defaults to today"
 27 |                   (as-of-date date)]
 28 |  [("-n" "--db-name") name
 29 |                      "Database name. Defaults to 'local'"
 30 |                      (db-name name)]
 31 |  [("-p" "--db-pass") password
 32 |                      "Database password"
 33 |                      (db-pass password)]
 34 |  [("-u" "--db-user") user
 35 |                      "Database user name. Defaults to 'user'"
 36 |                      (db-user user)])
 37 | 
 38 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 39 | 
 40 | ; earnings-calendar
 41 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt sql -q \"delete from earnings_calendar where date >= date_sub('" (as-of-date) "', interval 7 day)\";"))
 42 | 
 43 | (define earnings-calendar-file (string-append (base-folder) "/earnings-calendar-" (as-of-date) ".csv"))
 44 | 
 45 | (call-with-output-file* earnings-calendar-file
 46 |   (λ (out)
 47 |     (displayln "act_symbol,date,when" out)
 48 |     (for-each (λ (row)
 49 |                 (displayln (string-join (vector->list row) ",") out))
 50 |               (query-rows dbc "
 51 | select
 52 |   act_symbol::text,
 53 |   date::text,
 54 |   coalesce(\"when\"::text, '')
 55 | from
 56 |   zacks.earnings_calendar
 57 | where
 58 |   date >= $1::text::date - '7 days'::interval;
 59 | "
 60 |                           (as-of-date))))
 61 |   #:exists 'replace)
 62 | 
 63 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt sql -q \"
 64 | with ecm (act_symbol, max_date, bsa_date) as (
 65 |   select
 66 |     ec.act_symbol,
 67 |     max(ec.date),
 68 |     bsa.date
 69 |   from
 70 |     earnings_calendar ec
 71 |   join
 72 |     (select distinct
 73 |       act_symbol,
 74 |       date
 75 |     from
 76 |       balance_sheet_assets bsa
 77 |     union
 78 |       (select
 79 |         act_symbol,
 80 |         date_sub(date_add(date_add(max(date), interval 1 day), interval 3 month), interval 1 day)
 81 |       from
 82 |         balance_sheet_assets
 83 |       group by
 84 |         act_symbol)
 85 |     order by
 86 |       act_symbol,
 87 |       date) bsa
 88 |   on
 89 |     ec.act_symbol = bsa.act_symbol and
 90 |     ec.date > bsa.date and
 91 |     ec.date <= date_sub(date_add(date_add(bsa.date, interval 1 day), interval 3 month), interval 1 day)
 92 |   group by
 93 |     ec.act_symbol,
 94 |     bsa.date
 95 | )
 96 | delete
 97 |   ec
 98 | from
 99 |   earnings_calendar ec
100 | join
101 |   ecm
102 | where
103 |   ec.act_symbol = ecm.act_symbol and
104 |   ec.date != ecm.max_date and
105 |   ec.date > ecm.bsa_date and
106 |   ec.date <= date_sub(date_add(date_add(ecm.bsa_date, interval 1 day), interval 3 month), interval 1 day);
107 | \""))
108 | 
109 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue earnings_calendar earnings-calendar-" (as-of-date) ".csv"))
110 | 
111 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add earnings_calendar; "
112 |                        "/usr/local/bin/dolt commit -m 'earnings_calendar " (as-of-date) " update'; /usr/local/bin/dolt push --silent"))
113 | 


--------------------------------------------------------------------------------
/dump-dat.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          racket/cmdline
  6 |          racket/vector)
  7 | 
  8 | (define start-date (make-parameter (~t (today) "yyyy-MM-dd")))
  9 | 
 10 | (define end-date (make-parameter (~t (today) "yyyy-MM-dd")))
 11 | 
 12 | (define db-user (make-parameter "user"))
 13 | 
 14 | (define db-name (make-parameter "local"))
 15 | 
 16 | (define db-pass (make-parameter ""))
 17 | 
 18 | (command-line
 19 |  #:program "racket dump-dat.rkt"
 20 |  #:once-each
 21 |  [("-e" "--end-date") end
 22 |                       "Final date for history retrieval. Defaults to today"
 23 |                       (end-date end)]
 24 |  [("-n" "--db-name") name
 25 |                      "Database name. Defaults to 'local'"
 26 |                      (db-name name)]
 27 |  [("-p" "--db-pass") password
 28 |                      "Database password"
 29 |                      (db-pass password)]
 30 |  [("-s" "--start-date") start
 31 |                         "Earliest date for history retrieval. Defaults to today"
 32 |                         (start-date start)]
 33 |  [("-u" "--db-user") user
 34 |                      "Database user name. Defaults to 'user'"
 35 |                      (db-user user)])
 36 | 
 37 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 38 | 
 39 | (define (vector->csv-line vec)
 40 |   (if (= 1 (vector-length vec))
 41 |       (vector-ref vec 0)
 42 |       (string-append (vector-ref vec 0) "," (vector->csv-line (vector-drop vec 1)))))
 43 | 
 44 | (for-each (λ (date)
 45 |             (call-with-output-file (string-append "/var/local/dat/zacks/eps-estimate/" date ".csv")
 46 |               (λ (out)
 47 |                 (displayln "act_symbol,date,period,period_end_date,consensus,recent,count,high,low,year_ago" out)
 48 |                 (for-each (λ (row)
 49 |                             (displayln (vector->csv-line row) out))
 50 |                           (query-rows dbc "
 51 | select
 52 |   act_symbol::text,
 53 |   date::text,
 54 |   period::text,
 55 |   period_end_date::text,
 56 |   consensus::text,
 57 |   recent::text,
 58 |   count::text,
 59 |   high::text,
 60 |   low::text,
 61 |   year_ago::text
 62 | from
 63 |   zacks.eps_estimate
 64 | where
 65 |   date = $1::text::date and
 66 |   consensus is not null and
 67 |   recent is not null and
 68 |   count is not null and
 69 |   high is not null and
 70 |   low is not null and
 71 |   year_ago is not null
 72 | order by
 73 |   act_symbol, date, period, period_end_date
 74 | "
 75 |                                       date)))
 76 |               #:exists 'replace))
 77 |           (query-list dbc "
 78 | select distinct
 79 |   date::text
 80 | from
 81 |   zacks.eps_estimate
 82 | where
 83 |   date >= $1::text::date and
 84 |   date <= $2::text::date
 85 | order by
 86 |   date;
 87 | "
 88 |                       (start-date)
 89 |                       (end-date)))
 90 | 
 91 | (for-each (λ (date)
 92 |             (call-with-output-file (string-append "/var/local/dat/zacks/sales-estimate/" date ".csv")
 93 |               (λ (out)
 94 |                 (displayln "act_symbol,date,period,period_end_date,consensus,count,high,low,year_ago" out)
 95 |                 (for-each (λ (row)
 96 |                             (displayln (vector->csv-line row) out))
 97 |                           (query-rows dbc "
 98 | select
 99 |   act_symbol::text,
100 |   date::text,
101 |   period::text,
102 |   period_end_date::text,
103 |   consensus::text,
104 |   count::text,
105 |   high::text,
106 |   low::text,
107 |   year_ago::text
108 | from
109 |   zacks.sales_estimate
110 | where
111 |   date = $1::text::date and
112 |   consensus is not null and
113 |   count is not null and
114 |   high is not null and
115 |   low is not null and
116 |   year_ago is not null
117 | order by
118 |   act_symbol, date, period, period_end_date
119 | "
120 |                                       date)))
121 |               #:exists 'replace))
122 |           (query-list dbc "
123 | select distinct
124 |   date::text
125 | from
126 |   zacks.sales_estimate
127 | where
128 |   date >= $1::text::date and
129 |   date <= $2::text::date
130 | order by
131 |   date;
132 | "
133 |                       (start-date)
134 |                       (end-date)))
135 | 
136 | 


--------------------------------------------------------------------------------
/dividend-calendar-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          json
  6 |          racket/cmdline
  7 |          racket/list
  8 |          racket/port
  9 |          racket/sequence
 10 |          racket/string
 11 |          threading)
 12 | 
 13 | (define base-folder (make-parameter "/var/local/zacks/dividend-calendar"))
 14 | 
 15 | (define folder-date (make-parameter (today)))
 16 | 
 17 | (define db-user (make-parameter "user"))
 18 | 
 19 | (define db-name (make-parameter "local"))
 20 | 
 21 | (define db-pass (make-parameter ""))
 22 | 
 23 | (command-line
 24 |  #:program "racket transform-load.rkt"
 25 |  #:once-each
 26 |  [("-b" "--base-folder") folder
 27 |                          "Dividend Calendar base folder. Defaults to /var/local/zacks/dividend-calendar"
 28 |                          (base-folder folder)]
 29 |  [("-d" "--folder-date") date
 30 |                          "Dividend Calendar folder date. Defaults to today"
 31 |                          (folder-date (iso8601->date date))]
 32 |  [("-n" "--db-name") name
 33 |                      "Database name. Defaults to 'local'"
 34 |                      (db-name name)]
 35 |  [("-p" "--db-pass") password
 36 |                      "Database password"
 37 |                      (db-pass password)]
 38 |  [("-u" "--db-user") user
 39 |                      "Database user name. Defaults to 'user'"
 40 |                      (db-user user)])
 41 | 
 42 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 43 | 
 44 | ; we clean up the future part of the table in case dividend dates have been shifted
 45 | (query-exec dbc "
 46 | delete from
 47 |   zacks.dividend_calendar
 48 | where
 49 |   ex_date >= $1::text::date;
 50 | "
 51 |             (~t (folder-date) "yyyy-MM-dd"))
 52 | 
 53 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
 54 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".json")) (in-directory (current-directory)))])
 55 |     (let* ([file-name (path->string p)]
 56 |            [date-of-dividend (string-replace (string-replace file-name (path->string (current-directory)) "") ".json" "")])
 57 |       (call-with-input-file file-name
 58 |         (λ (in)
 59 |           (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to parse "
 60 |                                                                       file-name
 61 |                                                                       " for date "
 62 |                                                                       date-of-dividend))
 63 |                                        (displayln e))])
 64 |             (~> (port->string in)
 65 |                 (regexp-replace* #rx"<.*?>" _ "")
 66 |                 (regexp-replace* #rx"[A-Z\\.]+ Quick Quote" _ "")
 67 |                 (string-replace _ "window.app_data = " "")
 68 |                 (string->jsexpr _)
 69 |                 (hash-ref _ 'data)
 70 |                 (for-each (λ (ticker-div-list)
 71 |                             (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to insert "
 72 |                                                                                         (first ticker-div-list)
 73 |                                                                                         " for date "
 74 |                                                                                         date-of-dividend))
 75 |                                                          (displayln e)
 76 |                                                          (rollback-transaction dbc))])
 77 |                               (start-transaction dbc)
 78 |                               ; if we have a record from last week for this symbol, move it forward
 79 |                               (query-exec dbc "
 80 | delete from
 81 |   zacks.dividend_calendar
 82 | where
 83 |   act_symbol = $1 and
 84 |   ex_date >= $2::text::date - '7 days'::interval
 85 | "
 86 |                                           (first ticker-div-list)
 87 |                                           (~t (folder-date) "yyyy-MM-dd"))
 88 |                               (query-exec dbc "
 89 | insert into zacks.dividend_calendar (
 90 |   act_symbol,
 91 |   ex_date,
 92 |   amount,
 93 |   payable_date
 94 | ) values (
 95 |   $1,
 96 |   $2::text::date,
 97 |   $3::text::decimal,
 98 |   case
 99 |     when $4 = '--' then null
100 |     else $4::text::date
101 |   end
102 | ) on conflict do nothing;
103 | "
104 |                                           (first ticker-div-list)
105 |                                           (sixth ticker-div-list)
106 |                                           (string-replace (fourth ticker-div-list) "$" "")
107 |                                           (eighth ticker-div-list))
108 |                               (commit-transaction dbc))) _))))))))
109 | 
110 | ; vacuum (garbage collect) and reindex table as we deleted from it earlier
111 | (query-exec dbc "
112 | vacuum full freeze analyze zacks.dividend_calendar;
113 | ")
114 | 
115 | (query-exec dbc "
116 | reindex table zacks.dividend_calendar;
117 | ")
118 | 
119 | (disconnect dbc)
120 | 


--------------------------------------------------------------------------------
/financial-statement-extract.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          net/http-easy
  6 |          racket/cmdline
  7 |          racket/file
  8 |          racket/list
  9 |          racket/port
 10 |          tasks
 11 |          threading)
 12 | 
 13 | (define (download-income-statement symbol)
 14 |   (make-directory* (string-append "/var/local/zacks/income-statement/" (~t (today) "yyyy-MM-dd")))
 15 |   (with-handlers ([exn:fail?
 16 |                    (λ (error)
 17 |                      (displayln (string-append "Encountered error for " symbol))
 18 |                      (displayln error))])
 19 |     (call-with-output-file* (string-append "/var/local/zacks/income-statement/" (~t (today) "yyyy-MM-dd") "/" symbol ".income-statement.html")
 20 |       (λ (out)
 21 |         (~> (string-append "https://www.zacks.com/stock/quote/" symbol "/income-statement")
 22 |             (get _)
 23 |             (response-body _)
 24 |             (write-bytes _ out)))
 25 |       #:exists 'replace)))
 26 | 
 27 | (define (download-balance-sheet symbol)
 28 |   (make-directory* (string-append "/var/local/zacks/balance-sheet/" (~t (today) "yyyy-MM-dd")))
 29 |   (with-handlers ([exn:fail?
 30 |                    (λ (error)
 31 |                      (displayln (string-append "Encountered error for " symbol))
 32 |                      (displayln error))])
 33 |     (call-with-output-file* (string-append "/var/local/zacks/balance-sheet/" (~t (today) "yyyy-MM-dd") "/" symbol ".balance-sheet.html")
 34 |       (λ (out)
 35 |         (~> (string-append "https://www.zacks.com/stock/quote/" symbol "/balance-sheet")
 36 |             (get _)
 37 |             (response-body _)
 38 |             (write-bytes _ out)))
 39 |       #:exists 'replace)))
 40 | 
 41 | (define (download-cash-flow-statement symbol)
 42 |   (make-directory* (string-append "/var/local/zacks/cash-flow-statement/" (~t (today) "yyyy-MM-dd")))
 43 |   (with-handlers ([exn:fail?
 44 |                    (λ (error)
 45 |                      (displayln (string-append "Encountered error for " symbol))
 46 |                      (displayln error))])
 47 |     (call-with-output-file* (string-append "/var/local/zacks/cash-flow-statement/" (~t (today) "yyyy-MM-dd") "/" symbol ".cash-flow-statement.html")
 48 |       (λ (out)
 49 |         (~> (string-append "https://www.zacks.com/stock/quote/" symbol "/cash-flow-statements")
 50 |             (get _)
 51 |             (response-body _)
 52 |             (write-bytes _ out)))
 53 |       #:exists 'replace)))
 54 | 
 55 | (define db-user (make-parameter "user"))
 56 | 
 57 | (define db-name (make-parameter "local"))
 58 | 
 59 | (define db-pass (make-parameter ""))
 60 | 
 61 | (define first-symbol (make-parameter ""))
 62 | 
 63 | (define last-symbol (make-parameter ""))
 64 | 
 65 | (command-line
 66 |  #:program "racket financial-statement-extract.rkt"
 67 |  #:once-each
 68 |  [("-f" "--first-symbol") first
 69 |                           "First symbol to query. Defaults to nothing"
 70 |                           (first-symbol first)]
 71 |  [("-l" "--last-symbol") last
 72 |                          "Last symbol to query. Defaults to nothing"
 73 |                          (last-symbol last)]
 74 |  [("-n" "--db-name") name
 75 |                      "Database name. Defaults to 'local'"
 76 |                      (db-name name)]
 77 |  [("-p" "--db-pass") password
 78 |                      "Database password"
 79 |                      (db-pass password)]
 80 |  [("-u" "--db-user") user
 81 |                      "Database user name. Defaults to 'user'"
 82 |                      (db-user user)])
 83 | 
 84 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 85 | 
 86 | (define symbols (query-list dbc "
 87 | select
 88 |   act_symbol
 89 | from
 90 |   nasdaq.symbol
 91 | where
 92 |   is_etf = false and
 93 |   is_test_issue = false and
 94 |   is_next_shares = false and
 95 |   security_name !~ 'ETN' and
 96 |   nasdaq_symbol !~ '[-\\$\\+\\*#!@%\\^=~]' and
 97 |   case when nasdaq_symbol ~ '[A-Z]{4}[L-Z]'
 98 |     then security_name !~ '(Note|Preferred|Right|Unit|Warrant)'
 99 |     else true
100 |   end and
101 |   last_seen = (select max(last_seen) from nasdaq.symbol) and
102 |   case when $1 != ''
103 |     then act_symbol >= $1
104 |     else true
105 |   end and
106 |   case when $2 != ''
107 |     then act_symbol <= $2
108 |     else true
109 |   end
110 | order by
111 |   act_symbol;
112 | "
113 |                             (first-symbol)
114 |                             (last-symbol)))
115 | 
116 | (disconnect dbc)
117 | 
118 | (define delay-interval 12)
119 | 
120 | (define delays (map (λ (x) (* delay-interval x)) (range 0 (length symbols))))
121 | 
122 | (with-task-server (for-each (λ (l) (schedule-delayed-task (λ () (thread (λ () (download-income-statement (first l)))))
123 |                                                           (second l))
124 |                               (schedule-delayed-task (λ () (thread (λ () (download-balance-sheet (first l)))))
125 |                                                      (+ 4 (second l)))
126 |                               (schedule-delayed-task (λ () (thread (λ () (download-cash-flow-statement (first l)))))
127 |                                                      (+ 8 (second l))))
128 |                             (map list symbols delays))
129 |   ; add a final task that will halt the task server
130 |   (schedule-delayed-task (λ () (schedule-stop-task)) (* delay-interval (length delays)))
131 |   (run-tasks))
132 | 


--------------------------------------------------------------------------------
/earnings-calendar-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          json
  6 |          racket/cmdline
  7 |          racket/list
  8 |          racket/port
  9 |          racket/sequence
 10 |          racket/string
 11 |          threading)
 12 | 
 13 | (define base-folder (make-parameter "/var/local/zacks/earnings-calendar"))
 14 | 
 15 | (define folder-date (make-parameter (today)))
 16 | 
 17 | (define db-user (make-parameter "user"))
 18 | 
 19 | (define db-name (make-parameter "local"))
 20 | 
 21 | (define db-pass (make-parameter ""))
 22 | 
 23 | (command-line
 24 |  #:program "racket transform-load.rkt"
 25 |  #:once-each
 26 |  [("-b" "--base-folder") folder
 27 |                          "Earnings Calendar base folder. Defaults to /var/local/zacks/earnings-calendar"
 28 |                          (base-folder folder)]
 29 |  [("-d" "--folder-date") date
 30 |                          "Earnings Calendar folder date. Defaults to today"
 31 |                          (folder-date (iso8601->date date))]
 32 |  [("-n" "--db-name") name
 33 |                      "Database name. Defaults to 'local'"
 34 |                      (db-name name)]
 35 |  [("-p" "--db-pass") password
 36 |                      "Database password"
 37 |                      (db-pass password)]
 38 |  [("-u" "--db-user") user
 39 |                      "Database user name. Defaults to 'user'"
 40 |                      (db-user user)])
 41 | 
 42 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 43 | 
 44 | ; we clean up the future part of the table in case earnings dates have been shifted
 45 | (query-exec dbc "
 46 | delete from
 47 |   zacks.earnings_calendar
 48 | where
 49 |   date >= $1::text::date;
 50 | "
 51 |             (~t (folder-date) "yyyy-MM-dd"))
 52 | 
 53 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
 54 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".json")) (in-directory (current-directory)))])
 55 |     (let* ([file-name (path->string p)]
 56 |            [date-of-earnings (string-replace (string-replace file-name (path->string (current-directory)) "") ".json" "")])
 57 |       (call-with-input-file file-name
 58 |         (λ (in)
 59 |           (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to parse "
 60 |                                                                       file-name
 61 |                                                                       " for date "
 62 |                                                                       date-of-earnings))
 63 |                                        (displayln e))])
 64 |             (~> (port->string in)
 65 |                 (regexp-replace* #rx"<.*?>" _ "")
 66 |                 (regexp-replace* #rx"[A-Z\\.]+ Quick Quote" _ "")
 67 |                 (string-replace _ "window.app_data = " "")
 68 |                 (string->jsexpr _)
 69 |                 (hash-ref _ 'data)
 70 |                 (for-each (λ (ticker-when-list)
 71 |                             (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to insert "
 72 |                                                                                         (first ticker-when-list)
 73 |                                                                                         " for date "
 74 |                                                                                         date-of-earnings))
 75 |                                                          (displayln e)
 76 |                                                          (rollback-transaction dbc))])
 77 |                               (start-transaction dbc)
 78 |                               ; if we have a record from last week for this symbol, move it forward
 79 |                               (query-exec dbc "
 80 | delete from
 81 |   zacks.earnings_calendar
 82 | where
 83 |   act_symbol = $1 and
 84 |   date >= $2::text::date - '7 days'::interval
 85 | "
 86 |                                           (first ticker-when-list)
 87 |                                           (~t (folder-date) "yyyy-MM-dd"))
 88 |                               (query-exec dbc "
 89 | insert into zacks.earnings_calendar (
 90 |   act_symbol,
 91 |   date,
 92 |   \"when\"
 93 | ) values (
 94 |   $1,
 95 |   $2::text::date,
 96 |   case $3
 97 |     when 'amc' then 'After market close'::zacks.when
 98 |     when 'bmo' then 'Before market open'::zacks.when
 99 |     when '--' then NULL
100 |   end
101 | ) on conflict do nothing;
102 | "
103 |                                           (first ticker-when-list)
104 |                                           date-of-earnings
105 |                                           (fourth ticker-when-list))
106 |                               (commit-transaction dbc))) _))))))))
107 | 
108 | ; remove estimated dates when the estimate moves
109 | (query-exec dbc "
110 | delete from
111 |   zacks.earnings_calendar ec
112 | using
113 |   (select
114 |     ec.act_symbol,
115 |     max(ec.date) as max_date,
116 |     bsa.date as bsa_date
117 |   from
118 |     zacks.earnings_calendar ec
119 |   join
120 |     (select distinct
121 |       act_symbol,
122 |       date
123 |     from
124 |       zacks.balance_sheet_assets bsa
125 |     union
126 |       (select
127 |         act_symbol,
128 |         (((max(date) + '1 day'::interval) + '3 months'::interval) - '1 day'::interval)::date
129 |       from
130 |         zacks.balance_sheet_assets
131 |       group by
132 |         act_symbol)
133 |     order by
134 |       act_symbol,
135 |       date) bsa
136 |   on
137 |     ec.act_symbol = bsa.act_symbol and
138 |     ec.date > bsa.date and
139 |     ec.date <= ((bsa.date + '1 day'::interval) + '3 months'::interval) - '1 day'::interval
140 |   group by
141 |     ec.act_symbol,
142 |     bsa.date) ecm
143 | where
144 |   ec.act_symbol = ecm.act_symbol and
145 |   ec.date != max_date and
146 |   ec.date > bsa_date and
147 |   ec.date <= ((bsa_date + '1 day'::interval) + '3 months'::interval) - '1 day'::interval;
148 | ")
149 | 
150 | ; vacuum (garbage collect) and reindex table as we deleted from it earlier
151 | (query-exec dbc "
152 | vacuum full freeze analyze zacks.earnings_calendar;
153 | ")
154 | 
155 | (query-exec dbc "
156 | reindex table zacks.earnings_calendar;
157 | ")
158 | 
159 | (disconnect dbc)
160 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # zacks-estimates-financial-statements
  2 | 
  3 | These Racket programs will download the Zacks "Detailed Estimates" and "Financials" HTML documents and insert the 
  4 | estimates/statement data into a PostgreSQL database. The intended usage is:
  5 | 
  6 | ```bash
  7 | $ racket estimate-extract.rkt
  8 | $ racket estimate-transform-load.rkt
  9 | ```
 10 | 
 11 | ```bash
 12 | $ racket financial-statement-extract.rkt
 13 | $ racket balance-sheet-transform-load.rkt
 14 | $ racket cash-flow-statement-transform-load.rkt
 15 | $ racket income-statement-transform-load.rkt
 16 | ```
 17 | 
 18 | ```bash
 19 | $ racket earnings-calendar-extract.rkt
 20 | $ racket earnings-calendar-transform-load.rkt
 21 | ```
 22 | 
 23 | You will need to provide a database password for many of the above programs. The available parameters are:
 24 | 
 25 | ```bash
 26 | $ racket estimate-extract.rkt -h
 27 | racket estimate-extract.rkt [ <option> ... ]
 28 |  where <option> is one of
 29 |   -f <first>, --first-symbol <first> : First symbol to query. Defaults to nothing
 30 |   -l <last>, --last-symbol <last> : Last symbol to query. Defaults to nothing
 31 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 32 |   -p <password>, --db-pass <password> : Database password
 33 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 34 |   --help, -h : Show this help
 35 |   -- : Do not treat any remaining argument as a switch (at this level)
 36 |  Multiple single-letter switches can be combined after one `-`. For
 37 |   example: `-h-` is the same as `-h --`
 38 | 
 39 | $ racket estimate-transform-load.rkt -h
 40 | racket estimate-transform-load.rkt [ <option> ... ]
 41 |  where <option> is one of
 42 |   -b <folder>, --base-folder <folder> : Zacks estimates base folder. Defaults to /var/local/zacks/estimates
 43 |   -d <date>, --folder-date <date> : Zacks estimates folder date. Defaults to today
 44 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 45 |   -p <password>, --db-pass <password> : Database password
 46 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 47 |   --help, -h : Show this help
 48 |   -- : Do not treat any remaining argument as a switch (at this level)
 49 |  Multiple single-letter switches can be combined after one `-`. For
 50 |   example: `-h-` is the same as `-h --`
 51 | 
 52 | $ racket financial-statement-extract.rkt -h
 53 | racket financial-statement-extract.rkt [ <option> ... ]
 54 |  where <option> is one of
 55 |   -f <first>, --first-symbol <first> : First symbol to query. Defaults to nothing
 56 |   -l <last>, --last-symbol <last> : Last symbol to query. Defaults to nothing
 57 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 58 |   -p <password>, --db-pass <password> : Database password
 59 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 60 |   --help, -h : Show this help
 61 |   -- : Do not treat any remaining argument as a switch (at this level)
 62 |  Multiple single-letter switches can be combined after one `-`. For
 63 |   example: `-h-` is the same as `-h --`
 64 | 
 65 | $ racket balance-sheet-transform-load.rkt -h
 66 | racket balance-sheet-transform-load.rkt [ <option> ... ]
 67 |  where <option> is one of
 68 |   -b <folder>, --base-folder <folder> : Zacks balance sheet base folder. Defaults to /var/local/zacks/balance-sheet
 69 |   -d <date>, --folder-date <date> : Zacks balance sheet folder date. Defaults to today
 70 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 71 |   -p <password>, --db-pass <password> : Database password
 72 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 73 |   --help, -h : Show this help
 74 |   -- : Do not treat any remaining argument as a switch (at this level)
 75 |  Multiple single-letter switches can be combined after one `-`. For
 76 |   example: `-h-` is the same as `-h --`
 77 | 
 78 | $ racket cash-flow-statement-transform-load.rkt -h
 79 | racket cash-flow-statement-transform-load.rkt [ <option> ... ]
 80 |  where <option> is one of
 81 |   -b <folder>, --base-folder <folder> : Zacks cash flow statement base folder. Defaults to /var/local/zacks/cash-flow-statement
 82 |   -d <date>, --folder-date <date> : Zacks cash flow statement folder date. Defaults to today
 83 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 84 |   -p <password>, --db-pass <password> : Database password
 85 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 86 |   --help, -h : Show this help
 87 |   -- : Do not treat any remaining argument as a switch (at this level)
 88 |  Multiple single-letter switches can be combined after one `-`. For
 89 |   example: `-h-` is the same as `-h --`
 90 | 
 91 | $ racket income-statement-transform-load.rkt -h
 92 | racket income-statement-transform-load.rkt [ <option> ... ]
 93 |  where <option> is one of
 94 |   -b <folder>, --base-folder <folder> : Zacks income statement base folder. Defaults to /var/local/zacks/income-statement
 95 |   -d <date>, --folder-date <date> : Zacks income statement folder date. Defaults to today
 96 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
 97 |   -p <password>, --db-pass <password> : Database password
 98 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
 99 |   --help, -h : Show this help
100 |   -- : Do not treat any remaining argument as a switch (at this level)
101 |  Multiple single-letter switches can be combined after one `-`. For
102 |   example: `-h-` is the same as `-h --`
103 | 
104 | $ racket earnings-calendar-extract.rkt -h
105 | racket earnings-calendar-extract.rkt [ <option> ... ]
106 |  where <option> is one of
107 |   -e <ed>, --end-date <ed> : End date. Defaults to today + 6 weeks
108 |   -s <sd>, --start-date <sd> : Start date. Defaults to today
109 |   --help, -h : Show this help
110 |   -- : Do not treat any remaining argument as a switch (at this level)
111 |  Multiple single-letter switches can be combined after one `-`. For
112 |   example: `-h-` is the same as `-h --`
113 | 
114 | $ racket earnings-calendar-transform-load.rkt -h
115 | racket transform-load.rkt [ <option> ... ]
116 |  where <option> is one of
117 |   -b <folder>, --base-folder <folder> : Earnings Calendar base folder. Defaults to /var/local/zacks/earnings-calendar
118 |   -d <date>, --folder-date <date> : Earnings Calendar folder date. Defaults to today
119 |   -n <name>, --db-name <name> : Database name. Defaults to 'local'
120 |   -p <password>, --db-pass <password> : Database password
121 |   -u <user>, --db-user <user> : Database user name. Defaults to 'user'
122 |   --help, -h : Show this help
123 |   -- : Do not treat any remaining argument as a switch (at this level)
124 |  Multiple single-letter switches can be combined after one `-`. For
125 |   example: `-h-` is the same as `-h --`
126 | ```
127 | 
128 | The provided `schema.sql` file shows the expected schema within the target PostgreSQL instance. 
129 | This process assumes you can write to a `/var/local/zacks` folder. This process also assumes you have loaded your database with NASDAQ symbol
130 | file information. This data is provided by the [nasdaq-symbols](https://github.com/evdubs/nasdaq-symbols) project.
131 | 
132 | The above process will download around 2GB worth of HTML documents over many hours. It is encouraged to compress these files when you are 
133 | done processing them. It is also encouraged that you do not run the extract jobs too frequently. I think running the estimate-extract 
134 | once per week and the financial-statement-extract once per month is sufficient.
135 | 
136 | ### Dependencies
137 | 
138 | It is recommended that you start with the standard Racket distribution. With that, you will need to install the following packages:
139 | 
140 | ```bash
141 | $ raco pkg install --skip-installed gregor html-parsing sxml tasks threading
142 | ```
143 | 


--------------------------------------------------------------------------------
/dump-dolt-estimates.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          racket/cmdline
  6 |          racket/string
  7 |          racket/system)
  8 | 
  9 | (define base-folder (make-parameter "/var/local/dolt/earnings"))
 10 | 
 11 | (define start-date (make-parameter (~t (today) "yyyy-MM-dd")))
 12 | 
 13 | (define end-date (make-parameter (~t (today) "yyyy-MM-dd")))
 14 | 
 15 | (define db-user (make-parameter "user"))
 16 | 
 17 | (define db-name (make-parameter "local"))
 18 | 
 19 | (define db-pass (make-parameter ""))
 20 | 
 21 | (command-line
 22 |  #:program "racket dump-dolt-estimates.rkt"
 23 |  #:once-each
 24 |  [("-b" "--base-folder") folder
 25 |                          "Base dolt folder. Defaults to /var/local/dolt/earnings"
 26 |                          (base-folder folder)]
 27 |  [("-e" "--end-date") end
 28 |                       "Final date for history retrieval. Defaults to today"
 29 |                       (end-date end)]
 30 |  [("-n" "--db-name") name
 31 |                      "Database name. Defaults to 'local'"
 32 |                      (db-name name)]
 33 |  [("-p" "--db-pass") password
 34 |                      "Database password"
 35 |                      (db-pass password)]
 36 |  [("-s" "--start-date") start
 37 |                         "Earliest date for history retrieval. Defaults to today"
 38 |                         (start-date start)]
 39 |  [("-u" "--db-user") user
 40 |                      "Database user name. Defaults to 'user'"
 41 |                      (db-user user)])
 42 | 
 43 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 44 | 
 45 | ; rank-score
 46 | (for-each (λ (date)
 47 |             (define rank-score-file (string-append (base-folder) "/rank-score-" date ".csv"))
 48 |             (call-with-output-file* rank-score-file
 49 |               (λ (out)
 50 |                 (displayln "date,act_symbol,rank,value,growth,momentum,vgm" out)
 51 |                 (for-each (λ (row)
 52 |                             (displayln (string-join (vector->list row) ",") out))
 53 |                           (query-rows dbc "
 54 | select
 55 |   date::text,
 56 |   act_symbol::text,
 57 |   rank::text,
 58 |   value::text,
 59 |   growth::text,
 60 |   momentum::text,
 61 |   vgm::text
 62 | from
 63 |   zacks.rank_score
 64 | where
 65 |   date = $1::text::date
 66 | order by
 67 |   act_symbol, date;
 68 | "
 69 |                                       date)))
 70 |               #:exists 'replace)
 71 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue rank_score rank-score-" date ".csv")))
 72 |           (query-list dbc "
 73 | select distinct
 74 |   date::text
 75 | from
 76 |   zacks.rank_score
 77 | where
 78 |   date >= $1::text::date and
 79 |   date <= $2::text::date
 80 | order by
 81 |   date;
 82 | "
 83 |                       (start-date)
 84 |                       (end-date)))
 85 | 
 86 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add rank_score; "
 87 |                        "/usr/local/bin/dolt commit -m 'rank_score " (end-date) " update'; /usr/local/bin/dolt push --silent"))
 88 | 
 89 | ; eps-estimate
 90 | (for-each (λ (date)
 91 |             (define eps-estimate-file (string-append (base-folder) "/eps-estimate-" date ".csv"))
 92 |             (call-with-output-file* eps-estimate-file
 93 |               (λ (out)
 94 |                 (displayln "date,act_symbol,period,period_end_date,consensus,recent,count,high,low,year_ago" out)
 95 |                 (for-each (λ (row)
 96 |                             (displayln (string-join (vector->list row) ",") out))
 97 |                           (query-rows dbc "
 98 | select
 99 |   date::text,
100 |   act_symbol::text,
101 |   period::text,
102 |   period_end_date::text,
103 |   coalesce(consensus::text, ''),
104 |   coalesce(recent::text, ''),
105 |   coalesce(count::text, ''),
106 |   coalesce(high::text, ''),
107 |   coalesce(low::text, ''),
108 |   coalesce(year_ago::text, '')
109 | from
110 |   zacks.eps_estimate
111 | where
112 |   date = $1::text::date
113 | order by
114 |   act_symbol, date, period;
115 | "
116 |                                       date)))
117 |               #:exists 'replace)
118 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue eps_estimate eps-estimate-" date ".csv")))
119 |           (query-list dbc "
120 | select distinct
121 |   date::text
122 | from
123 |   zacks.eps_estimate
124 | where
125 |   date >= $1::text::date and
126 |   date <= $2::text::date
127 | order by
128 |   date;
129 | "
130 |                       (start-date)
131 |                       (end-date)))
132 | 
133 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add eps_estimate; "
134 |                        "/usr/local/bin/dolt commit -m 'eps_estimate " (end-date) " update'; /usr/local/bin/dolt push --silent"))
135 | 
136 | ; sales-estimate
137 | (for-each (λ (date)
138 |             (define sales-estimate-file (string-append (base-folder) "/sales-estimate-" date ".csv"))
139 |             (call-with-output-file* sales-estimate-file
140 |               (λ (out)
141 |                 (displayln "date,act_symbol,period,period_end_date,consensus,count,high,low,year_ago" out)
142 |                 (for-each (λ (row)
143 |                             (displayln (string-join (vector->list row) ",") out))
144 |                           (query-rows dbc "
145 | select
146 |   date::text,
147 |   act_symbol::text,
148 |   period::text,
149 |   period_end_date::text,
150 |   coalesce(consensus::text, ''),
151 |   coalesce(count::text, ''),
152 |   coalesce(high::text, ''),
153 |   coalesce(low::text, ''),
154 |   coalesce(year_ago::text, '')
155 | from
156 |   zacks.sales_estimate
157 | where
158 |   date = $1::text::date
159 | order by
160 |   act_symbol, date, period;
161 | "
162 |                                       date)))
163 |               #:exists 'replace)
164 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue sales_estimate sales-estimate-" date ".csv")))
165 |           (query-list dbc "
166 | select distinct
167 |   date::text
168 | from
169 |   zacks.sales_estimate
170 | where
171 |   date >= $1::text::date and
172 |   date <= $2::text::date
173 | order by
174 |   date;
175 | "
176 |                       (start-date)
177 |                       (end-date)))
178 | 
179 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add sales_estimate; "
180 |                        "/usr/local/bin/dolt commit -m 'sales_estimate " (end-date) " update'; /usr/local/bin/dolt push --silent"))
181 | 
182 | ; eps-history
183 | (for-each (λ (date)
184 |             (define eps-history-file (string-append (base-folder) "/eps-history-" date ".csv"))
185 |             (call-with-output-file* eps-history-file
186 |               (λ (out)
187 |                 (displayln "act_symbol,period_end_date,reported,estimate" out)
188 |                 (for-each (λ (row)
189 |                             (displayln (string-join (vector->list row) ",") out))
190 |                           (query-rows dbc "
191 | select distinct
192 |   act_symbol::text,
193 |   period_end_date::text,
194 |   coalesce(reported::text, ''),
195 |   coalesce(estimate::text, '')
196 | from
197 |   zacks.eps_history
198 | where
199 |   period_end_date = $1::text::date
200 | order by
201 |   act_symbol, period_end_date;
202 | "
203 |                                       date)))
204 |               #:exists 'replace)
205 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue eps_history eps-history-" date ".csv")))
206 |           (query-list dbc "
207 | select distinct
208 |   period_end_date::text
209 | from
210 |   zacks.eps_history
211 | where
212 |   period_end_date >= ($1::text::date - '6 months'::interval) and
213 |   period_end_date <= $2::text::date
214 | order by
215 |   period_end_date;
216 | "
217 |                       (start-date)
218 |                       (end-date)))
219 | 
220 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add eps_history; "
221 |                        "/usr/local/bin/dolt commit -m 'eps_history " (end-date) " update'; /usr/local/bin/dolt push --silent"))
222 | 


--------------------------------------------------------------------------------
/schema.sql:
--------------------------------------------------------------------------------
  1 | CREATE SCHEMA zacks;
  2 | 
  3 | CREATE TYPE zacks.estimate_period AS ENUM
  4 |     ('Current Quarter', 'Next Quarter', 'Current Year', 'Next Year');
  5 | 	
  6 | CREATE TYPE zacks.rank AS ENUM
  7 |     ('Strong Buy', 'Buy', 'Hold', 'Sell', 'Strong Sell');
  8 | 	
  9 | CREATE TYPE zacks.score AS ENUM
 10 |     ('A', 'B', 'C', 'D', 'F');
 11 | 	
 12 | CREATE TYPE zacks.statement_period AS ENUM
 13 |     ('Year', 'Quarter');
 14 | 
 15 | CREATE TYPE zacks."when" AS ENUM
 16 |     ('Before market open', 'After market close');
 17 | 
 18 | CREATE TABLE zacks.balance_sheet_assets
 19 | (
 20 |     act_symbol text NOT NULL,
 21 |     date date NOT NULL,
 22 |     period zacks.statement_period NOT NULL,
 23 |     cash_and_equivalents numeric,
 24 |     receivables numeric,
 25 |     notes_receivable numeric,
 26 |     inventories numeric,
 27 |     other_current_assets numeric,
 28 |     total_current_assets numeric,
 29 |     net_property_and_equipment numeric,
 30 |     investments_and_advances numeric,
 31 |     other_non_current_assets numeric,
 32 |     deferred_charges numeric,
 33 |     intangibles numeric,
 34 |     deposits_and_other_assets numeric,
 35 |     total_assets numeric,
 36 |     CONSTRAINT balance_sheet_assets_pkey PRIMARY KEY (act_symbol, date, period),
 37 |     CONSTRAINT balance_sheet_assets_act_symbol_fkey FOREIGN KEY (act_symbol)
 38 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
 39 |         ON UPDATE NO ACTION
 40 |         ON DELETE NO ACTION
 41 | );
 42 | 
 43 | CREATE TABLE zacks.balance_sheet_equity
 44 | (
 45 |     act_symbol text NOT NULL,
 46 |     date date NOT NULL,
 47 |     period zacks.statement_period NOT NULL,
 48 |     preferred_stock numeric,
 49 |     common_stock numeric,
 50 |     capital_surplus numeric,
 51 |     retained_earnings numeric,
 52 |     other_equity numeric,
 53 |     treasury_stock numeric,
 54 |     total_equity numeric,
 55 |     total_liabilities_and_equity numeric,
 56 |     shares_outstanding numeric,
 57 |     book_value_per_share numeric,
 58 |     CONSTRAINT balance_sheet_equity_pkey PRIMARY KEY (act_symbol, date, period),
 59 |     CONSTRAINT balance_sheet_equity_act_symbol_fkey FOREIGN KEY (act_symbol)
 60 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
 61 |         ON UPDATE NO ACTION
 62 |         ON DELETE NO ACTION
 63 | );
 64 | 
 65 | CREATE TABLE zacks.balance_sheet_liabilities
 66 | (
 67 |     act_symbol text NOT NULL,
 68 |     date date NOT NULL,
 69 |     period zacks.statement_period NOT NULL,
 70 |     notes_payable numeric,
 71 |     accounts_payable numeric,
 72 |     current_portion_long_term_debt numeric,
 73 |     current_portion_capital_leases numeric,
 74 |     accrued_expenses numeric,
 75 |     income_taxes_payable numeric,
 76 |     other_current_liabilities numeric,
 77 |     total_current_liabilities numeric,
 78 |     mortgages numeric,
 79 |     deferred_taxes_or_income numeric,
 80 |     convertible_debt numeric,
 81 |     long_term_debt numeric,
 82 |     non_current_capital_leases numeric,
 83 |     other_non_current_liabilities numeric,
 84 |     minority_interest numeric,
 85 |     total_liabilities numeric,
 86 |     CONSTRAINT balance_sheet_liabilities_pkey PRIMARY KEY (act_symbol, date, period),
 87 |     CONSTRAINT balance_sheet_liabilities_act_symbol_fkey FOREIGN KEY (act_symbol)
 88 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
 89 |         ON UPDATE NO ACTION
 90 |         ON DELETE NO ACTION
 91 | );
 92 | 
 93 | CREATE TABLE zacks.cash_flow_statement
 94 | (
 95 |     act_symbol text NOT NULL,
 96 |     date date NOT NULL,
 97 |     period zacks.statement_period NOT NULL,
 98 |     net_income numeric,
 99 |     depreciation_amortization_and_depletion numeric,
100 |     net_change_from_assets numeric,
101 |     net_cash_from_discontinued_operations numeric,
102 |     other_operating_activities numeric,
103 |     net_cash_from_operating_activities numeric,
104 |     property_and_equipment numeric,
105 |     acquisition_of_subsidiaries numeric,
106 |     investments numeric,
107 |     other_investing_activities numeric,
108 |     net_cash_from_investing_activities numeric,
109 |     issuance_of_capital_stock numeric,
110 |     issuance_of_debt numeric,
111 |     increase_short_term_debt numeric,
112 |     payment_of_dividends_and_other_distributions numeric,
113 |     other_financing_activities numeric,
114 |     net_cash_from_financing_activities numeric,
115 |     effect_of_exchange_rate_changes numeric,
116 |     net_change_in_cash_and_equivalents numeric,
117 |     cash_at_beginning_of_period numeric,
118 |     cash_at_end_of_period numeric,
119 |     diluted_net_eps numeric,
120 |     CONSTRAINT cash_flow_statement_pkey PRIMARY KEY (act_symbol, date, period),
121 |     CONSTRAINT cash_flow_statement_act_symbol_fkey FOREIGN KEY (act_symbol)
122 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
123 |         ON UPDATE NO ACTION
124 |         ON DELETE NO ACTION
125 | );
126 | 
127 | CREATE TABLE zacks.eps_estimate
128 | (
129 |     act_symbol text NOT NULL,
130 |     date date NOT NULL,
131 |     period zacks.estimate_period NOT NULL,
132 |     period_end_date date NOT NULL,
133 |     consensus numeric,
134 |     recent numeric,
135 |     count smallint,
136 |     high numeric,
137 |     low numeric,
138 |     year_ago numeric,
139 |     CONSTRAINT eps_estimate_pkey PRIMARY KEY (date, act_symbol, period),
140 |     CONSTRAINT eps_estimate_act_symbol_fkey FOREIGN KEY (act_symbol)
141 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
142 |         ON UPDATE NO ACTION
143 |         ON DELETE NO ACTION
144 | );
145 | 
146 | CREATE TABLE zacks.eps_history
147 | (
148 |     act_symbol text NOT NULL,
149 |     period_end_date date NOT NULL,
150 |     reported numeric,
151 |     estimate numeric,
152 |     CONSTRAINT eps_history_pkey PRIMARY KEY (act_symbol, period_end_date),
153 |     CONSTRAINT eps_history_act_symbol_fkey FOREIGN KEY (act_symbol)
154 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
155 |         ON UPDATE NO ACTION
156 |         ON DELETE NO ACTION
157 | );
158 | 
159 | CREATE TABLE zacks.eps_perception
160 | (
161 |     act_symbol text NOT NULL,
162 |     date date NOT NULL,
163 |     period zacks.estimate_period NOT NULL,
164 |     period_end_date date NOT NULL,
165 |     most_accurate numeric,
166 |     CONSTRAINT eps_perception_pkey PRIMARY KEY (date, act_symbol, period),
167 |     CONSTRAINT eps_perception_act_symbol_fkey FOREIGN KEY (act_symbol)
168 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
169 |         ON UPDATE NO ACTION
170 |         ON DELETE NO ACTION
171 | );
172 | 
173 | CREATE TABLE zacks.eps_revision
174 | (
175 |     act_symbol text NOT NULL,
176 |     date date NOT NULL,
177 |     period zacks.estimate_period NOT NULL,
178 |     period_end_date date NOT NULL,
179 |     up_7 smallint,
180 |     up_30 smallint,
181 |     up_60 smallint,
182 |     down_7 smallint,
183 |     down_30 smallint,
184 |     down_60 smallint,
185 |     CONSTRAINT eps_revision_pkey PRIMARY KEY (date, act_symbol, period),
186 |     CONSTRAINT eps_revision_act_symbol_fkey FOREIGN KEY (act_symbol)
187 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
188 |         ON UPDATE NO ACTION
189 |         ON DELETE NO ACTION
190 | );
191 | 
192 | CREATE TABLE zacks.income_statement
193 | (
194 |     act_symbol text NOT NULL,
195 |     date date NOT NULL,
196 |     period zacks.statement_period NOT NULL,
197 |     sales numeric,
198 |     cost_of_goods numeric,
199 |     gross_profit numeric,
200 |     selling_administrative_depreciation_amortization_expenses numeric,
201 |     income_after_depreciation_and_amortization numeric,
202 |     non_operating_income numeric,
203 |     interest_expense numeric,
204 |     pretax_income numeric,
205 |     income_taxes numeric,
206 |     minority_interest numeric,
207 |     investment_gains numeric,
208 |     other_income numeric,
209 |     income_from_continuing_operations numeric,
210 |     extras_and_discontinued_operations numeric,
211 |     net_income numeric,
212 |     income_before_depreciation_and_amortization numeric,
213 |     depreciation_and_amortization numeric,
214 |     average_shares numeric,
215 |     diluted_eps_before_non_recurring_items numeric,
216 |     diluted_net_eps numeric,
217 |     CONSTRAINT income_statement_pkey PRIMARY KEY (act_symbol, date, period),
218 |     CONSTRAINT income_statement_act_symbol_fkey FOREIGN KEY (act_symbol)
219 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
220 |         ON UPDATE NO ACTION
221 |         ON DELETE NO ACTION
222 | );
223 | 
224 | CREATE TABLE zacks.rank_score
225 | (
226 |     act_symbol text NOT NULL,
227 |     date date NOT NULL,
228 |     rank zacks.rank NOT NULL,
229 |     value zacks.score NOT NULL,
230 |     growth zacks.score NOT NULL,
231 |     momentum zacks.score NOT NULL,
232 |     vgm zacks.score NOT NULL,
233 |     CONSTRAINT rank_score_pkey PRIMARY KEY (date, act_symbol),
234 |     CONSTRAINT rank_score_act_symbol_fkey FOREIGN KEY (act_symbol)
235 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
236 |         ON UPDATE NO ACTION
237 |         ON DELETE NO ACTION
238 | );
239 | 
240 | CREATE TABLE zacks.sales_estimate
241 | (
242 |     act_symbol text NOT NULL,
243 |     date date NOT NULL,
244 |     period zacks.estimate_period NOT NULL,
245 |     period_end_date date NOT NULL,
246 |     consensus numeric,
247 |     count smallint,
248 |     high numeric,
249 |     low numeric,
250 |     year_ago numeric,
251 |     CONSTRAINT sales_estimate_pkey PRIMARY KEY (date, act_symbol, period),
252 |     CONSTRAINT sales_estimate_act_symbol_fkey FOREIGN KEY (act_symbol)
253 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
254 |         ON UPDATE NO ACTION
255 |         ON DELETE NO ACTION
256 | );
257 | 
258 | CREATE TABLE zacks.dividend_calendar
259 | (
260 |     act_symbol text NOT NULL,
261 |     ex_date date NOT NULL,
262 |     amount decimal NOT NULL,
263 |     payable_date date,
264 |     CONSTRAINT dividend_calendar_pk PRIMARY KEY (act_symbol, ex_date),
265 |     CONSTRAINT dividend_calendar_act_symbol_fkey FOREIGN KEY (act_symbol) REFERENCES nasdaq.symbol(act_symbol)
266 | );
267 | 
268 | CREATE TABLE zacks.earnings_calendar
269 | (
270 |     act_symbol text NOT NULL,
271 |     date date NOT NULL,
272 |     "when" zacks."when",
273 |     CONSTRAINT earnings_calendar_pkey PRIMARY KEY (act_symbol, date),
274 |     CONSTRAINT earnings_calendar_act_symbol_fkey FOREIGN KEY (act_symbol)
275 |         REFERENCES nasdaq.symbol (act_symbol) MATCH SIMPLE
276 |         ON UPDATE NO ACTION
277 |         ON DELETE NO ACTION
278 | );
279 | 
280 | CREATE OR REPLACE FUNCTION zacks.to_integer_rank(
281 | 	rank zacks.rank)
282 |     RETURNS integer
283 |     LANGUAGE 'sql'
284 | AS $BODY$
285 | select
286 |   case "rank"::text
287 |     when 'Strong Buy' then 1
288 |     when 'Buy' then 2
289 |     when 'Hold' then 3
290 |     when 'Sell' then 4
291 |     when 'Strong Sell' then 5
292 |   end;
293 | $BODY$;
294 | 


--------------------------------------------------------------------------------
/dump-dolt-statements.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          racket/cmdline
  6 |          racket/string
  7 |          racket/system)
  8 | 
  9 | (define base-folder (make-parameter "/var/local/dolt/earnings"))
 10 | 
 11 | (define start-date (make-parameter (~t (-days (today) 250) "yyyy-MM-dd")))
 12 | 
 13 | (define end-date (make-parameter (~t (today) "yyyy-MM-dd")))
 14 | 
 15 | (define db-user (make-parameter "user"))
 16 | 
 17 | (define db-name (make-parameter "local"))
 18 | 
 19 | (define db-pass (make-parameter ""))
 20 | 
 21 | (command-line
 22 |  #:program "racket dump-dolt-statements.rkt"
 23 |  #:once-each
 24 |  [("-b" "--base-folder") folder
 25 |                          "Base dolt folder. Defaults to /var/local/dolt/earnings"
 26 |                          (base-folder folder)]
 27 |  [("-e" "--end-date") end
 28 |                       "Final date for history retrieval. Defaults to today"
 29 |                       (end-date end)]
 30 |  [("-n" "--db-name") name
 31 |                      "Database name. Defaults to 'local'"
 32 |                      (db-name name)]
 33 |  [("-p" "--db-pass") password
 34 |                      "Database password"
 35 |                      (db-pass password)]
 36 |  [("-s" "--start-date") start
 37 |                         "Earliest date for history retrieval. Defaults to today minus 250 days"
 38 |                         (start-date start)]
 39 |  [("-u" "--db-user") user
 40 |                      "Database user name. Defaults to 'user'"
 41 |                      (db-user user)])
 42 | 
 43 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 44 | 
 45 | ; balance-sheet-assets
 46 | (for-each (λ (date)
 47 |             (define balance-sheet-assets-file (string-append (base-folder) "/balance-sheet-assets-" date ".csv"))
 48 |             (call-with-output-file* balance-sheet-assets-file
 49 |               (λ (out)
 50 |                 (displayln "act_symbol,date,period,cash_and_equivalents,receivables,notes_receivable,inventories,other_current_assets,total_current_assets,net_property_and_equipment,investments_and_advances,other_non_current_assets,deferred_charges,intangibles,deposits_and_other_assets,total_assets" out)
 51 |                 (for-each (λ (row)
 52 |                             (displayln (string-join (vector->list row) ",") out))
 53 |                           (query-rows dbc "
 54 | select
 55 |   act_symbol::text,
 56 |   date::text,
 57 |   period::text,
 58 |   coalesce(cash_and_equivalents::text, ''),
 59 |   coalesce(receivables::text, ''),
 60 |   coalesce(notes_receivable::text, ''),
 61 |   coalesce(inventories::text, ''),
 62 |   coalesce(other_current_assets::text, ''),
 63 |   coalesce(total_current_assets::text, ''),
 64 |   coalesce(net_property_and_equipment::text, ''),
 65 |   coalesce(investments_and_advances::text, ''),
 66 |   coalesce(other_non_current_assets::text, ''),
 67 |   coalesce(deferred_charges::text, ''),
 68 |   coalesce(intangibles::text, ''),
 69 |   coalesce(deposits_and_other_assets::text, ''),
 70 |   coalesce(total_assets::text, '')
 71 | from
 72 |   zacks.balance_sheet_assets
 73 | where
 74 |   date = $1::text::date
 75 | order by
 76 |   act_symbol, date, period;
 77 | "
 78 |                                       date)))
 79 |               #:exists 'replace)
 80 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue balance_sheet_assets balance-sheet-assets-" date ".csv")))
 81 |           (query-list dbc "
 82 | select distinct
 83 |   date::text
 84 | from
 85 |   zacks.balance_sheet_assets
 86 | where
 87 |   date >= $1::text::date and
 88 |   date <= $2::text::date
 89 | order by
 90 |   date;
 91 | "
 92 |                       (start-date)
 93 |                       (end-date)))
 94 | 
 95 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add balance_sheet_assets; "
 96 |                        "/usr/local/bin/dolt commit -m 'balance_sheet_assets " (end-date) " update'; /usr/local/bin/dolt push --silent"))
 97 | 
 98 | ; balance-sheet-equity
 99 | (for-each (λ (date)
100 |             (define balance-sheet-equity-file (string-append (base-folder) "/balance-sheet-equity-" date ".csv"))
101 |             (call-with-output-file* balance-sheet-equity-file
102 |               (λ (out)
103 |                 (displayln "act_symbol,date,period,preferred_stock,common_stock,capital_surplus,retained_earnings,other_equity,treasury_stock,total_equity,total_liabilities_and_equity,shares_outstanding,book_value_per_share" out)
104 |                 (for-each (λ (row)
105 |                             (displayln (string-join (vector->list row) ",") out))
106 |                           (query-rows dbc "
107 | select
108 |   act_symbol::text,
109 |   date::text,
110 |   period::text,
111 |   coalesce(preferred_stock::text, ''),
112 |   coalesce(common_stock::text, ''),
113 |   coalesce(capital_surplus::text, ''),
114 |   coalesce(retained_earnings::text, ''),
115 |   coalesce(other_equity::text, ''),
116 |   coalesce(treasury_stock::text, ''),
117 |   coalesce(total_equity::text, ''),
118 |   coalesce(total_liabilities_and_equity::text, ''),
119 |   coalesce(shares_outstanding::text, ''),
120 |   coalesce(book_value_per_share::text, '')
121 | from
122 |   zacks.balance_sheet_equity
123 | where
124 |   date = $1::text::date
125 | order by
126 |   act_symbol, date, period;
127 | "
128 |                                       date)))
129 |               #:exists 'replace)
130 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue balance_sheet_equity balance-sheet-equity-" date ".csv")))
131 |           (query-list dbc "
132 | select distinct
133 |   date::text
134 | from
135 |   zacks.balance_sheet_equity
136 | where
137 |   date >= $1::text::date and
138 |   date <= $2::text::date
139 | order by
140 |   date;
141 | "
142 |                       (start-date)
143 |                       (end-date)))
144 | 
145 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add balance_sheet_equity; "
146 |                        "/usr/local/bin/dolt commit -m 'balance_sheet_equity " (end-date) " update'; /usr/local/bin/dolt push --silent"))
147 | 
148 | ; balance-sheet-liabilities
149 | (for-each (λ (date)
150 |             (define balance-sheet-liabilities-file (string-append (base-folder) "/balance-sheet-liabilities-" date ".csv"))
151 |             (call-with-output-file* balance-sheet-liabilities-file
152 |               (λ (out)
153 |                 (displayln "act_symbol,date,period,notes_payable,accounts_payable,current_portion_long_term_debt,current_portion_capital_leases,accrued_expenses,income_taxes_payable,other_current_liabilities,total_current_liabilities,mortgages,deferred_taxes_or_income,convertible_debt,long_term_debt,non_current_capital_leases,other_non_current_liabilities,minority_interest,total_liabilities" out)
154 |                 (for-each (λ (row)
155 |                             (displayln (string-join (vector->list row) ",") out))
156 |                           (query-rows dbc "
157 | select
158 |   act_symbol::text,
159 |   date::text,
160 |   period::text,
161 |   coalesce(notes_payable::text, ''),
162 |   coalesce(accounts_payable::text, ''),
163 |   coalesce(current_portion_long_term_debt::text, ''),
164 |   coalesce(current_portion_capital_leases::text, ''),
165 |   coalesce(accrued_expenses::text, ''),
166 |   coalesce(income_taxes_payable::text, ''),
167 |   coalesce(other_current_liabilities::text, ''),
168 |   coalesce(total_current_liabilities::text, ''),
169 |   coalesce(mortgages::text, ''),
170 |   coalesce(deferred_taxes_or_income::text, ''),
171 |   coalesce(convertible_debt::text, ''),
172 |   coalesce(long_term_debt::text, ''),
173 |   coalesce(non_current_capital_leases::text, ''),
174 |   coalesce(other_non_current_liabilities::text, ''),
175 |   coalesce(minority_interest::text, ''),
176 |   coalesce(total_liabilities::text, '')
177 | from
178 |   zacks.balance_sheet_liabilities
179 | where
180 |   date = $1::text::date
181 | order by
182 |   act_symbol, date, period;
183 | "
184 |                                       date)))
185 |               #:exists 'replace)
186 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue balance_sheet_liabilities balance-sheet-liabilities-" date ".csv")))
187 |           (query-list dbc "
188 | select distinct
189 |   date::text
190 | from
191 |   zacks.balance_sheet_liabilities
192 | where
193 |   date >= $1::text::date and
194 |   date <= $2::text::date
195 | order by
196 |   date;
197 | "
198 |                       (start-date)
199 |                       (end-date)))
200 | 
201 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add balance_sheet_liabilities; "
202 |                        "/usr/local/bin/dolt commit -m 'balance_sheet_liabilities " (end-date) " update'; /usr/local/bin/dolt push --silent"))
203 | 
204 | ; cash-flow-statement
205 | (for-each (λ (date)
206 |             (define cash-flow-statement-file (string-append (base-folder) "/cash-flow-statement-" date ".csv"))
207 |             (call-with-output-file* cash-flow-statement-file
208 |               (λ (out)
209 |                 (displayln "act_symbol,date,period,net_income,depreciation_amortization_and_depletion,net_change_from_assets,net_cash_from_discontinued_operations,other_operating_activities,net_cash_from_operating_activities,property_and_equipment,acquisition_of_subsidiaries,investments,other_investing_activities,net_cash_from_investing_activities,issuance_of_capital_stock,issuance_of_debt,increase_short_term_debt,payment_of_dividends_and_other_distributions,other_financing_activities,net_cash_from_financing_activities,effect_of_exchange_rate_changes,net_change_in_cash_and_equivalents,cash_at_beginning_of_period,cash_at_end_of_period,diluted_net_eps" out)
210 |                 (for-each (λ (row)
211 |                             (displayln (string-join (vector->list row) ",") out))
212 |                           (query-rows dbc "
213 | select
214 |   act_symbol::text,
215 |   date::text,
216 |   period::text,
217 |   coalesce(net_income::text, ''),
218 |   coalesce(depreciation_amortization_and_depletion::text, ''),
219 |   coalesce(net_change_from_assets::text, ''),
220 |   coalesce(net_cash_from_discontinued_operations::text, ''),
221 |   coalesce(other_operating_activities::text, ''),
222 |   coalesce(net_cash_from_operating_activities::text, ''),
223 |   coalesce(property_and_equipment::text, ''),
224 |   coalesce(acquisition_of_subsidiaries::text, ''),
225 |   coalesce(investments::text, ''),
226 |   coalesce(other_investing_activities::text, ''),
227 |   coalesce(net_cash_from_investing_activities::text, ''),
228 |   coalesce(issuance_of_capital_stock::text, ''),
229 |   coalesce(issuance_of_debt::text, ''),
230 |   coalesce(increase_short_term_debt::text, ''),
231 |   coalesce(payment_of_dividends_and_other_distributions::text, ''),
232 |   coalesce(other_financing_activities::text, ''),
233 |   coalesce(net_cash_from_financing_activities::text, ''),
234 |   coalesce(effect_of_exchange_rate_changes::text, ''),
235 |   coalesce(net_change_in_cash_and_equivalents::text, ''),
236 |   coalesce(cash_at_beginning_of_period::text, ''),
237 |   coalesce(cash_at_end_of_period::text, ''),
238 |   coalesce(diluted_net_eps::text, '')
239 | from
240 |   zacks.cash_flow_statement
241 | where
242 |   date = $1::text::date
243 | order by
244 |   act_symbol, date, period;
245 | "
246 |                                       date)))
247 |               #:exists 'replace)
248 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue cash_flow_statement cash-flow-statement-" date ".csv")))
249 |           (query-list dbc "
250 | select distinct
251 |   date::text
252 | from
253 |   zacks.cash_flow_statement
254 | where
255 |   date >= $1::text::date and
256 |   date <= $2::text::date
257 | order by
258 |   date;
259 | "
260 |                       (start-date)
261 |                       (end-date)))
262 | 
263 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add cash_flow_statement; "
264 |                        "/usr/local/bin/dolt commit -m 'cash_flow_statement " (end-date) " update'; /usr/local/bin/dolt push --silent"))
265 | 
266 | ; income-statement
267 | (for-each (λ (date)
268 |             (define income-statement-file (string-append (base-folder) "/income-statement-" date ".csv"))
269 |             (call-with-output-file* income-statement-file
270 |               (λ (out)
271 |                 (displayln "act_symbol,date,period,sales,cost_of_goods,gross_profit,selling_administrative_depreciation_amortization_expenses,income_after_depreciation_and_amortization,non_operating_income,interest_expense,pretax_income,income_taxes,minority_interest,investment_gains,other_income,income_from_continuing_operations,extras_and_discontinued_operations,net_income,income_before_depreciation_and_amortization,depreciation_and_amortization,average_shares,diluted_eps_before_non_recurring_items,diluted_net_eps" out)
272 |                 (for-each (λ (row)
273 |                             (displayln (string-join (vector->list row) ",") out))
274 |                           (query-rows dbc "
275 | select
276 |   act_symbol::text,
277 |   date::text,
278 |   period::text,
279 |   coalesce(sales::text, ''),
280 |   coalesce(cost_of_goods::text, ''),
281 |   coalesce(gross_profit::text, ''),
282 |   coalesce(selling_administrative_depreciation_amortization_expenses::text, ''),
283 |   coalesce(income_after_depreciation_and_amortization::text, ''),
284 |   coalesce(non_operating_income::text, ''),
285 |   coalesce(interest_expense::text, ''),
286 |   coalesce(pretax_income::text, ''),
287 |   coalesce(income_taxes::text, ''),
288 |   coalesce(minority_interest::text, ''),
289 |   coalesce(investment_gains::text, ''),
290 |   coalesce(other_income::text, ''),
291 |   coalesce(income_from_continuing_operations::text, ''),
292 |   coalesce(extras_and_discontinued_operations::text, ''),
293 |   coalesce(net_income::text, ''),
294 |   coalesce(income_before_depreciation_and_amortization::text, ''),
295 |   coalesce(depreciation_and_amortization::text, ''),
296 |   coalesce(average_shares::text, ''),
297 |   coalesce(diluted_eps_before_non_recurring_items::text, ''),
298 |   coalesce(diluted_net_eps::text, '')
299 | from
300 |   zacks.income_statement
301 | where
302 |   date = $1::text::date
303 | order by
304 |   act_symbol, date, period;
305 | "
306 |                                       date)))
307 |               #:exists 'replace)
308 |             (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt table import -u --continue income_statement income-statement-" date ".csv")))
309 |           (query-list dbc "
310 | select distinct
311 |   date::text
312 | from
313 |   zacks.income_statement
314 | where
315 |   date >= $1::text::date and
316 |   date <= $2::text::date
317 | order by
318 |   date;
319 | "
320 |                       (start-date)
321 |                       (end-date)))
322 | 
323 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt add income_statement; "
324 |                        "/usr/local/bin/dolt commit -m 'income_statement " (end-date) " update'; /usr/local/bin/dolt push --silent"))
325 | 
326 | (system (string-append "cd " (base-folder) "; /usr/local/bin/dolt gc"))
327 | 


--------------------------------------------------------------------------------
/cash-flow-statement-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          gregor/period
  6 |          html-parsing
  7 |          racket/cmdline
  8 |          racket/list
  9 |          racket/sequence
 10 |          racket/string
 11 |          sxml
 12 |          threading)
 13 | 
 14 | (define (cash-flow-statement-figure xexp #:section section #:date date #:entry entry)
 15 |   (let*-values ([(section-id) (case section
 16 |                                 ['cash-flow "cash_flow_operation"]
 17 |                                 ['uses-of-funds "cash_flow_use"])]
 18 |                 [(col) (case date
 19 |                          ['most-recent 2]
 20 |                          ['second-most-recent 3]
 21 |                          ['third-most-recent 4]
 22 |                          ['fourth-most-recent 5]
 23 |                          ['fifth-most-recent 6])]
 24 |                 [(row thead-tbody th-td) (case entry
 25 |                                            ['date (values 1 `thead `th)]
 26 |                                            ['net-income (values 2 `tbody `td)]
 27 |                                            ['depreciation-amortization-and-depletion (values 3 `tbody `td)]
 28 |                                            ['net-change-from-assets (values 4 `tbody `td)]
 29 |                                            ['net-cash-from-discontinued-operations (values 5 `tbody `td)]
 30 |                                            ['other-operating-activities (values 6 `tbody `td)]
 31 |                                            ['net-cash-from-operating-activities (values 7 `tbody `td)]
 32 |                                            ['property-and-equipment (values 8 `tbody `td)]
 33 |                                            ['acquisition-of-subsidiaries (values 9 `tbody `td)]
 34 |                                            ['investments (values 10 `tbody `td)]
 35 |                                            ['other-investing-activities (values 11 `tbody `td)]
 36 |                                            ['net-cash-from-investing-activities (values 12 `tbody `td)]
 37 |                                            ['issuance-of-capital-stock (values 1 `tbody `td)]
 38 |                                            ['issuance-of-debt (values 2 `tbody `td)]
 39 |                                            ['increase-short-term-debt (values 3 `tbody `td)]
 40 |                                            ['payment-of-dividends-and-other-distributions (values 4 `tbody `td)]
 41 |                                            ['other-financing-activities (values 5 `tbody `td)]
 42 |                                            ['net-cash-from-financing-activities (values 6 `tbody `td)]
 43 |                                            ['effect-of-exchange-rate-changes (values 7 `tbody `td)]
 44 |                                            ['net-change-in-cash-and-equivalents (values 8 `tbody `td)]
 45 |                                            ['cash-at-beginning-of-period (values 9 `tbody `td)]
 46 |                                            ['cash-at-end-of-period (values 10 `tbody `td)]
 47 |                                            ['diluted-net-eps (values 11 `tbody `td)])])
 48 |     (~>
 49 |      ; (define in-file (open-input-file "/var/local/zacks/cash-flow-statement/2018-05-08/AA.cash-flow-statement.html"))
 50 |      ; (define in-xexp (html->xexp in-file))
 51 |      ; (webscraperhelper '(td (@ (class "alpha")) "Investments") in-xexp)
 52 |      ((sxpath `(// (section (@ (equal? (id ,section-id))))
 53 |                    table ,thead-tbody (tr ,row) (,th-td ,col))) xexp)
 54 |      (flatten _)
 55 |      (last _)
 56 |      (string-trim _)
 57 |      (string-replace _ "," ""))))
 58 | 
 59 | (define base-folder (make-parameter "/var/local/zacks/cash-flow-statement"))
 60 | 
 61 | (define folder-date (make-parameter (today)))
 62 | 
 63 | (define db-user (make-parameter "user"))
 64 | 
 65 | (define db-name (make-parameter "local"))
 66 | 
 67 | (define db-pass (make-parameter ""))
 68 | 
 69 | (command-line
 70 |  #:program "racket cash-flow-statement-transform-load.rkt"
 71 |  #:once-each
 72 |  [("-b" "--base-folder") folder
 73 |                          "Zacks cash flow statement base folder. Defaults to /var/local/zacks/cash-flow-statement"
 74 |                          (base-folder folder)]
 75 |  [("-d" "--folder-date") date
 76 |                          "Zacks cash flow statement folder date. Defaults to today"
 77 |                          (folder-date (iso8601->date date))]
 78 |  [("-n" "--db-name") name
 79 |                      "Database name. Defaults to 'local'"
 80 |                      (db-name name)]
 81 |  [("-p" "--db-pass") password
 82 |                      "Database password"
 83 |                      (db-pass password)]
 84 |  [("-u" "--db-user") user
 85 |                      "Database user name. Defaults to 'user'"
 86 |                      (db-user user)])
 87 | 
 88 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 89 | 
 90 | (define insert-counter 0)
 91 | (define insert-success-counter 0)
 92 | (define insert-failure-counter 0)
 93 | 
 94 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
 95 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".cash-flow-statement.html")) (in-directory (current-directory)))])
 96 |     (let* ([file-name (path->string p)]
 97 |            [ticker-symbol (string-replace (string-replace file-name (path->string (current-directory)) "") ".cash-flow-statement.html" "")])
 98 |       (call-with-input-file file-name
 99 |         (λ (in) (let ([xexp (html->xexp in)])
100 |                   ; we assume that if we have an updated value that is very close to our extract date that the whole set of data is bad.
101 |                   (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to extract a date from " ticker-symbol)))])
102 |                     (cond [(< 15 (period-ref (date-period-between (parse-date (cash-flow-statement-figure xexp #:section 'cash-flow
103 |                                                                                                           #:date 'most-recent #:entry 'date)
104 |                                                                               "M/dd/yyyy")
105 |                                                                   (folder-date)
106 |                                                                   (list 'days))
107 |                                              'days))
108 |                            (for-each (λ (date)
109 |                                        (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to process "
110 |                                                                                                    ticker-symbol
111 |                                                                                                    " for date "
112 |                                                                                                    (~t (folder-date) "yyyy-MM-dd")))
113 |                                                                      (displayln e)
114 |                                                                      (rollback-transaction dbc)
115 |                                                                      (set! insert-failure-counter (add1 insert-failure-counter)))])
116 |                                          (set! insert-counter (add1 insert-counter))
117 |                                          (start-transaction dbc)
118 |                                          (query-exec dbc "
119 | -- We use the common table expression below in order to check if we're receiving
120 | -- bad values from Zacks. When the new fiscal year occurs, Zacks apparently has a
121 | -- bug where values from the prior year are copied into the new year column. We
122 | -- guard against that by checking data against the previous year's data and not
123 | -- inserting it if it is exactly the same by trying to insert a NULL act_symbol.
124 | -- A better way might be with a stored procedure that can return a meaningful error.
125 | with should_not_insert as (
126 |   select
127 |     bool_and(
128 |       net_income = $3::text::decimal * 1e6 and
129 |       depreciation_amortization_and_depletion = $4::text::decimal * 1e6 and
130 |       net_change_from_assets = $5::text::decimal * 1e6 and
131 |       net_cash_from_discontinued_operations = $6::text::decimal * 1e6 and
132 |       other_operating_activities = $7::text::decimal * 1e6 and
133 |       net_cash_from_operating_activities = $8::text::decimal * 1e6 and
134 |       property_and_equipment = $9::text::decimal * 1e6 and
135 |       acquisition_of_subsidiaries = $10::text::decimal * 1e6 and
136 |       investments = $11::text::decimal * 1e6 and
137 |       other_investing_activities = $12::text::decimal * 1e6 and
138 |       net_cash_from_investing_activities = $13::text::decimal * 1e6 and
139 |       issuance_of_capital_stock = $14::text::decimal * 1e6 and
140 |       issuance_of_debt = $15::text::decimal * 1e6 and
141 |       increase_short_term_debt = $16::text::decimal * 1e6 and
142 |       payment_of_dividends_and_other_distributions = $17::text::decimal * 1e6 and
143 |       other_financing_activities = $18::text::decimal * 1e6 and
144 |       net_cash_from_financing_activities = $19::text::decimal * 1e6 and
145 |       effect_of_exchange_rate_changes = $20::text::decimal * 1e6 and
146 |       net_change_in_cash_and_equivalents = $21::text::decimal * 1e6 and
147 |       cash_at_beginning_of_period = $22::text::decimal * 1e6 and
148 |       cash_at_end_of_period = $23::text::decimal * 1e6 and
149 |       diluted_net_eps = $24::text::decimal
150 |     ) as sni
151 |   from
152 |     zacks.cash_flow_statement
153 |   where
154 |     act_symbol = $1 and
155 |     date = $2::text::date - interval '1 year' and
156 |     period = 'Year'::zacks.statement_period
157 | )
158 | insert into zacks.cash_flow_statement
159 | (
160 |   act_symbol,
161 |   date,
162 |   period,
163 |   net_income,
164 |   depreciation_amortization_and_depletion,
165 |   net_change_from_assets,
166 |   net_cash_from_discontinued_operations,
167 |   other_operating_activities,
168 |   net_cash_from_operating_activities,
169 |   property_and_equipment,
170 |   acquisition_of_subsidiaries,
171 |   investments,
172 |   other_investing_activities,
173 |   net_cash_from_investing_activities,
174 |   issuance_of_capital_stock,
175 |   issuance_of_debt,
176 |   increase_short_term_debt,
177 |   payment_of_dividends_and_other_distributions,
178 |   other_financing_activities,
179 |   net_cash_from_financing_activities,
180 |   effect_of_exchange_rate_changes,
181 |   net_change_in_cash_and_equivalents,
182 |   cash_at_beginning_of_period,
183 |   cash_at_end_of_period,
184 |   diluted_net_eps
185 | ) values (
186 |   case (select sni from should_not_insert)
187 |     when true then NULL
188 |     else $1
189 |   end,
190 |   $2::text::date,
191 |   'Year'::zacks.statement_period,
192 |   $3::text::decimal * 1e6,
193 |   $4::text::decimal * 1e6,
194 |   $5::text::decimal * 1e6,
195 |   $6::text::decimal * 1e6,
196 |   $7::text::decimal * 1e6,
197 |   $8::text::decimal * 1e6,
198 |   $9::text::decimal * 1e6,
199 |   $10::text::decimal * 1e6,
200 |   $11::text::decimal * 1e6,
201 |   $12::text::decimal * 1e6,
202 |   $13::text::decimal * 1e6,
203 |   $14::text::decimal * 1e6,
204 |   $15::text::decimal * 1e6,
205 |   $16::text::decimal * 1e6,
206 |   $17::text::decimal * 1e6,
207 |   $18::text::decimal * 1e6,
208 |   $19::text::decimal * 1e6,
209 |   $20::text::decimal * 1e6,
210 |   $21::text::decimal * 1e6,
211 |   $22::text::decimal * 1e6,
212 |   $23::text::decimal * 1e6,
213 |   $24::text::decimal
214 | ) on conflict (act_symbol, date, period) do nothing;
215 | "
216 |                                                      ticker-symbol
217 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
218 |                                                                                  #:entry 'date)
219 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
220 |                                                                                  #:entry 'net-income)
221 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
222 |                                                                                  #:entry 'depreciation-amortization-and-depletion)
223 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
224 |                                                                                  #:entry 'net-change-from-assets)
225 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
226 |                                                                                  #:entry 'net-cash-from-discontinued-operations)
227 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
228 |                                                                                  #:entry 'other-operating-activities)
229 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
230 |                                                                                  #:entry 'net-cash-from-operating-activities)
231 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
232 |                                                                                  #:entry 'property-and-equipment)
233 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
234 |                                                                                  #:entry 'acquisition-of-subsidiaries)
235 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
236 |                                                                                  #:entry 'investments)
237 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
238 |                                                                                  #:entry 'other-investing-activities)
239 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:date date
240 |                                                                                  #:entry 'net-cash-from-investing-activities)
241 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
242 |                                                                                  #:entry 'issuance-of-capital-stock)
243 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
244 |                                                                                  #:entry 'issuance-of-debt)
245 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
246 |                                                                                  #:entry 'increase-short-term-debt)
247 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
248 |                                                                                  #:entry 'payment-of-dividends-and-other-distributions)
249 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
250 |                                                                                  #:entry 'other-financing-activities)
251 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
252 |                                                                                  #:entry 'net-cash-from-financing-activities)
253 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
254 |                                                                                  #:entry 'effect-of-exchange-rate-changes)
255 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
256 |                                                                                  #:entry 'net-change-in-cash-and-equivalents)
257 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
258 |                                                                                  #:entry 'cash-at-beginning-of-period)
259 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
260 |                                                                                  #:entry 'cash-at-end-of-period)
261 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:date date
262 |                                                                                  #:entry 'diluted-net-eps))
263 |                                          (commit-transaction dbc)
264 |                                          (set! insert-success-counter (add1 insert-success-counter))))
265 |                                      (list 'fifth-most-recent 'fourth-most-recent 'third-most-recent 'second-most-recent 'most-recent))]
266 |                           [else (displayln (string-append "Skipping " ticker-symbol " as the data is most likely using the wrong date."))]))))))))
267 | 
268 | (disconnect dbc)
269 | 
270 | (displayln (string-append "Attempted to insert " (number->string insert-counter) " rows. "
271 |                           (number->string insert-success-counter) " were successful. "
272 |                           (number->string insert-failure-counter) " failed."))
273 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | Mozilla Public License Version 2.0
  2 | ==================================
  3 | 
  4 | 1. Definitions
  5 | --------------
  6 | 
  7 | 1.1. "Contributor"
  8 |     means each individual or legal entity that creates, contributes to
  9 |     the creation of, or owns Covered Software.
 10 | 
 11 | 1.2. "Contributor Version"
 12 |     means the combination of the Contributions of others (if any) used
 13 |     by a Contributor and that particular Contributor's Contribution.
 14 | 
 15 | 1.3. "Contribution"
 16 |     means Covered Software of a particular Contributor.
 17 | 
 18 | 1.4. "Covered Software"
 19 |     means Source Code Form to which the initial Contributor has attached
 20 |     the notice in Exhibit A, the Executable Form of such Source Code
 21 |     Form, and Modifications of such Source Code Form, in each case
 22 |     including portions thereof.
 23 | 
 24 | 1.5. "Incompatible With Secondary Licenses"
 25 |     means
 26 | 
 27 |     (a) that the initial Contributor has attached the notice described
 28 |         in Exhibit B to the Covered Software; or
 29 | 
 30 |     (b) that the Covered Software was made available under the terms of
 31 |         version 1.1 or earlier of the License, but not also under the
 32 |         terms of a Secondary License.
 33 | 
 34 | 1.6. "Executable Form"
 35 |     means any form of the work other than Source Code Form.
 36 | 
 37 | 1.7. "Larger Work"
 38 |     means a work that combines Covered Software with other material, in
 39 |     a separate file or files, that is not Covered Software.
 40 | 
 41 | 1.8. "License"
 42 |     means this document.
 43 | 
 44 | 1.9. "Licensable"
 45 |     means having the right to grant, to the maximum extent possible,
 46 |     whether at the time of the initial grant or subsequently, any and
 47 |     all of the rights conveyed by this License.
 48 | 
 49 | 1.10. "Modifications"
 50 |     means any of the following:
 51 | 
 52 |     (a) any file in Source Code Form that results from an addition to,
 53 |         deletion from, or modification of the contents of Covered
 54 |         Software; or
 55 | 
 56 |     (b) any new file in Source Code Form that contains any Covered
 57 |         Software.
 58 | 
 59 | 1.11. "Patent Claims" of a Contributor
 60 |     means any patent claim(s), including without limitation, method,
 61 |     process, and apparatus claims, in any patent Licensable by such
 62 |     Contributor that would be infringed, but for the grant of the
 63 |     License, by the making, using, selling, offering for sale, having
 64 |     made, import, or transfer of either its Contributions or its
 65 |     Contributor Version.
 66 | 
 67 | 1.12. "Secondary License"
 68 |     means either the GNU General Public License, Version 2.0, the GNU
 69 |     Lesser General Public License, Version 2.1, the GNU Affero General
 70 |     Public License, Version 3.0, or any later versions of those
 71 |     licenses.
 72 | 
 73 | 1.13. "Source Code Form"
 74 |     means the form of the work preferred for making modifications.
 75 | 
 76 | 1.14. "You" (or "Your")
 77 |     means an individual or a legal entity exercising rights under this
 78 |     License. For legal entities, "You" includes any entity that
 79 |     controls, is controlled by, or is under common control with You. For
 80 |     purposes of this definition, "control" means (a) the power, direct
 81 |     or indirect, to cause the direction or management of such entity,
 82 |     whether by contract or otherwise, or (b) ownership of more than
 83 |     fifty percent (50%) of the outstanding shares or beneficial
 84 |     ownership of such entity.
 85 | 
 86 | 2. License Grants and Conditions
 87 | --------------------------------
 88 | 
 89 | 2.1. Grants
 90 | 
 91 | Each Contributor hereby grants You a world-wide, royalty-free,
 92 | non-exclusive license:
 93 | 
 94 | (a) under intellectual property rights (other than patent or trademark)
 95 |     Licensable by such Contributor to use, reproduce, make available,
 96 |     modify, display, perform, distribute, and otherwise exploit its
 97 |     Contributions, either on an unmodified basis, with Modifications, or
 98 |     as part of a Larger Work; and
 99 | 
100 | (b) under Patent Claims of such Contributor to make, use, sell, offer
101 |     for sale, have made, import, and otherwise transfer either its
102 |     Contributions or its Contributor Version.
103 | 
104 | 2.2. Effective Date
105 | 
106 | The licenses granted in Section 2.1 with respect to any Contribution
107 | become effective for each Contribution on the date the Contributor first
108 | distributes such Contribution.
109 | 
110 | 2.3. Limitations on Grant Scope
111 | 
112 | The licenses granted in this Section 2 are the only rights granted under
113 | this License. No additional rights or licenses will be implied from the
114 | distribution or licensing of Covered Software under this License.
115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a
116 | Contributor:
117 | 
118 | (a) for any code that a Contributor has removed from Covered Software;
119 |     or
120 | 
121 | (b) for infringements caused by: (i) Your and any other third party's
122 |     modifications of Covered Software, or (ii) the combination of its
123 |     Contributions with other software (except as part of its Contributor
124 |     Version); or
125 | 
126 | (c) under Patent Claims infringed by Covered Software in the absence of
127 |     its Contributions.
128 | 
129 | This License does not grant any rights in the trademarks, service marks,
130 | or logos of any Contributor (except as may be necessary to comply with
131 | the notice requirements in Section 3.4).
132 | 
133 | 2.4. Subsequent Licenses
134 | 
135 | No Contributor makes additional grants as a result of Your choice to
136 | distribute the Covered Software under a subsequent version of this
137 | License (see Section 10.2) or under the terms of a Secondary License (if
138 | permitted under the terms of Section 3.3).
139 | 
140 | 2.5. Representation
141 | 
142 | Each Contributor represents that the Contributor believes its
143 | Contributions are its original creation(s) or it has sufficient rights
144 | to grant the rights to its Contributions conveyed by this License.
145 | 
146 | 2.6. Fair Use
147 | 
148 | This License is not intended to limit any rights You have under
149 | applicable copyright doctrines of fair use, fair dealing, or other
150 | equivalents.
151 | 
152 | 2.7. Conditions
153 | 
154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
155 | in Section 2.1.
156 | 
157 | 3. Responsibilities
158 | -------------------
159 | 
160 | 3.1. Distribution of Source Form
161 | 
162 | All distribution of Covered Software in Source Code Form, including any
163 | Modifications that You create or to which You contribute, must be under
164 | the terms of this License. You must inform recipients that the Source
165 | Code Form of the Covered Software is governed by the terms of this
166 | License, and how they can obtain a copy of this License. You may not
167 | attempt to alter or restrict the recipients' rights in the Source Code
168 | Form.
169 | 
170 | 3.2. Distribution of Executable Form
171 | 
172 | If You distribute Covered Software in Executable Form then:
173 | 
174 | (a) such Covered Software must also be made available in Source Code
175 |     Form, as described in Section 3.1, and You must inform recipients of
176 |     the Executable Form how they can obtain a copy of such Source Code
177 |     Form by reasonable means in a timely manner, at a charge no more
178 |     than the cost of distribution to the recipient; and
179 | 
180 | (b) You may distribute such Executable Form under the terms of this
181 |     License, or sublicense it under different terms, provided that the
182 |     license for the Executable Form does not attempt to limit or alter
183 |     the recipients' rights in the Source Code Form under this License.
184 | 
185 | 3.3. Distribution of a Larger Work
186 | 
187 | You may create and distribute a Larger Work under terms of Your choice,
188 | provided that You also comply with the requirements of this License for
189 | the Covered Software. If the Larger Work is a combination of Covered
190 | Software with a work governed by one or more Secondary Licenses, and the
191 | Covered Software is not Incompatible With Secondary Licenses, this
192 | License permits You to additionally distribute such Covered Software
193 | under the terms of such Secondary License(s), so that the recipient of
194 | the Larger Work may, at their option, further distribute the Covered
195 | Software under the terms of either this License or such Secondary
196 | License(s).
197 | 
198 | 3.4. Notices
199 | 
200 | You may not remove or alter the substance of any license notices
201 | (including copyright notices, patent notices, disclaimers of warranty,
202 | or limitations of liability) contained within the Source Code Form of
203 | the Covered Software, except that You may alter any license notices to
204 | the extent required to remedy known factual inaccuracies.
205 | 
206 | 3.5. Application of Additional Terms
207 | 
208 | You may choose to offer, and to charge a fee for, warranty, support,
209 | indemnity or liability obligations to one or more recipients of Covered
210 | Software. However, You may do so only on Your own behalf, and not on
211 | behalf of any Contributor. You must make it absolutely clear that any
212 | such warranty, support, indemnity, or liability obligation is offered by
213 | You alone, and You hereby agree to indemnify every Contributor for any
214 | liability incurred by such Contributor as a result of warranty, support,
215 | indemnity or liability terms You offer. You may include additional
216 | disclaimers of warranty and limitations of liability specific to any
217 | jurisdiction.
218 | 
219 | 4. Inability to Comply Due to Statute or Regulation
220 | ---------------------------------------------------
221 | 
222 | If it is impossible for You to comply with any of the terms of this
223 | License with respect to some or all of the Covered Software due to
224 | statute, judicial order, or regulation then You must: (a) comply with
225 | the terms of this License to the maximum extent possible; and (b)
226 | describe the limitations and the code they affect. Such description must
227 | be placed in a text file included with all distributions of the Covered
228 | Software under this License. Except to the extent prohibited by statute
229 | or regulation, such description must be sufficiently detailed for a
230 | recipient of ordinary skill to be able to understand it.
231 | 
232 | 5. Termination
233 | --------------
234 | 
235 | 5.1. The rights granted under this License will terminate automatically
236 | if You fail to comply with any of its terms. However, if You become
237 | compliant, then the rights granted under this License from a particular
238 | Contributor are reinstated (a) provisionally, unless and until such
239 | Contributor explicitly and finally terminates Your grants, and (b) on an
240 | ongoing basis, if such Contributor fails to notify You of the
241 | non-compliance by some reasonable means prior to 60 days after You have
242 | come back into compliance. Moreover, Your grants from a particular
243 | Contributor are reinstated on an ongoing basis if such Contributor
244 | notifies You of the non-compliance by some reasonable means, this is the
245 | first time You have received notice of non-compliance with this License
246 | from such Contributor, and You become compliant prior to 30 days after
247 | Your receipt of the notice.
248 | 
249 | 5.2. If You initiate litigation against any entity by asserting a patent
250 | infringement claim (excluding declaratory judgment actions,
251 | counter-claims, and cross-claims) alleging that a Contributor Version
252 | directly or indirectly infringes any patent, then the rights granted to
253 | You by any and all Contributors for the Covered Software under Section
254 | 2.1 of this License shall terminate.
255 | 
256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all
257 | end user license agreements (excluding distributors and resellers) which
258 | have been validly granted by You or Your distributors under this License
259 | prior to termination shall survive termination.
260 | 
261 | ************************************************************************
262 | *                                                                      *
263 | *  6. Disclaimer of Warranty                                           *
264 | *  -------------------------                                           *
265 | *                                                                      *
266 | *  Covered Software is provided under this License on an "as is"       *
267 | *  basis, without warranty of any kind, either expressed, implied, or  *
268 | *  statutory, including, without limitation, warranties that the       *
269 | *  Covered Software is free of defects, merchantable, fit for a        *
270 | *  particular purpose or non-infringing. The entire risk as to the     *
271 | *  quality and performance of the Covered Software is with You.        *
272 | *  Should any Covered Software prove defective in any respect, You     *
273 | *  (not any Contributor) assume the cost of any necessary servicing,   *
274 | *  repair, or correction. This disclaimer of warranty constitutes an   *
275 | *  essential part of this License. No use of any Covered Software is   *
276 | *  authorized under this License except under this disclaimer.         *
277 | *                                                                      *
278 | ************************************************************************
279 | 
280 | ************************************************************************
281 | *                                                                      *
282 | *  7. Limitation of Liability                                          *
283 | *  --------------------------                                          *
284 | *                                                                      *
285 | *  Under no circumstances and under no legal theory, whether tort      *
286 | *  (including negligence), contract, or otherwise, shall any           *
287 | *  Contributor, or anyone who distributes Covered Software as          *
288 | *  permitted above, be liable to You for any direct, indirect,         *
289 | *  special, incidental, or consequential damages of any character      *
290 | *  including, without limitation, damages for lost profits, loss of    *
291 | *  goodwill, work stoppage, computer failure or malfunction, or any    *
292 | *  and all other commercial damages or losses, even if such party      *
293 | *  shall have been informed of the possibility of such damages. This   *
294 | *  limitation of liability shall not apply to liability for death or   *
295 | *  personal injury resulting from such party's negligence to the       *
296 | *  extent applicable law prohibits such limitation. Some               *
297 | *  jurisdictions do not allow the exclusion or limitation of           *
298 | *  incidental or consequential damages, so this exclusion and          *
299 | *  limitation may not apply to You.                                    *
300 | *                                                                      *
301 | ************************************************************************
302 | 
303 | 8. Litigation
304 | -------------
305 | 
306 | Any litigation relating to this License may be brought only in the
307 | courts of a jurisdiction where the defendant maintains its principal
308 | place of business and such litigation shall be governed by laws of that
309 | jurisdiction, without reference to its conflict-of-law provisions.
310 | Nothing in this Section shall prevent a party's ability to bring
311 | cross-claims or counter-claims.
312 | 
313 | 9. Miscellaneous
314 | ----------------
315 | 
316 | This License represents the complete agreement concerning the subject
317 | matter hereof. If any provision of this License is held to be
318 | unenforceable, such provision shall be reformed only to the extent
319 | necessary to make it enforceable. Any law or regulation which provides
320 | that the language of a contract shall be construed against the drafter
321 | shall not be used to construe this License against a Contributor.
322 | 
323 | 10. Versions of the License
324 | ---------------------------
325 | 
326 | 10.1. New Versions
327 | 
328 | Mozilla Foundation is the license steward. Except as provided in Section
329 | 10.3, no one other than the license steward has the right to modify or
330 | publish new versions of this License. Each version will be given a
331 | distinguishing version number.
332 | 
333 | 10.2. Effect of New Versions
334 | 
335 | You may distribute the Covered Software under the terms of the version
336 | of the License under which You originally received the Covered Software,
337 | or under the terms of any subsequent version published by the license
338 | steward.
339 | 
340 | 10.3. Modified Versions
341 | 
342 | If you create software not governed by this License, and you want to
343 | create a new license for such software, you may create and use a
344 | modified version of this License if you rename the license and remove
345 | any references to the name of the license steward (except to note that
346 | such modified license differs from this License).
347 | 
348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary
349 | Licenses
350 | 
351 | If You choose to distribute Source Code Form that is Incompatible With
352 | Secondary Licenses under the terms of this version of the License, the
353 | notice described in Exhibit B of this License must be attached.
354 | 
355 | Exhibit A - Source Code Form License Notice
356 | -------------------------------------------
357 | 
358 |   This Source Code Form is subject to the terms of the Mozilla Public
359 |   License, v. 2.0. If a copy of the MPL was not distributed with this
360 |   file, You can obtain one at http://mozilla.org/MPL/2.0/.
361 | 
362 | If it is not possible or desirable to put the notice in a particular
363 | file, then You may include the notice in a location (such as a LICENSE
364 | file in a relevant directory) where a recipient would be likely to look
365 | for such a notice.
366 | 
367 | You may add additional accurate notices of copyright ownership.
368 | 
369 | Exhibit B - "Incompatible With Secondary Licenses" Notice
370 | ---------------------------------------------------------
371 | 
372 |   This Source Code Form is "Incompatible With Secondary Licenses", as
373 |   defined by the Mozilla Public License, v. 2.0.
374 | 


--------------------------------------------------------------------------------
/income-statement-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          gregor/period
  6 |          html-parsing
  7 |          racket/cmdline
  8 |          racket/list
  9 |          racket/sequence
 10 |          racket/string
 11 |          sxml
 12 |          threading)
 13 | 
 14 | (define (income-statement-figure xexp #:period period #:date date #:entry entry)
 15 |   (let*-values ([(section-id period-offset) (case period
 16 |                                               ['annual (values "annual_income_statement" 1)]
 17 |                                               ['quarterly (values "quarterly_income_statement" 0)])]
 18 |                 [(col) (case date
 19 |                          ['most-recent 2]
 20 |                          ['second-most-recent 3]
 21 |                          ['third-most-recent 4]
 22 |                          ['fourth-most-recent 5]
 23 |                          ['fifth-most-recent 6])]
 24 |                 [(table-id row thead-tbody th-td) (case entry
 25 |                                                     ['date (values 1 1 `thead `th)]
 26 |                                                     ['sales (values 1 1 `tbody `td)]
 27 |                                                     ['cost-of-goods (values 1 2 `tbody `td)]
 28 |                                                     ['gross-profit (values 1 3 `tbody `td)]
 29 |                                                     ['selling-administrative-depreciation-amortization-expenses (values 1 4 `tbody `td)]
 30 |                                                     ['income-after-depreciation-and-amortization (values 1 5 `tbody `td)]
 31 |                                                     ['non-operating-income (values 1 6 `tbody `td)]
 32 |                                                     ['interest-expense (values 1 7 `tbody `td)]
 33 |                                                     ['pretax-income (values 1 8 `tbody `td)]
 34 |                                                     ['income-taxes (values 1 9 `tbody `td)]
 35 |                                                     ['minority-interest (values 1 10 `tbody `td)]
 36 |                                                     ['investment-gains (values 1 11 `tbody `td)]
 37 |                                                     ['other-income (values 1 12 `tbody `td)]
 38 |                                                     ['income-from-continuing-operations (values 1 13 `tbody `td)]
 39 |                                                     ['extras-and-discontinued-operations (values 1 14 `tbody `td)]
 40 |                                                     ['net-income (values 1 15 `tbody `td)]
 41 |                                                     ['income-before-depreciation-and-amortization (values 2 1 `tbody `td)]
 42 |                                                     ['depreciation-and-amortization (values 2 2 `tbody `td)]
 43 |                                                     ['average-shares (values (+ 2 period-offset) 1 `tbody `td)]
 44 |                                                     ['diluted-eps-before-non-recurring-items (values (+ 2 period-offset) 2 `tbody `td)]
 45 |                                                     ['diluted-net-eps (values (+ 2 period-offset) 3 `tbody `td)])])
 46 |     (~>
 47 |      ; (define in-file (open-input-file "/var/local/zacks/income-statement/2018-05-08/AA.income-statement.html"))
 48 |      ; (define in-xexp (html->xexp in-file))
 49 |      ; (webscraperhelper '(td (@ (class "alpha")) "Sales") in-xexp)
 50 |      ((sxpath `(// (div (@ (equal? (id ,section-id))))
 51 |                    (table ,table-id) ,thead-tbody (tr ,row) (,th-td ,col))) xexp)
 52 |      (flatten _)
 53 |      (last _)
 54 |      (string-trim _)
 55 |      (string-replace _ "," ""))))
 56 | 
 57 | (define base-folder (make-parameter "/var/local/zacks/income-statement"))
 58 | 
 59 | (define folder-date (make-parameter (today)))
 60 | 
 61 | (define db-user (make-parameter "user"))
 62 | 
 63 | (define db-name (make-parameter "local"))
 64 | 
 65 | (define db-pass (make-parameter ""))
 66 | 
 67 | (command-line
 68 |  #:program "racket income-statement-transform-load.rkt"
 69 |  #:once-each
 70 |  [("-b" "--base-folder") folder
 71 |                          "Zacks income statement base folder. Defaults to /var/local/zacks/income-statement"
 72 |                          (base-folder folder)]
 73 |  [("-d" "--folder-date") date
 74 |                          "Zacks income statement folder date. Defaults to today"
 75 |                          (folder-date (iso8601->date date))]
 76 |  [("-n" "--db-name") name
 77 |                      "Database name. Defaults to 'local'"
 78 |                      (db-name name)]
 79 |  [("-p" "--db-pass") password
 80 |                      "Database password"
 81 |                      (db-pass password)]
 82 |  [("-u" "--db-user") user
 83 |                      "Database user name. Defaults to 'user'"
 84 |                      (db-user user)])
 85 | 
 86 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 87 | 
 88 | (define insert-counter 0)
 89 | (define insert-success-counter 0)
 90 | (define insert-failure-counter 0)
 91 | 
 92 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
 93 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".income-statement.html")) (in-directory (current-directory)))])
 94 |     (let* ([file-name (path->string p)]
 95 |            [ticker-symbol (string-replace (string-replace file-name (path->string (current-directory)) "") ".income-statement.html" "")])
 96 |       (call-with-input-file file-name
 97 |         (λ (in) (let ([xexp (html->xexp in)])
 98 |                   ; we assume that if we have an updated value that is very close to our extract date that the whole set of data is bad.
 99 |                   (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to extract a date from " ticker-symbol)))])
100 |                     (cond [(< 15 (period-ref (date-period-between (parse-date (income-statement-figure xexp #:period 'quarterly
101 |                                                                                                        #:date 'most-recent #:entry 'date)
102 |                                                                               "M/dd/yy")
103 |                                                                   (folder-date)
104 |                                                                   (list 'days))
105 |                                              'days))
106 |                            (for-each (λ (period-date)
107 |                                        (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to process "
108 |                                                                                                    ticker-symbol
109 |                                                                                                    " for date "
110 |                                                                                                    (~t (folder-date) "yyyy-MM-dd")))
111 |                                                                      (displayln e)
112 |                                                                      (rollback-transaction dbc)
113 |                                                                      (set! insert-failure-counter (add1 insert-failure-counter)))])
114 |                                          (set! insert-counter (add1 insert-counter))
115 |                                          (start-transaction dbc)
116 |                                          (query-exec dbc "
117 | -- We use the common table expression below in order to check if we're receiving
118 | -- bad values from Zacks. When the new fiscal year occurs, Zacks apparently has a
119 | -- bug where values from the prior year are copied into the new year column. We
120 | -- guard against that by checking data against the previous year's data and not
121 | -- inserting it if it is exactly the same by trying to insert a NULL act_symbol.
122 | -- A better way might be with a stored procedure that can return a meaningful error.
123 | with should_not_insert as (
124 |   select
125 |     bool_and(
126 |       sales = $4::text::decimal * 1e6 and
127 |       cost_of_goods = $5::text::decimal * 1e6 and
128 |       gross_profit = $6::text::decimal * 1e6 and
129 |       selling_administrative_depreciation_amortization_expenses = $7::text::decimal * 1e6 and
130 |       income_after_depreciation_and_amortization = $8::text::decimal * 1e6 and
131 |       non_operating_income = $9::text::decimal * 1e6 and
132 |       interest_expense = $10::text::decimal * 1e6 and
133 |       pretax_income = $11::text::decimal * 1e6 and
134 |       income_taxes = $12::text::decimal * 1e6 and
135 |       minority_interest = $13::text::decimal * 1e6 and
136 |       investment_gains = $14::text::decimal * 1e6 and
137 |       other_income = $15::text::decimal * 1e6 and
138 |       income_from_continuing_operations = $16::text::decimal * 1e6 and
139 |       extras_and_discontinued_operations = $17::text::decimal * 1e6 and
140 |       net_income = $18::text::decimal * 1e6 and
141 |       case $3
142 |         when 'annual' then income_before_depreciation_and_amortization = $19::text::decimal * 1e6
143 |         when 'quarterly' then income_before_depreciation_and_amortization is null
144 |       end and
145 |       case $3
146 |         when 'annual' then depreciation_and_amortization = $20::text::decimal * 1e6
147 |         when 'quarterly' then depreciation_and_amortization is null
148 |       end and
149 |       average_shares = $21::text::decimal * 1e6 and
150 |       diluted_eps_before_non_recurring_items = $22::text::decimal and
151 |       diluted_net_eps = $23::text::decimal
152 |     ) as sni
153 |   from
154 |     zacks.income_statement
155 |   where
156 |     act_symbol = $1 and
157 |     case $3
158 |       when 'annual' then
159 |         period = 'Year'::zacks.statement_period and
160 |         date = $2::text::date - interval '1 year'
161 |       when 'quarterly' then
162 |         period = 'Quarter'::zacks.statement_period and
163 |         date = $2::text::date + interval '1 day' - interval '3 months' - interval '1 day'
164 |     end
165 | )
166 | insert into zacks.income_statement
167 | (
168 |   act_symbol,
169 |   date,
170 |   period,
171 |   sales,
172 |   cost_of_goods,
173 |   gross_profit,
174 |   selling_administrative_depreciation_amortization_expenses,
175 |   income_after_depreciation_and_amortization,
176 |   non_operating_income,
177 |   interest_expense,
178 |   pretax_income,
179 |   income_taxes,
180 |   minority_interest,
181 |   investment_gains,
182 |   other_income,
183 |   income_from_continuing_operations,
184 |   extras_and_discontinued_operations,
185 |   net_income,
186 |   income_before_depreciation_and_amortization,
187 |   depreciation_and_amortization,
188 |   average_shares,
189 |   diluted_eps_before_non_recurring_items,
190 |   diluted_net_eps
191 | ) values (
192 |   case (select sni from should_not_insert)
193 |     when true then NULL
194 |     else $1
195 |   end,
196 |   $2::text::date,
197 |   case $3
198 |     when 'annual' then 'Year'::zacks.statement_period
199 |     when 'quarterly' then 'Quarter'::zacks.statement_period
200 |   end,
201 |   $4::text::decimal * 1e6,
202 |   $5::text::decimal * 1e6,
203 |   $6::text::decimal * 1e6,
204 |   $7::text::decimal * 1e6,
205 |   $8::text::decimal * 1e6,
206 |   $9::text::decimal * 1e6,
207 |   $10::text::decimal * 1e6,
208 |   $11::text::decimal * 1e6,
209 |   $12::text::decimal * 1e6,
210 |   $13::text::decimal * 1e6,
211 |   $14::text::decimal * 1e6,
212 |   $15::text::decimal * 1e6,
213 |   $16::text::decimal * 1e6,
214 |   $17::text::decimal * 1e6,
215 |   $18::text::decimal * 1e6,
216 |   case $3
217 |     when 'annual' then $19::text::decimal * 1e6
218 |     when 'quarterly' then null
219 |   end,
220 |   case $3
221 |     when 'annual' then $20::text::decimal * 1e6
222 |     when 'quarterly' then null
223 |   end,
224 |   $21::text::decimal * 1e6,
225 |   $22::text::decimal,
226 |   $23::text::decimal
227 | ) on conflict (act_symbol, date, period) do nothing;
228 | "
229 |                                                      ticker-symbol
230 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
231 |                                                                               #:entry 'date)
232 |                                                      (symbol->string (first period-date))
233 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
234 |                                                                               #:entry 'sales)
235 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
236 |                                                                               #:entry 'cost-of-goods)
237 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
238 |                                                                               #:entry 'gross-profit)
239 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
240 |                                                                               #:entry 'selling-administrative-depreciation-amortization-expenses)
241 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
242 |                                                                               #:entry 'income-after-depreciation-and-amortization)
243 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
244 |                                                                               #:entry 'non-operating-income)
245 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
246 |                                                                               #:entry 'interest-expense)
247 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
248 |                                                                               #:entry 'pretax-income)
249 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
250 |                                                                               #:entry 'income-taxes)
251 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
252 |                                                                               #:entry 'minority-interest)
253 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
254 |                                                                               #:entry 'investment-gains)
255 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
256 |                                                                               #:entry 'other-income)
257 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
258 |                                                                               #:entry 'income-from-continuing-operations)
259 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
260 |                                                                               #:entry 'extras-and-discontinued-operations)
261 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
262 |                                                                               #:entry 'net-income)
263 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
264 |                                                                               #:entry 'income-before-depreciation-and-amortization)
265 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
266 |                                                                               #:entry 'depreciation-and-amortization)
267 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
268 |                                                                               #:entry 'average-shares)
269 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
270 |                                                                               #:entry 'diluted-eps-before-non-recurring-items)
271 |                                                      (income-statement-figure xexp #:period (first period-date) #:date (second period-date)
272 |                                                                               #:entry 'diluted-net-eps))
273 |                                          (commit-transaction dbc)
274 |                                          (set! insert-success-counter (add1 insert-success-counter))))
275 |                                      (cartesian-product (list 'annual 'quarterly)
276 |                                                         (list 'fifth-most-recent 'fourth-most-recent 'third-most-recent 'second-most-recent 'most-recent)))]
277 |                           [else (displayln (string-append "Skipping " ticker-symbol " as the data is most likely using the wrong date."))]))))))))
278 | 
279 | (disconnect dbc)
280 | 
281 | (displayln (string-append "Attempted to insert " (number->string insert-counter) " rows. "
282 |                           (number->string insert-success-counter) " were successful. "
283 |                           (number->string insert-failure-counter) " failed."))
284 | 


--------------------------------------------------------------------------------
/cash-flow-statement-transform-load.2024-02-01.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          gregor/period
  6 |          html-parsing
  7 |          racket/cmdline
  8 |          racket/list
  9 |          racket/sequence
 10 |          racket/string
 11 |          sxml
 12 |          threading)
 13 | 
 14 | (define (cash-flow-statement-figure xexp #:section section #:period period #:date date #:entry entry)
 15 |   (let*-values ([(table-id) (case section
 16 |                                 ['cash-flow 1]
 17 |                                 ['uses-of-funds 2])]
 18 |                 [(section-id) (case period
 19 |                                 ['annual "annual_cash_flow_statement"]
 20 |                                 ['quarterly "quarterly_cash_flow_statement"])]
 21 |                 [(col) (case date
 22 |                          ['most-recent 2]
 23 |                          ['second-most-recent 3]
 24 |                          ['third-most-recent 4]
 25 |                          ['fourth-most-recent 5]
 26 |                          ['fifth-most-recent 6])]
 27 |                 [(row thead-tbody th-td) (case entry
 28 |                                            ['date (values 1 `thead `th)]
 29 |                                            ['net-income (values 2 `tbody `td)]
 30 |                                            ['depreciation-amortization-and-depletion (values 3 `tbody `td)]
 31 |                                            ['net-change-from-assets (values 4 `tbody `td)]
 32 |                                            ['net-cash-from-discontinued-operations (values 5 `tbody `td)]
 33 |                                            ['other-operating-activities (values 6 `tbody `td)]
 34 |                                            ['net-cash-from-operating-activities (values 7 `tbody `td)]
 35 |                                            ['property-and-equipment (values 8 `tbody `td)]
 36 |                                            ['acquisition-of-subsidiaries (values 9 `tbody `td)]
 37 |                                            ['investments (values 10 `tbody `td)]
 38 |                                            ['other-investing-activities (values 11 `tbody `td)]
 39 |                                            ['net-cash-from-investing-activities (values 12 `tbody `td)]
 40 |                                            ['issuance-of-capital-stock (values 1 `tbody `td)]
 41 |                                            ['issuance-of-debt (values 2 `tbody `td)]
 42 |                                            ['increase-short-term-debt (values 3 `tbody `td)]
 43 |                                            ['payment-of-dividends-and-other-distributions (values 4 `tbody `td)]
 44 |                                            ['other-financing-activities (values 5 `tbody `td)]
 45 |                                            ['net-cash-from-financing-activities (values 6 `tbody `td)]
 46 |                                            ['effect-of-exchange-rate-changes (values 7 `tbody `td)]
 47 |                                            ['net-change-in-cash-and-equivalents (values 8 `tbody `td)]
 48 |                                            ['cash-at-beginning-of-period (values 9 `tbody `td)]
 49 |                                            ['cash-at-end-of-period (values 10 `tbody `td)]
 50 |                                            ['diluted-net-eps (values 11 `tbody `td)])])
 51 |     (~>
 52 |      ; (define in-file (open-input-file "/var/local/zacks/cash-flow-statement/2018-05-08/AA.cash-flow-statement.html"))
 53 |      ; (define in-xexp (html->xexp in-file))
 54 |      ; (webscraperhelper '(td (@ (class "alpha")) "Investments") in-xexp)
 55 |      ((sxpath `(// (div (@ (equal? (id ,section-id)))) (div ,table-id)
 56 |                    table ,thead-tbody (tr ,row) (,th-td ,col))) xexp)
 57 |      (flatten _)
 58 |      (last _)
 59 |      (string-trim _)
 60 |      (string-replace _ "," ""))))
 61 | 
 62 | (define base-folder (make-parameter "/var/local/zacks/cash-flow-statement"))
 63 | 
 64 | (define folder-date (make-parameter (today)))
 65 | 
 66 | (define db-user (make-parameter "user"))
 67 | 
 68 | (define db-name (make-parameter "local"))
 69 | 
 70 | (define db-pass (make-parameter ""))
 71 | 
 72 | (command-line
 73 |  #:program "racket cash-flow-statement-transform-load.rkt"
 74 |  #:once-each
 75 |  [("-b" "--base-folder") folder
 76 |                          "Zacks cash flow statement base folder. Defaults to /var/local/zacks/cash-flow-statement"
 77 |                          (base-folder folder)]
 78 |  [("-d" "--folder-date") date
 79 |                          "Zacks cash flow statement folder date. Defaults to today"
 80 |                          (folder-date (iso8601->date date))]
 81 |  [("-n" "--db-name") name
 82 |                      "Database name. Defaults to 'local'"
 83 |                      (db-name name)]
 84 |  [("-p" "--db-pass") password
 85 |                      "Database password"
 86 |                      (db-pass password)]
 87 |  [("-u" "--db-user") user
 88 |                      "Database user name. Defaults to 'user'"
 89 |                      (db-user user)])
 90 | 
 91 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
 92 | 
 93 | (define insert-counter 0)
 94 | (define insert-success-counter 0)
 95 | (define insert-failure-counter 0)
 96 | 
 97 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
 98 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".cash-flow-statement.html")) (in-directory (current-directory)))])
 99 |     (let* ([file-name (path->string p)]
100 |            [ticker-symbol (string-replace (string-replace file-name (path->string (current-directory)) "") ".cash-flow-statement.html" "")])
101 |       (call-with-input-file file-name
102 |         (λ (in) (let ([xexp (html->xexp in)])
103 |                   ; we assume that if we have an updated value that is very close to our extract date that the whole set of data is bad.
104 |                   (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to extract a date from " ticker-symbol)))])
105 |                     (cond [(< 15 (period-ref (date-period-between (parse-date (cash-flow-statement-figure xexp #:section 'cash-flow #:period 'annual
106 |                                                                                                           #:date 'most-recent #:entry 'date)
107 |                                                                               "M/dd/yyyy")
108 |                                                                   (folder-date)
109 |                                                                   (list 'days))
110 |                                              'days))
111 |                            (for-each (λ (period-date)
112 |                                        (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to process "
113 |                                                                                                    ticker-symbol
114 |                                                                                                    " for date "
115 |                                                                                                    (~t (folder-date) "yyyy-MM-dd")))
116 |                                                                      (displayln e)
117 |                                                                      (rollback-transaction dbc)
118 |                                                                      (set! insert-failure-counter (add1 insert-failure-counter)))])
119 |                                          (set! insert-counter (add1 insert-counter))
120 |                                          (start-transaction dbc)
121 |                                          (query-exec dbc "
122 | -- We use the common table expression below in order to check if we're receiving
123 | -- bad values from Zacks. When the new fiscal year occurs, Zacks apparently has a
124 | -- bug where values from the prior year are copied into the new year column. We
125 | -- guard against that by checking data against the previous year's data and not
126 | -- inserting it if it is exactly the same by trying to insert a NULL act_symbol.
127 | -- A better way might be with a stored procedure that can return a meaningful error.
128 | with should_not_insert as (
129 |   select
130 |     bool_and(
131 |       net_income = $4::text::decimal * 1e6 and
132 |       depreciation_amortization_and_depletion = $5::text::decimal * 1e6 and
133 |       net_change_from_assets = $6::text::decimal * 1e6 and
134 |       net_cash_from_discontinued_operations = $7::text::decimal * 1e6 and
135 |       other_operating_activities = $8::text::decimal * 1e6 and
136 |       net_cash_from_operating_activities = $9::text::decimal * 1e6 and
137 |       property_and_equipment = $10::text::decimal * 1e6 and
138 |       acquisition_of_subsidiaries = $11::text::decimal * 1e6 and
139 |       investments = $12::text::decimal * 1e6 and
140 |       other_investing_activities = $13::text::decimal * 1e6 and
141 |       net_cash_from_investing_activities = $14::text::decimal * 1e6 and
142 |       issuance_of_capital_stock = $15::text::decimal * 1e6 and
143 |       issuance_of_debt = $16::text::decimal * 1e6 and
144 |       increase_short_term_debt = $17::text::decimal * 1e6 and
145 |       payment_of_dividends_and_other_distributions = $18::text::decimal * 1e6 and
146 |       other_financing_activities = $19::text::decimal * 1e6 and
147 |       net_cash_from_financing_activities = $20::text::decimal * 1e6 and
148 |       effect_of_exchange_rate_changes = $21::text::decimal * 1e6 and
149 |       net_change_in_cash_and_equivalents = $22::text::decimal * 1e6 and
150 |       cash_at_beginning_of_period = $23::text::decimal * 1e6 and
151 |       cash_at_end_of_period = $24::text::decimal * 1e6 and
152 |       diluted_net_eps = $25::text::decimal
153 |     ) as sni
154 |   from
155 |     zacks.cash_flow_statement
156 |   where
157 |     act_symbol = $1 and
158 |     case $3
159 |       when 'annual' then
160 |         period = 'Year'::zacks.statement_period and
161 |         date = $2::text::date - interval '1 year'
162 |       when 'quarterly' then
163 |         period = 'Quarter'::zacks.statement_period and
164 |         date = $2::text::date + interval '1 day' - interval '3 months' - interval '1 day'
165 |     end
166 | )
167 | insert into zacks.cash_flow_statement
168 | (
169 |   act_symbol,
170 |   date,
171 |   period,
172 |   net_income,
173 |   depreciation_amortization_and_depletion,
174 |   net_change_from_assets,
175 |   net_cash_from_discontinued_operations,
176 |   other_operating_activities,
177 |   net_cash_from_operating_activities,
178 |   property_and_equipment,
179 |   acquisition_of_subsidiaries,
180 |   investments,
181 |   other_investing_activities,
182 |   net_cash_from_investing_activities,
183 |   issuance_of_capital_stock,
184 |   issuance_of_debt,
185 |   increase_short_term_debt,
186 |   payment_of_dividends_and_other_distributions,
187 |   other_financing_activities,
188 |   net_cash_from_financing_activities,
189 |   effect_of_exchange_rate_changes,
190 |   net_change_in_cash_and_equivalents,
191 |   cash_at_beginning_of_period,
192 |   cash_at_end_of_period,
193 |   diluted_net_eps
194 | ) values (
195 |   case (select sni from should_not_insert)
196 |     when true then NULL
197 |     else $1
198 |   end,
199 |   $2::text::date,
200 |   case $3
201 |     when 'annual' then 'Year'::zacks.statement_period
202 |     when 'quarterly' then 'Quarter'::zacks.statement_period
203 |   end,
204 |   $4::text::decimal * 1e6,
205 |   $5::text::decimal * 1e6,
206 |   $6::text::decimal * 1e6,
207 |   $7::text::decimal * 1e6,
208 |   $8::text::decimal * 1e6,
209 |   $9::text::decimal * 1e6,
210 |   $10::text::decimal * 1e6,
211 |   $11::text::decimal * 1e6,
212 |   $12::text::decimal * 1e6,
213 |   $13::text::decimal * 1e6,
214 |   $14::text::decimal * 1e6,
215 |   $15::text::decimal * 1e6,
216 |   $16::text::decimal * 1e6,
217 |   $17::text::decimal * 1e6,
218 |   $18::text::decimal * 1e6,
219 |   $19::text::decimal * 1e6,
220 |   $20::text::decimal * 1e6,
221 |   $21::text::decimal * 1e6,
222 |   $22::text::decimal * 1e6,
223 |   $23::text::decimal * 1e6,
224 |   $24::text::decimal * 1e6,
225 |   $25::text::decimal
226 | ) on conflict (act_symbol, date, period) do nothing;
227 | "
228 |                                                      ticker-symbol
229 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
230 |                                                                                  #:date (second period-date) #:entry 'date)
231 |                                                      (symbol->string (first period-date))
232 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
233 |                                                                                  #:date (second period-date) #:entry 'net-income)
234 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
235 |                                                                                  #:date (second period-date) #:entry 'depreciation-amortization-and-depletion)
236 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
237 |                                                                                  #:date (second period-date) #:entry 'net-change-from-assets)
238 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
239 |                                                                                  #:date (second period-date) #:entry 'net-cash-from-discontinued-operations)
240 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
241 |                                                                                  #:date (second period-date) #:entry 'other-operating-activities)
242 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
243 |                                                                                  #:date (second period-date) #:entry 'net-cash-from-operating-activities)
244 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
245 |                                                                                  #:date (second period-date) #:entry 'property-and-equipment)
246 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
247 |                                                                                  #:date (second period-date) #:entry 'acquisition-of-subsidiaries)
248 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
249 |                                                                                  #:date (second period-date) #:entry 'investments)
250 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
251 |                                                                                  #:date (second period-date) #:entry 'other-investing-activities)
252 |                                                      (cash-flow-statement-figure xexp #:section 'cash-flow #:period (first period-date)
253 |                                                                                  #:date (second period-date) #:entry 'net-cash-from-investing-activities)
254 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
255 |                                                                                  #:date (second period-date) #:entry 'issuance-of-capital-stock)
256 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
257 |                                                                                  #:date (second period-date) #:entry 'issuance-of-debt)
258 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
259 |                                                                                  #:date (second period-date) #:entry 'increase-short-term-debt)
260 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
261 |                                                                                  #:date (second period-date) #:entry 'payment-of-dividends-and-other-distributions)
262 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
263 |                                                                                  #:date (second period-date) #:entry 'other-financing-activities)
264 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
265 |                                                                                  #:date (second period-date) #:entry 'net-cash-from-financing-activities)
266 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
267 |                                                                                  #:date (second period-date) #:entry 'effect-of-exchange-rate-changes)
268 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
269 |                                                                                  #:date (second period-date) #:entry 'net-change-in-cash-and-equivalents)
270 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
271 |                                                                                  #:date (second period-date) #:entry 'cash-at-beginning-of-period)
272 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
273 |                                                                                  #:date (second period-date) #:entry 'cash-at-end-of-period)
274 |                                                      (cash-flow-statement-figure xexp #:section 'uses-of-funds #:period (first period-date)
275 |                                                                                  #:date (second period-date) #:entry 'diluted-net-eps))
276 |                                          (commit-transaction dbc)
277 |                                          (set! insert-success-counter (add1 insert-success-counter))))
278 |                                      (cartesian-product (list 'annual 'quarterly)
279 |                                                         (list 'fifth-most-recent 'fourth-most-recent 'third-most-recent 'second-most-recent 'most-recent)))]
280 |                           [else (displayln (string-append "Skipping " ticker-symbol " as the data is most likely using the wrong date."))]))))))))
281 | 
282 | (disconnect dbc)
283 | 
284 | (displayln (string-append "Attempted to insert " (number->string insert-counter) " rows. "
285 |                           (number->string insert-success-counter) " were successful. "
286 |                           (number->string insert-failure-counter) " failed."))
287 | 


--------------------------------------------------------------------------------
/estimate-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          html-parsing
  6 |          racket/cmdline
  7 |          racket/list
  8 |          racket/sequence
  9 |          racket/string
 10 |          sxml
 11 |          threading)
 12 | 
 13 | (define (rank xexp)
 14 |   (let* ([zrank-path (cond
 15 |                        [(date<? (folder-date) (date 2024 11 10))
 16 |                         (sxpath '(html (body (@ (equal? (id "home"))))
 17 |                                        (div (@ (equal? (id "main_content"))))
 18 |                                        (div (@ (equal? (id "right_content"))))
 19 |                                        (section (@ (equal? (id "quote_ribbon_v2"))))
 20 |                                        (div 2) (div 1) p))]
 21 |                        [else
 22 |                         (sxpath '(html (body (@ (equal? (id "home"))))
 23 |                                        (div (@ (equal? (id "main_content"))))
 24 |                                        (div (@ (equal? (id "right_content"))))
 25 |                                        (section (@ (equal? (class "quote_page_hero_section"))))
 26 |                                        (section (@ (equal? (id "quote_ribbon_v2"))))
 27 |                                        (div 2) (div 1) p))])]
 28 |          [zrank-node (zrank-path xexp)])
 29 |     (substring (string-trim (second ((select-kids (ntype?? '*text*)) zrank-node))) 2)))
 30 | 
 31 | (define (style-score xexp #:style style-type)
 32 |   (let* ([n (case style-type
 33 |               ['value 1]
 34 |               ['growth 2]
 35 |               ['momentum 3]
 36 |               ['vgm 4])]
 37 |          [x (cond
 38 |               ; Reports downloaded before 2018-10-07 used a slightly different sxpath
 39 |               ; The following code might be helpful for finding these differences in the future:
 40 |               ; 
 41 |               ; (define in-file (open-input-file "/var/local/zacks/estimates/2017-10-26/AA.detailed-estimates.html"))
 42 |               ; (define in-xexp (html->xexp in-file))
 43 |               ; (webscraperhelper '(td (@ (class "alpha")) "Down Last 60 Days") in-xexp)
 44 |               [(date<? (folder-date) (date 2018 10 7))
 45 |                ((sxpath `(html (body (@ (equal? (id "home"))))
 46 |                                (div (@ (equal? (id "main_content"))))
 47 |                                (div (@ (equal? (id "right_content"))))
 48 |                                (section (@ (equal? (id "quote_ribbon_v2"))))
 49 |                                (div 2) (div 2) p (span ,n))) xexp)]
 50 |               [(date<? (folder-date) (date 2020 7 4))
 51 |                ((sxpath `(html (body (@ (equal? (id "home"))))
 52 |                                (div (@ (equal? (id "main_content"))))
 53 |                                (div (@ (equal? (id "right_content"))))
 54 |                                (section (@ (equal? (id "quote_ribbon_v2"))))
 55 |                                (div 2) (div 3) p (span ,n))) xexp)]
 56 |               ; Pipes between the style scores now show up as their own span element, so we have a hack to skip them
 57 |               [(date<? (folder-date) (date 2020 9 20))
 58 |                ((sxpath `(html (body (@ (equal? (id "home"))))
 59 |                                (div (@ (equal? (id "main_content"))))
 60 |                                (div (@ (equal? (id "right_content"))))
 61 |                                (section (@ (equal? (id "quote_ribbon_v2"))))
 62 |                                (div 2) (div 3) p (span ,(- (* n 2) 1)))) xexp)]
 63 |               [(date<? (folder-date) (date 2024 11 10))
 64 |                ((sxpath `(html (body (@ (equal? (id "home"))))
 65 |                                (div (@ (equal? (id "main_content"))))
 66 |                                (div (@ (equal? (id "right_content"))))
 67 |                                (section (@ (equal? (id "quote_ribbon_v2"))))
 68 |                                (div 2) (div 2) p (span ,(- (* n 2) 1)))) xexp)]
 69 |               ; New "quote_page_hero_section" added
 70 |               [else
 71 |                ((sxpath `(html (body (@ (equal? (id "home"))))
 72 |                                (div (@ (equal? (id "main_content"))))
 73 |                                (div (@ (equal? (id "right_content"))))
 74 |                                (section (@ (equal? (class "quote_page_hero_section"))))
 75 |                                (section (@ (equal? (id "quote_ribbon_v2"))))
 76 |                                (div 2) (div 2) p (span ,(- (* n 2) 1)))) xexp)])])
 77 |     (third (first x))))
 78 | 
 79 | (define (estimate-figure xexp #:section section #:period period #:entry entry)
 80 |   (let*-values ([(section-id first-second index-offset)
 81 |                  (case section
 82 |                    ['sales-estimates (values "detailed_earnings_estimates" first 0)]
 83 |                    ['eps-estimates (values "detailed_earnings_estimates" second 1)]
 84 |                    ['eps-revisions (values "agreement_estimate" first 0)]
 85 |                    ['eps-upside (values "quote_upside" first 0)]
 86 |                    ['eps-surprise (values "surprised_reported" first 0)])]
 87 |                 [(col) (case period
 88 |                          ['current-quarter 2]
 89 |                          ['next-quarter 3]
 90 |                          ['current-year 4]
 91 |                          ['next-year 5]
 92 |                          ['last-quarter 2]
 93 |                          ['two-quarters-ago 3]
 94 |                          ['three-quarters-ago 4]
 95 |                          ['four-quarters-ago 5])]
 96 |                 [(row thead-tbody th-td) (case entry
 97 |                                            ['consensus (values 1 `tbody `td)]
 98 |                                            ['count (values 2 `tbody `td)]
 99 |                                            ['recent (values 3 `tbody `td)]
100 |                                            ['high (values (+ 3 index-offset) `tbody `td)]
101 |                                            ['low (values (+ 4 index-offset) `tbody `td)]
102 |                                            ['year-ago (values (+ 5 index-offset) `tbody `td)]
103 |                                            ['up-7 (values 1 `tbody `td)]
104 |                                            ['up-30 (values 2 `tbody `td)]
105 |                                            ['up-60 (values 3 `tbody `td)]
106 |                                            ['down-7 (values 4 `tbody `td)]
107 |                                            ['down-30 (values 5 `tbody `td)]
108 |                                            ['down-60 (values 6 `tbody `td)]
109 |                                            ['most-accurate (values 1 `tbody `td)]
110 |                                            ['date (values 1 `thead `th)]
111 |                                            ['reported (values 1 `tbody `td)]
112 |                                            ['estimate (values 2 `tbody `td)])])
113 |     (~> ((sxpath `(// (section (@ (equal? (id ,section-id))))
114 |                       table ,thead-tbody (tr ,row) (,th-td ,col))) xexp)
115 |         (first-second _)
116 |         (flatten _)
117 |         (last _)
118 |         (string-trim _)
119 |         (string-replace _ "T" "e12")
120 |         (string-replace _ "B" "e9")
121 |         (regexp-replace #rx"^M$" _ "NA") ; no clue what the "M" value means in the estimate cells, but we set it to NA
122 |         (string-replace _ "M" "e6")
123 |         (string-replace _ "(" "")
124 |         (string-replace _ ")" "")
125 |         (string-replace _ "," ""))))
126 | 
127 | (define base-folder (make-parameter "/var/local/zacks/estimates"))
128 | 
129 | (define folder-date (make-parameter (today)))
130 | 
131 | (define db-user (make-parameter "user"))
132 | 
133 | (define db-name (make-parameter "local"))
134 | 
135 | (define db-pass (make-parameter ""))
136 | 
137 | (command-line
138 |  #:program "racket estimate-transform-load.rkt"
139 |  #:once-each
140 |  [("-b" "--base-folder") folder
141 |                          "Zacks estimates base folder. Defaults to /var/local/zacks/estimates"
142 |                          (base-folder folder)]
143 |  [("-d" "--folder-date") date
144 |                          "Zacks estimates folder date. Defaults to today"
145 |                          (folder-date (iso8601->date date))]
146 |  [("-n" "--db-name") name
147 |                      "Database name. Defaults to 'local'"
148 |                      (db-name name)]
149 |  [("-p" "--db-pass") password
150 |                      "Database password"
151 |                      (db-pass password)]
152 |  [("-u" "--db-user") user
153 |                      "Database user name. Defaults to 'user'"
154 |                      (db-user user)])
155 | 
156 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
157 | 
158 | (define insert-counter 0)
159 | (define insert-success-counter 0)
160 | (define insert-failure-counter 0)
161 | 
162 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
163 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".detailed-estimates.html")) (in-directory (current-directory)))])
164 |     (let* ([file-name (path->string p)]
165 |            [ticker-symbol (string-replace (string-replace file-name (path->string (current-directory)) "") ".detailed-estimates.html" "")])
166 |       (call-with-input-file file-name
167 |         (λ (in) (let ([xexp (html->xexp in)])
168 |                   (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to process "
169 |                                                                               ticker-symbol
170 |                                                                               " for date "
171 |                                                                               (~t (folder-date) "yyyy-MM-dd")))
172 |                                                (displayln e)
173 |                                                (rollback-transaction dbc)
174 |                                                (set! insert-failure-counter (add1 insert-failure-counter)))])
175 |                     (set! insert-counter (add1 insert-counter))
176 |                     (start-transaction dbc)
177 |                     (query-exec dbc "
178 | insert into zacks.rank_score
179 | (
180 |   act_symbol,
181 |   date,
182 |   rank,
183 |   value,
184 |   growth,
185 |   momentum,
186 |   vgm
187 | ) values (
188 |   $1,
189 |   $2::text::date,
190 |   $3::text::zacks.rank,
191 |   $4::text::zacks.score,
192 |   $5::text::zacks.score,
193 |   $6::text::zacks.score,
194 |   $7::text::zacks.score
195 | ) on conflict (act_symbol, date) do nothing;
196 | "
197 |                                 ticker-symbol
198 |                                 (~t (folder-date) "yyyy-MM-dd")
199 |                                 (rank xexp)
200 |                                 (style-score xexp #:style 'value)
201 |                                 (style-score xexp #:style 'growth)
202 |                                 (style-score xexp #:style 'momentum)
203 |                                 (style-score xexp #:style 'vgm))
204 |                     (for-each (λ (period)
205 |                                 (query-exec dbc "
206 | insert into zacks.sales_estimate
207 | (
208 |   act_symbol,
209 |   date,
210 |   period,
211 |   period_end_date,
212 |   consensus,
213 |   count,
214 |   high,
215 |   low,
216 |   year_ago
217 | ) values (
218 |   $1,
219 |   $2::text::date,
220 |   case $3
221 |     when 'current-quarter' then 'Current Quarter'::zacks.estimate_period
222 |     when 'next-quarter' then 'Next Quarter'::zacks.estimate_period
223 |     when 'current-year' then 'Current Year'::zacks.estimate_period
224 |     when 'next-year' then 'Next Year'::zacks.estimate_period
225 |   end,
226 |   to_date($4, 'DD/MM/YYYY') + interval '1 month' - interval '1 day',
227 |   case $5
228 |     when 'NA' then NULL
229 |     else $5::decimal
230 |   end,
231 |   case $6
232 |     when 'NA' then NULL
233 |     else $6::smallint
234 |   end,
235 |   case $7
236 |     when 'NA' then NULL
237 |     else $7::decimal
238 |   end,
239 |   case $8
240 |     when 'NA' then NULL
241 |     else $8::decimal
242 |   end,
243 |   case $9
244 |     when 'NA' then NULL
245 |     else $9::decimal
246 |   end
247 | ) on conflict (act_symbol, date, period) do nothing;
248 | "
249 |                                             ticker-symbol
250 |                                             (~t (folder-date) "yyyy-MM-dd")
251 |                                             (symbol->string period)
252 |                                             ; Use eps-estimates date as we sometimes have worse coverage with sales estimates than eps estimates
253 |                                             (string-append "01/" (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'date))
254 |                                             (estimate-figure xexp #:section 'sales-estimates #:period period #:entry 'consensus)
255 |                                             (estimate-figure xexp #:section 'sales-estimates #:period period #:entry 'count)
256 |                                             (estimate-figure xexp #:section 'sales-estimates #:period period #:entry 'high)
257 |                                             (estimate-figure xexp #:section 'sales-estimates #:period period #:entry 'low)
258 |                                             (estimate-figure xexp #:section 'sales-estimates #:period period #:entry 'year-ago))
259 |                                 (query-exec dbc "
260 | insert into zacks.eps_estimate
261 | (
262 |   act_symbol,
263 |   date,
264 |   period,
265 |   period_end_date,
266 |   consensus,
267 |   count,
268 |   recent,
269 |   high,
270 |   low,
271 |   year_ago
272 | ) values (
273 |   $1,
274 |   $2::text::date,
275 |   case $3
276 |     when 'current-quarter' then 'Current Quarter'::zacks.estimate_period
277 |     when 'next-quarter' then 'Next Quarter'::zacks.estimate_period
278 |     when 'current-year' then 'Current Year'::zacks.estimate_period
279 |     when 'next-year' then 'Next Year'::zacks.estimate_period
280 |   end,
281 |   to_date($4, 'DD/MM/YYYY') + interval '1 month' - interval '1 day',
282 |   case $5
283 |     when 'NA' then NULL
284 |     else $5::decimal
285 |   end,
286 |   case $6
287 |     when 'NA' then NULL
288 |     else $6::smallint
289 |   end,
290 |   case $7
291 |     when 'NA' then NULL
292 |     else $7::decimal
293 |   end,
294 |   case $8
295 |     when 'NA' then NULL
296 |     else $8::decimal
297 |   end,
298 |   case $9
299 |     when 'NA' then NULL
300 |     else $9::decimal
301 |   end,
302 |   case $10
303 |     when 'NA' then NULL
304 |     else $10::decimal
305 |   end
306 | ) on conflict (act_symbol, date, period) do nothing;
307 | "
308 |                                             ticker-symbol
309 |                                             (~t (folder-date) "yyyy-MM-dd")
310 |                                             (symbol->string period)
311 |                                             (string-append "01/" (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'date))
312 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'consensus)
313 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'count)
314 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'recent)
315 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'high)
316 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'low)
317 |                                             (estimate-figure xexp #:section 'eps-estimates #:period period #:entry 'year-ago))
318 |                                 (query-exec dbc "
319 | insert into zacks.eps_revision
320 | (
321 |   act_symbol,
322 |   date,
323 |   period,
324 |   period_end_date,
325 |   up_7,
326 |   up_30,
327 |   up_60,
328 |   down_7,
329 |   down_30,
330 |   down_60
331 | ) values (
332 |   $1,
333 |   $2::text::date,
334 |   case $3
335 |     when 'current-quarter' then 'Current Quarter'::zacks.estimate_period
336 |     when 'next-quarter' then 'Next Quarter'::zacks.estimate_period
337 |     when 'current-year' then 'Current Year'::zacks.estimate_period
338 |     when 'next-year' then 'Next Year'::zacks.estimate_period
339 |   end,
340 |   to_date($4, 'DD/MM/YYYY') + interval '1 month' - interval '1 day',
341 |   case $5
342 |     when 'NA' then NULL
343 |     else $5::smallint
344 |   end,
345 |   case $6
346 |     when 'NA' then NULL
347 |     else $6::smallint
348 |   end,
349 |   case $7
350 |     when 'NA' then NULL
351 |     else $7::smallint
352 |   end,
353 |   case $8
354 |     when 'NA' then NULL
355 |     else $8::smallint
356 |   end,
357 |   case $9
358 |     when 'NA' then NULL
359 |     else $9::smallint
360 |   end,
361 |   case $10
362 |     when 'NA' then NULL
363 |     else $10::smallint
364 |   end
365 | ) on conflict (act_symbol, date, period) do nothing;
366 | "
367 |                                             ticker-symbol
368 |                                             (~t (folder-date) "yyyy-MM-dd")
369 |                                             (symbol->string period)
370 |                                             (string-append "01/" (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'date))
371 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'up-7)
372 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'up-30)
373 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'up-60)
374 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'down-7)
375 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'down-30)
376 |                                             (estimate-figure xexp #:section 'eps-revisions #:period period #:entry 'down-60))
377 |                                 (query-exec dbc "
378 | insert into zacks.eps_perception
379 | (
380 |   act_symbol,
381 |   date,
382 |   period,
383 |   period_end_date,
384 |   most_accurate
385 | ) values (
386 |   $1,
387 |   $2::text::date,
388 |   case $3
389 |     when 'current-quarter' then 'Current Quarter'::zacks.estimate_period
390 |     when 'next-quarter' then 'Next Quarter'::zacks.estimate_period
391 |     when 'current-year' then 'Current Year'::zacks.estimate_period
392 |     when 'next-year' then 'Next Year'::zacks.estimate_period
393 |   end,
394 |   to_date($4, 'DD/MM/YYYY') + interval '1 month' - interval '1 day',
395 |   case $5
396 |     when 'NA' then NULL
397 |     else $5::decimal
398 |   end
399 | ) on conflict (act_symbol, date, period) do nothing;
400 | "
401 |                                             ticker-symbol
402 |                                             (~t (folder-date) "yyyy-MM-dd")
403 |                                             (symbol->string period)
404 |                                             (string-append "01/" (estimate-figure xexp #:section 'eps-upside #:period period #:entry 'date))
405 |                                             (estimate-figure xexp #:section 'eps-upside #:period period #:entry 'most-accurate)))
406 |                               (list 'current-quarter 'next-quarter 'current-year 'next-year))
407 |                     (for-each (λ (quarter)
408 |                                 (query-exec dbc "
409 | insert into zacks.eps_history
410 | (
411 |   act_symbol,
412 |   period_end_date,
413 |   reported,
414 |   estimate
415 | ) values (
416 |   $1,
417 |   to_date($2, 'DD/MM/YYYY') + interval '1 month' - interval '1 day',
418 |   case $3
419 |     when 'NA' then NULL
420 |     else $3::decimal
421 |   end,
422 |   case $4
423 |     when 'NA' then NULL
424 |     else $4::decimal
425 |   end
426 | ) on conflict (act_symbol, period_end_date) do nothing;
427 | "
428 |                                             ticker-symbol
429 |                                             (string-append "01/" (estimate-figure xexp #:section 'eps-surprise #:period quarter #:entry 'date))
430 |                                             (estimate-figure xexp #:section 'eps-surprise #:period quarter #:entry 'reported)
431 |                                             (estimate-figure xexp #:section 'eps-surprise #:period quarter #:entry 'estimate)))
432 |                               (list 'last-quarter 'two-quarters-ago 'three-quarters-ago 'four-quarters-ago))
433 |                     (commit-transaction dbc)
434 |                     (set! insert-success-counter (add1 insert-success-counter)))))))))
435 | 
436 | (disconnect dbc)
437 | 
438 | (displayln (string-append "Attempted to insert " (number->string insert-counter) " rows. "
439 |                           (number->string insert-success-counter) " were successful. "
440 |                           (number->string insert-failure-counter) " failed."))
441 | 


--------------------------------------------------------------------------------
/balance-sheet-transform-load.rkt:
--------------------------------------------------------------------------------
  1 | #lang racket/base
  2 | 
  3 | (require db
  4 |          gregor
  5 |          gregor/period
  6 |          html-parsing
  7 |          racket/cmdline
  8 |          racket/list
  9 |          racket/sequence
 10 |          racket/string
 11 |          sxml
 12 |          threading)
 13 | 
 14 | (define (balance-sheet-figure xexp #:section section #:period period #:date date #:entry entry)
 15 |   (let*-values ([(table-id) (case section
 16 |                               ['assets 1]
 17 |                               ['liabilities 2]
 18 |                               ['equity 3])]
 19 |                 [(section-id) (case period
 20 |                                 ['annual "annual_income_statement"]
 21 |                                 ['quarterly "quarterly_income_statement"])]
 22 |                 [(col) (case date
 23 |                          ['most-recent 2]
 24 |                          ['second-most-recent 3]
 25 |                          ['third-most-recent 4]
 26 |                          ['fourth-most-recent 5]
 27 |                          ['fifth-most-recent 6])]
 28 |                 [(row thead-tbody th-td) (case entry
 29 |                                            ['date (values 1 `thead `th)]
 30 |                                            ['cash-and-equivalents (values 2 `tbody `td)]
 31 |                                            ['receivables (values 3 `tbody `td)]
 32 |                                            ['notes-receivable (values 4 `tbody `td)]
 33 |                                            ['inventories (values 5 `tbody `td)]
 34 |                                            ['other-current-assets (values 6 `tbody `td)]
 35 |                                            ['total-current-assets (values 7 `tbody `td)]
 36 |                                            ['net-property-and-equipment (values 8 `tbody `td)]
 37 |                                            ['investments-and-advances (values 9 `tbody `td)]
 38 |                                            ['other-non-current-assets (values 10 `tbody `td)]
 39 |                                            ['deferred-charges (values 11 `tbody `td)]
 40 |                                            ['intangibles (values 12 `tbody `td)]
 41 |                                            ['deposits-and-other-assets (values 13 `tbody `td)]
 42 |                                            ['total-assets (values 14 `tbody `td)]
 43 |                                            ['notes-payable (values 1 `tbody `td)]
 44 |                                            ['accounts-payable (values 2 `tbody `td)]
 45 |                                            ['current-portion-long-term-debt (values 3 `tbody `td)]
 46 |                                            ['current-portion-capital-leases (values 4 `tbody `td)]
 47 |                                            ['accrued-expenses (values 5 `tbody `td)]
 48 |                                            ['income-taxes-payable (values 6 `tbody `td)]
 49 |                                            ['other-current-liabilities (values 7 `tbody `td)]
 50 |                                            ['total-current-liabilities (values 8 `tbody `td)]
 51 |                                            ['mortgages (values 9 `tbody `td)]
 52 |                                            ['deferred-taxes-or-income (values 10 `tbody `td)]
 53 |                                            ['convertible-debt (values 11 `tbody `td)]
 54 |                                            ['long-term-debt (values 12 `tbody `td)]
 55 |                                            ['non-current-capital-leases (values 13 `tbody `td)]
 56 |                                            ['other-non-current-liabilities (values 14 `tbody `td)]
 57 |                                            ['minority-interest (values 15 `tbody `td)]
 58 |                                            ['total-liabilities (values 16 `tbody `td)]
 59 |                                            ['preferred-stock (values 1 `tbody `td)]
 60 |                                            ['common-stock (values 2 `tbody `td)]
 61 |                                            ['capital-surplus (values 3 `tbody `td)]
 62 |                                            ['retained-earnings (values 4 `tbody `td)]
 63 |                                            ['other-equity (values 5 `tbody `td)]
 64 |                                            ['treasury-stock (values 6 `tbody `td)]
 65 |                                            ['total-equity (values 7 `tbody `td)]
 66 |                                            ['total-liabilities-and-equity (values 8 `tbody `td)]
 67 |                                            ['shares-outstanding (values 10 `tbody `td)]
 68 |                                            ['book-value-per-share (values 11 `tbody `td)])])
 69 |     (~>
 70 |      ; (define in-file (open-input-file "/var/local/zacks/balance-sheet/2018-05-08/AA.balance-sheet.html"))
 71 |      ; (define in-xexp (html->xexp in-file))
 72 |      ; (webscraperhelper '(td (@ (class "alpha")) "Preferred Stock") in-xexp)
 73 |      ((sxpath `(// (div (@ (equal? (id ,section-id))))
 74 |                    (table ,table-id) ,thead-tbody (tr ,row) (,th-td ,col))) xexp)
 75 |      (flatten _)
 76 |      (last _)
 77 |      (string-trim _)
 78 |      (string-replace _ "," ""))))
 79 | 
 80 | (define base-folder (make-parameter "/var/local/zacks/balance-sheet"))
 81 | 
 82 | (define folder-date (make-parameter (today)))
 83 | 
 84 | (define db-user (make-parameter "user"))
 85 | 
 86 | (define db-name (make-parameter "local"))
 87 | 
 88 | (define db-pass (make-parameter ""))
 89 | 
 90 | (command-line
 91 |  #:program "racket balance-sheet-transform-load.rkt"
 92 |  #:once-each
 93 |  [("-b" "--base-folder") folder
 94 |                          "Zacks balance sheet base folder. Defaults to /var/local/zacks/balance-sheet"
 95 |                          (base-folder folder)]
 96 |  [("-d" "--folder-date") date
 97 |                          "Zacks balance sheet folder date. Defaults to today"
 98 |                          (folder-date (iso8601->date date))]
 99 |  [("-n" "--db-name") name
100 |                      "Database name. Defaults to 'local'"
101 |                      (db-name name)]
102 |  [("-p" "--db-pass") password
103 |                      "Database password"
104 |                      (db-pass password)]
105 |  [("-u" "--db-user") user
106 |                      "Database user name. Defaults to 'user'"
107 |                      (db-user user)])
108 | 
109 | (define dbc (postgresql-connect #:user (db-user) #:database (db-name) #:password (db-pass)))
110 | 
111 | (define insert-counter 0)
112 | (define insert-success-counter 0)
113 | (define insert-failure-counter 0)
114 | 
115 | (parameterize ([current-directory (string-append (base-folder) "/" (~t (folder-date) "yyyy-MM-dd") "/")])
116 |   (for ([p (sequence-filter (λ (p) (string-contains? (path->string p) ".balance-sheet.html")) (in-directory (current-directory)))])
117 |     (let* ([file-name (path->string p)]
118 |            [ticker-symbol (string-replace (string-replace file-name (path->string (current-directory)) "") ".balance-sheet.html" "")])
119 |       (call-with-input-file file-name
120 |         (λ (in) (let ([xexp (html->xexp in)])
121 |                   ; we assume that if we have an updated value that is very close to our extract date that the whole set of data is bad.
122 |                   (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to extract a date from " ticker-symbol)))])
123 |                     (cond [(< 15 (period-ref (date-period-between (parse-date (balance-sheet-figure xexp #:section 'assets #:period 'quarterly
124 |                                                                                                     #:date 'most-recent #:entry 'date)
125 |                                                                               "M/dd/yyyy")
126 |                                                                   (folder-date)
127 |                                                                   (list 'days))
128 |                                              'days))
129 |                            (for-each (λ (period-date)
130 |                                        (with-handlers ([exn:fail? (λ (e) (displayln (string-append "Failed to insert data for "
131 |                                                                                                    ticker-symbol
132 |                                                                                                    " for date "
133 |                                                                                                    (~t (folder-date) "yyyy-MM-dd")))
134 |                                                                      (displayln e)
135 |                                                                      (rollback-transaction dbc)
136 |                                                                      (set! insert-failure-counter (add1 insert-failure-counter)))])
137 |                                          (set! insert-counter (add1 insert-counter))
138 |                                          (start-transaction dbc)
139 |                                          (query-exec dbc "
140 | -- We use the common table expression below in order to check if we're receiving
141 | -- bad values from Zacks. When the new fiscal year occurs, Zacks apparently has a
142 | -- bug where values from the prior year are copied into the new year column. We
143 | -- guard against that by checking data against the previous year's data and not
144 | -- inserting it if it is exactly the same by trying to insert a NULL act_symbol.
145 | -- A better way might be with a stored procedure that can return a meaningful error.
146 | with should_not_insert as (
147 |   select
148 |     bool_and(
149 |       cash_and_equivalents = $4::text::decimal * 1e6 and
150 |       receivables = $5::text::decimal * 1e6 and
151 |       notes_receivable = $6::text::decimal * 1e6 and
152 |       inventories = $7::text::decimal * 1e6 and
153 |       other_current_assets = $8::text::decimal * 1e6 and
154 |       total_current_assets = $9::text::decimal * 1e6 and
155 |       net_property_and_equipment = $10::text::decimal * 1e6 and
156 |       investments_and_advances = $11::text::decimal * 1e6 and
157 |       other_non_current_assets = $12::text::decimal * 1e6 and
158 |       deferred_charges = $13::text::decimal * 1e6 and
159 |       intangibles = $14::text::decimal * 1e6 and
160 |       deposits_and_other_assets = $15::text::decimal * 1e6 and
161 |       total_assets = $16::text::decimal * 1e6
162 |     ) as sni
163 |   from
164 |     zacks.balance_sheet_assets
165 |   where
166 |     act_symbol = $1 and
167 |     case $3
168 |       when 'annual' then
169 |         period = 'Year'::zacks.statement_period and
170 |         date = $2::text::date - interval '1 year'
171 |       when 'quarterly' then
172 |         period = 'Quarter'::zacks.statement_period and
173 |         date = $2::text::date + interval '1 day' - interval '3 months' - interval '1 day'
174 |     end
175 | )
176 | insert into zacks.balance_sheet_assets
177 | (
178 |   act_symbol,
179 |   date,
180 |   period,
181 |   cash_and_equivalents,
182 |   receivables,
183 |   notes_receivable,
184 |   inventories,
185 |   other_current_assets,
186 |   total_current_assets,
187 |   net_property_and_equipment,
188 |   investments_and_advances,
189 |   other_non_current_assets,
190 |   deferred_charges,
191 |   intangibles,
192 |   deposits_and_other_assets,
193 |   total_assets
194 | ) values (
195 |   case (select sni from should_not_insert)
196 |     when true then NULL
197 |     else $1
198 |   end,
199 |   $2::text::date,
200 |   case $3
201 |     when 'annual' then 'Year'::zacks.statement_period
202 |     when 'quarterly' then 'Quarter'::zacks.statement_period
203 |   end,
204 |   $4::text::decimal * 1e6,
205 |   $5::text::decimal * 1e6,
206 |   $6::text::decimal * 1e6,
207 |   $7::text::decimal * 1e6,
208 |   $8::text::decimal * 1e6,
209 |   $9::text::decimal * 1e6,
210 |   $10::text::decimal * 1e6,
211 |   $11::text::decimal * 1e6,
212 |   $12::text::decimal * 1e6,
213 |   $13::text::decimal * 1e6,
214 |   $14::text::decimal * 1e6,
215 |   $15::text::decimal * 1e6,
216 |   $16::text::decimal * 1e6
217 | ) on conflict (act_symbol, date, period) do nothing;
218 | "
219 |                                                      ticker-symbol
220 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
221 |                                                                            #:entry 'date)
222 |                                                      (symbol->string (first period-date))
223 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
224 |                                                                            #:entry 'cash-and-equivalents)
225 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
226 |                                                                            #:entry 'receivables)
227 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
228 |                                                                            #:entry 'notes-receivable)
229 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
230 |                                                                            #:entry 'inventories)
231 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
232 |                                                                            #:entry 'other-current-assets)
233 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
234 |                                                                            #:entry 'total-current-assets)
235 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
236 |                                                                            #:entry 'net-property-and-equipment)
237 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
238 |                                                                            #:entry 'investments-and-advances)
239 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
240 |                                                                            #:entry 'other-non-current-assets)
241 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
242 |                                                                            #:entry 'deferred-charges)
243 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
244 |                                                                            #:entry 'intangibles)
245 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
246 |                                                                            #:entry 'deposits-and-other-assets)
247 |                                                      (balance-sheet-figure xexp #:section 'assets #:period (first period-date) #:date (second period-date)
248 |                                                                            #:entry 'total-assets))
249 |                                          (query-exec dbc "
250 | -- See note in balance_sheet_assets query above.
251 | with should_not_insert as (
252 |   select
253 |     bool_and(
254 |       notes_payable = $4::text::decimal * 1e6 and
255 |       accounts_payable = $5::text::decimal * 1e6 and
256 |       current_portion_long_term_debt = $6::text::decimal * 1e6 and
257 |       current_portion_capital_leases = $7::text::decimal * 1e6 and
258 |       accrued_expenses = $8::text::decimal * 1e6 and
259 |       income_taxes_payable = $9::text::decimal * 1e6 and
260 |       other_current_liabilities = $10::text::decimal * 1e6 and
261 |       total_current_liabilities = $11::text::decimal * 1e6 and
262 |       mortgages = $12::text::decimal * 1e6 and
263 |       deferred_taxes_or_income = $13::text::decimal * 1e6 and
264 |       convertible_debt = $14::text::decimal * 1e6 and
265 |       long_term_debt = $15::text::decimal * 1e6 and
266 |       non_current_capital_leases = $16::text::decimal * 1e6 and
267 |       other_non_current_liabilities = $17::text::decimal * 1e6 and
268 |       minority_interest = $18::text::decimal * 1e6 and
269 |       total_liabilities = $19::text::decimal * 1e6
270 |     ) as sni
271 |   from
272 |     zacks.balance_sheet_liabilities
273 |   where
274 |     act_symbol = $1 and
275 |     case $3
276 |       when 'annual' then
277 |         period = 'Year'::zacks.statement_period and
278 |         date = $2::text::date - interval '1 year'
279 |       when 'quarterly' then
280 |         period = 'Quarter'::zacks.statement_period and
281 |         date = $2::text::date + interval '1 day' - interval '3 months' - interval '1 day'
282 |     end
283 | )
284 | insert into zacks.balance_sheet_liabilities
285 | (
286 |   act_symbol,
287 |   date,
288 |   period,
289 |   notes_payable,
290 |   accounts_payable,
291 |   current_portion_long_term_debt,
292 |   current_portion_capital_leases,
293 |   accrued_expenses,
294 |   income_taxes_payable,
295 |   other_current_liabilities,
296 |   total_current_liabilities,
297 |   mortgages,
298 |   deferred_taxes_or_income,
299 |   convertible_debt,
300 |   long_term_debt,
301 |   non_current_capital_leases,
302 |   other_non_current_liabilities,
303 |   minority_interest,
304 |   total_liabilities
305 | ) values (
306 |   case (select sni from should_not_insert)
307 |     when true then NULL
308 |     else $1
309 |   end,
310 |   $2::text::date,
311 |   case $3
312 |     when 'annual' then 'Year'::zacks.statement_period
313 |     when 'quarterly' then 'Quarter'::zacks.statement_period
314 |   end,
315 |   $4::text::decimal * 1e6,
316 |   $5::text::decimal * 1e6,
317 |   $6::text::decimal * 1e6,
318 |   $7::text::decimal * 1e6,
319 |   $8::text::decimal * 1e6,
320 |   $9::text::decimal * 1e6,
321 |   $10::text::decimal * 1e6,
322 |   $11::text::decimal * 1e6,
323 |   $12::text::decimal * 1e6,
324 |   $13::text::decimal * 1e6,
325 |   $14::text::decimal * 1e6,
326 |   $15::text::decimal * 1e6,
327 |   $16::text::decimal * 1e6,
328 |   $17::text::decimal * 1e6,
329 |   $18::text::decimal * 1e6,
330 |   $19::text::decimal * 1e6
331 | ) on conflict (act_symbol, date, period) do nothing;
332 | "
333 |                                                      ticker-symbol
334 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
335 |                                                                            #:entry 'date)
336 |                                                      (symbol->string (first period-date))
337 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
338 |                                                                            #:entry 'notes-payable)
339 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
340 |                                                                            #:entry 'accounts-payable)
341 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
342 |                                                                            #:entry 'current-portion-long-term-debt)
343 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
344 |                                                                            #:entry 'current-portion-capital-leases)
345 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
346 |                                                                            #:entry 'accrued-expenses)
347 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
348 |                                                                            #:entry 'income-taxes-payable)
349 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
350 |                                                                            #:entry 'other-current-liabilities)
351 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
352 |                                                                            #:entry 'total-current-liabilities)
353 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
354 |                                                                            #:entry 'mortgages)
355 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
356 |                                                                            #:entry 'deferred-taxes-or-income)
357 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
358 |                                                                            #:entry 'convertible-debt)
359 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
360 |                                                                            #:entry 'long-term-debt)
361 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
362 |                                                                            #:entry 'non-current-capital-leases)
363 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
364 |                                                                            #:entry 'other-non-current-liabilities)
365 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
366 |                                                                            #:entry 'minority-interest)
367 |                                                      (balance-sheet-figure xexp #:section 'liabilities #:period (first period-date) #:date (second period-date)
368 |                                                                            #:entry 'total-liabilities))
369 |                                          (query-exec dbc "
370 | -- See note in balance_sheet_assets query above.
371 | with should_not_insert as (
372 |   select
373 |     bool_and(
374 |       preferred_stock = $4::text::decimal * 1e6 and
375 |       common_stock = $5::text::decimal * 1e6 and
376 |       capital_surplus = $6::text::decimal * 1e6 and
377 |       retained_earnings = $7::text::decimal * 1e6 and
378 |       other_equity = $8::text::decimal * 1e6 and
379 |       treasury_stock = $9::text::decimal * 1e6 and
380 |       total_equity = $10::text::decimal * 1e6 and
381 |       total_liabilities_and_equity = $11::text::decimal * 1e6 and
382 |       shares_outstanding = $12::text::decimal * 1e6 and
383 |       book_value_per_share = $13::text::decimal
384 |     ) as sni
385 |   from
386 |     zacks.balance_sheet_equity
387 |   where
388 |     act_symbol = $1 and
389 |     case $3
390 |       when 'annual' then
391 |         period = 'Year'::zacks.statement_period and
392 |         date = $2::text::date - interval '1 year'
393 |       when 'quarterly' then
394 |         period = 'Quarter'::zacks.statement_period and
395 |         date = $2::text::date + interval '1 day' - interval '3 months' - interval '1 day'
396 |     end
397 | )
398 | insert into zacks.balance_sheet_equity
399 | (
400 |   act_symbol,
401 |   date,
402 |   period,
403 |   preferred_stock,
404 |   common_stock,
405 |   capital_surplus,
406 |   retained_earnings,
407 |   other_equity,
408 |   treasury_stock,
409 |   total_equity,
410 |   total_liabilities_and_equity,
411 |   shares_outstanding,
412 |   book_value_per_share
413 | ) values (
414 |   case (select sni from should_not_insert)
415 |     when true then NULL
416 |     else $1
417 |   end,
418 |   $2::text::date,
419 |   case $3
420 |     when 'annual' then 'Year'::zacks.statement_period
421 |     when 'quarterly' then 'Quarter'::zacks.statement_period
422 |   end,
423 |   $4::text::decimal * 1e6,
424 |   $5::text::decimal * 1e6,
425 |   $6::text::decimal * 1e6,
426 |   $7::text::decimal * 1e6,
427 |   $8::text::decimal * 1e6,
428 |   $9::text::decimal * 1e6,
429 |   $10::text::decimal * 1e6,
430 |   $11::text::decimal * 1e6,
431 |   $12::text::decimal * 1e6,
432 |   $13::text::decimal
433 | ) on conflict (act_symbol, date, period) do nothing;
434 | "
435 |                                                      ticker-symbol
436 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
437 |                                                                            #:entry 'date)
438 |                                                      (symbol->string (first period-date))
439 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
440 |                                                                            #:entry 'preferred-stock)
441 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
442 |                                                                            #:entry 'common-stock)
443 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
444 |                                                                            #:entry 'capital-surplus)
445 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
446 |                                                                            #:entry 'retained-earnings)
447 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
448 |                                                                            #:entry 'other-equity)
449 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
450 |                                                                            #:entry 'treasury-stock)
451 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
452 |                                                                            #:entry 'total-equity)
453 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
454 |                                                                            #:entry 'total-liabilities-and-equity)
455 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
456 |                                                                            #:entry 'shares-outstanding)
457 |                                                      (balance-sheet-figure xexp #:section 'equity #:period (first period-date) #:date (second period-date)
458 |                                                                            #:entry 'book-value-per-share))
459 |                                          (commit-transaction dbc)
460 |                                          (set! insert-success-counter (add1 insert-success-counter))))
461 |                                      (cartesian-product (list 'annual 'quarterly)
462 |                                                         (list 'fifth-most-recent 'fourth-most-recent 'third-most-recent 'second-most-recent 'most-recent)))]
463 |                           [else (displayln (string-append "Skipping " ticker-symbol " as the data is most likely using the wrong date."))]))))))))
464 | 
465 | (disconnect dbc)
466 | 
467 | (displayln (string-append "Attempted to insert " (number->string insert-counter) " rows. "
468 |                           (number->string insert-success-counter) " were successful. "
469 |                           (number->string insert-failure-counter) " failed."))
470 | 


--------------------------------------------------------------------------------