├── .gitignore ├── .vscode └── launch.json ├── CHANGELOG.md ├── README.md ├── npMediaSave.rb ├── npTools-create_event.gif ├── npTools.rb └── tidyClippings.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /test/tmp/ 10 | /test/version_tmp/ 11 | /tmp/ 12 | 13 | # Used by dotenv library to load environment variables. 14 | # .env 15 | 16 | ## Specific to RubyMotion: 17 | .dat* 18 | .repl_history 19 | build/ 20 | *.bridgesupport 21 | build-iPhoneOS/ 22 | build-iPhoneSimulator/ 23 | 24 | ## Specific to RubyMotion (use of CocoaPods): 25 | # 26 | # We recommend against adding the Pods directory to your .gitignore. However 27 | # you should judge for yourself, the pros and cons are mentioned at: 28 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 29 | # 30 | # vendor/Pods/ 31 | 32 | ## Documentation cache and generated files: 33 | /.yardoc/ 34 | /_yardoc/ 35 | /doc/ 36 | /rdoc/ 37 | 38 | ## Environment normalization: 39 | /.bundle/ 40 | /vendor/bundle 41 | /lib/bundler/man/ 42 | 43 | # for a library or gem, you might want to ignore these files since the code is 44 | # intended to run in multiple environments; otherwise, check them in: 45 | # Gemfile.lock 46 | # .ruby-version 47 | # .ruby-gemset 48 | 49 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 50 | .rvmrc 51 | .solargraph.yml 52 | .vscode/settings.json 53 | .vscode/launch.json 54 | *.code-workspace 55 | .DS_Store 56 | npSave.rb 57 | .history/**/*.* 58 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Debug Local File", 9 | "type": "Ruby", 10 | "request": "launch", 11 | "program": "${workspaceRoot}/main.rb" 12 | }, 13 | { 14 | "name": "Listen for rdebug-ide", 15 | "type": "Ruby", 16 | "request": "attach", 17 | "remoteHost": "127.0.0.1", 18 | "remotePort": "1234", 19 | "remoteWorkspaceRoot": "${workspaceRoot}" 20 | // "program": ".../file.rb" 21 | // "args": ["...", "..."] 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG 2 | As always, `npTools -h` to see the full list of current options. 3 | 4 | ## v2.4.1, 2023-01-26 5 | - [Added] Support for moving lines when its checklist is completed (extending `-t` feature). 6 | 7 | ## v2.4.0, 2022-12-31 8 | - [Added] function to remove `@done(...)` markers from completed checklist items (introduced in NotePlan 3.8). This is triggered by new command line option `-r`. 9 | 10 | ## v2.3.1, 2022-10-03 11 | - [Fixed] bug with improperly closed frontmatter 12 | 13 | ## v2.3.0, 2022-06-26 14 | - [Added] supports new Weekly notes, available with NotePlan 3.6.0. 15 | 16 | ## v2.2.2, 2022-05-17 17 | - [Change] Removed the code to make new notes where linked notes aren't found, as it was unreliable. It will now simply warn the user. 18 | 19 | ## v2.2.0, 2022-05-10 20 | - [Add] Adds the `--movecomplete` option to move a task from a calendar note to the project note indicated by [[note#title]] but only when the task has been completed. (The earlier '--move' option remains; this does the same, but operates whether or not the task has been completed.) 21 | - [Add] Understands notes with titles in the frontmatter (added NP v3.4.x) 22 | - [Change] Removed `--archive` option, as it isn't ready for use. (It was already turned off by default.) 23 | - [Change] Tidy up logging 24 | 25 | ## v2.1.0, 31.10.2021 26 | - [Change] The @repeat(..) function now creates the next copy of the repeated task without using the scheduled indicator `[>]` but the regular open task indicator `[ ]`. 27 | 28 | ## v2.0.0, 31.10.2021 29 | - [Change] Change to using `#!/usr/bin/env ruby` at the start of scripts to make it easier to pick up whatever ruby installation is the user's preference. 30 | 31 | ## v1.9.7, 28.2.2021 32 | - [Improve] The place where lines moved from daily notes are inserted is now smarter still. 33 | - [Fix] Bugs introduced in refactoring regexes in last release. 34 | 35 | ## v1.9.4, 23.1.2021 36 | - [Improve] The place where lines moved from daily notes are inserted is now smarter [Issue 39]. Note this removes the NUM_HEADER_LINES user-settable variable. 37 | 38 | ## v1.9.3, 23.1.2021 39 | - [Fix] Fixed cosmetic bug in moving titled section from daily notes to a project note 40 | - [Fix] Removed stray `>dates` when creating new `@repeat(...)`s 41 | 42 | ## v1.9.2, 14.2.2021 43 | - [New] Now allow for `>date` events to be moved to from one daily to the one it points to. This has to be activated with the `-m` option, as it will be quite a significant change for some users. 44 | 45 | ## v1.9.1, 18.1.2021 46 | - [Fix] Fixed bug in file-matching logic [thanks to Dimitry for reporting] 47 | - [Fix] Fixed bug that stopped date pattern matching in calendar files [thanks to Dimitry for reporting] 48 | - [Improve] Clarified the README for how dates are defined when using #create_event. 49 | 50 | ## v1.9.0, 16.1.2021 51 | - [Add] Command line option `-c` to override how many hours to look back to find changed notes to process 52 | - [Improve] Allow more types of time spec when creating events (e.g. "2.45PM" or "3.15-5.00" 53 | - [Improve] For template dates be more discriminating about where to find dates to match on, so now ignores dates embedded in certain URLs. 54 | - [Fix] spacing around processed template dates (e.g. {3d}) 55 | - [Fix] removing headers with empty sections now won't remove header if next content is a lower-level header 56 | - [Clean up] Remove some obsolete code and add some more logging 57 | 58 | ## v1.8.6 8.1.2021 59 | - [Fix] Allow event creation to work with "Language & Region" settings that use a 12-hour not a 24-hour clock [Issue 37] 60 | 61 | ## v1.8.6 8.1.2021 62 | - [Add] Works with tasks that use the `- ` and `- [ ]` markers, as well as `* ` and `* [ ]` [Issue 24] 63 | 64 | ## v1.8.5 8.1.2021 65 | - [Add] For event creation can now also specify time patterns of form 4-6PM and just 3-4 66 | - [Add] Extend logging to see why some files aren't matching for a user 67 | 68 | ## v1.8.4 8.1.2021 69 | - [Add] Add a second custom date match style for base dates to use in date offset patterns. See README for RE_DATE_OFFSET_CUSTOM. 70 | - [Change] The built-in date match style for base dates is now NotePlan's usual YYYY-MM-DD, not DD-MM-YYYY etc. 71 | 72 | ## v1.8.3 6.1.2021 73 | - [Improve] Can now use simpler '3PM' type of time spec when creating events. 74 | 75 | ## v1.8.2. 2.1.2021 76 | - [Improve] Can now customise the `#create_event` tag used to trigger creating events. Also opens the selected Calendar app if needed first, and reduces the delay if not. [Issue 36] 77 | 78 | ## v1.8.1. 23.12.2020 79 | - [Improve] Can now add location to created events, and copies any description from following indented lines. [Issue 36] 80 | 81 | ## v1.8.0. 21.12.2020 82 | - [New] Add ability to create events in the Calendar, based on time-blocking syntax. See README for more details. [Issue 36] 83 | 84 | ## v1.7.4. 19.12.2020 85 | - [Improve] Clarify documentation to show all sorts of lines in daily notes are moved to the mentioned [[note title]], not just task or header lines. Also loosened unnecessarily strict regexes used here. 86 | 87 | ## v1.7.3. 10.12.2020 88 | - [New] Allow use of weekdays in repeats and template dates (using 'b' rather than usual 'd' for days) [Issue 32] 89 | 90 | ## v1.7.2, 5.12.2020 91 | - [Improve] Extended the command line option --skiptoday to allow comma-separated list of notes to ignore [thanks to @BMStroh PR32] 92 | 93 | ## 1.7.1, 26.11.2020 94 | - [New] add --skipfile=file option to ignore particular files [thanks to @BMStroh, issue 30] 95 | - [Fix] Blank headers at EOF not removed [thanks to @BMStroh, PR31] 96 | 97 | ## 1.7.0, 19.11.2020 98 | - [New] where the note for a `[[Note link]]` doesn't exist, it is created in the top-level Notes folder first 99 | 100 | ## v1.6.1, 13.11.2020 101 | - [New] remove [>] tasks from calendar notes, as there will be a duplicate (whether or not the 'Append links when scheduling' option is set or not) 102 | 103 | ## v1.6, 13.11.2020 104 | - [New] Added the command line info for --skiptoday [thanks to @BMStroh, PR28] 105 | - [Improve] Make the configuration easier for first time users [thanks to @BMStroh, PR27] 106 | 107 | ## v1.5.1, 3.11.2020 108 | - [Change] Now default to using the sandbox location for CloudKit storage (change from NotePlan 3.0.15 beta) 109 | - [Fix] Calendar files apparently disappearing if the default file extension is set to .md 110 | 111 | ## v1.5.0, 25.10.2020 112 | - [New] Remove empty header lines and empty header sections 113 | 114 | ## v1.4.9, 17.10.2020 115 | - [New] Add -q (--quiet) option to suppress all output apart from errors 116 | 117 | ## v1.4.8, 23.9.2020 118 | - [Improve] Handling of edge case where there are two identically-named notes in different sub-folders. When moving a task to them, pick the most recently note to move it to. (issue 21) 119 | 120 | ## v1.4.7, 20.9.2020 121 | - [Fix] Improve finding files with .md as well as .txt extensions, as well as more smartly handling supplied filename patterns 122 | 123 | ## v1.4.6, 19.8.2020 124 | - [Improve] Ignore empty NotePlan data files (issue 12), and simplify file-glob coding to ignore @Archive and @Trash sub-directories 125 | 126 | ## v1.4.5, 19.8.2020 127 | - [Fix] nil error in moving tasks to [[Note]] (issue 19) 128 | 129 | ## v1.4.4, 19.8.2020 130 | - [New] Allow for future NP change to allow .md files not just .txt files (issue 20) 131 | 132 | ## v1.4.3, 2.8.2020 133 | - [Fix] Error in calculation of yearly repeats (issue 18) 134 | 135 | ## v1.4.2, 1.8.2020 136 | - [Change] allow @done(date) to be tided up when time has AM/PM suffix (issue 17) 137 | 138 | ## v1.4.1, 1.8.2020 139 | - [New] add new --noarchive option (issue 16) 140 | - [New] add new --keepscheduled option (issue 14,15) 141 | 142 | ## v1.4.0, 26.7.2020 143 | - [Change] Script now called `npTools` 144 | - [Improve] Significant improvements to documentation 145 | 146 | ## v1.3.0, 19.7.2020 147 | - [New] Make work with CloudKit storage, available for NP v3 beta (issue 11), 148 | 149 | ## v1.2.8, 13.2020 150 | - [Fix] infinite loop on missing note (issue 8) 151 | 152 | ## v1.2.6, 8.6.2020 153 | - [New] remove empty trailing lines (issue 10) 154 | 155 | ## v1.2.4, 2.5.2020 156 | - [New] also move headings with a [[Note]] marker and all its child tasks, notes and comments (issue 6) 157 | 158 | ## v1.2, 1.5.2020 159 | - [New] add generation of @repeat-ed tasks (issue 2) 160 | - [Improve] documentation 161 | 162 | ## v1.1, 16.3.2020 163 | - [New] add ability to find and clean notes in folders (from NP v2.4) (issue 1) 164 | - [Improve] file error handling 165 | 166 | ## v1.0, 28.2.2020 167 | - [New] added first set of command line options (-h, -v, -w) 168 | - [Change] date offsets are now ignored in a section with a heading that includes a #template hashtag. 169 | 170 | ## v0.6.9, 26.2.2020 171 | * [New] changes any mentions of date offset patterns (e.g. {-10d}, {+2w}, {-3m}) to being scheduled dates (e.g. >2020-02-27), if it can find a DD-MM-YYYY date pattern in the previous markdown heading 172 | 173 | ## v0.6.7, 24.2.2020 174 | * [New] adds colouration of output (using https://github.com/fazibear/colorize) 175 | * [Change] move open and now closed tasks with [[Note]] mentions 176 | 177 | ## v0.6.0, 27.11.2019 178 | - [New] remove a set of user-specified tags from @done tasks, via constant `TagsToRemove` 179 | 180 | ## 0.5.0, 26.11.2019 181 | Initial commit to GitHub repository. Already does the following cleaning up: 182 | 183 | - removes the time component of any @done() mentions that NP automatically adds 184 | - removes #waiting or #high tags from @done tasks 185 | - remove any lines with just * or - 186 | - moves any calendar entries with [[Note link]] in it to that note, after the header section. 187 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NotePlan Tools 2 | 3 | **Note: All of the capability of this script is now available through my NotePlan Plugins available directly in the app. Please see [this overview](https://help.noteplan.co/article/65-commandbar-plugins), and the individual [plugins' details in the GitHub repository](https://github.com/NotePlan/plugins). In particular see 'Tidy Up', 'Repeat Extensions' and 'Event Helpers'.** 4 | 5 | `npTools.rb` is a Ruby script that adds functionality to the [NotePlan app](https://noteplan.co/). Particularly when run frequently, this provides a more flexible system for repeating tasks, allows for due dates to be expressed as offsets and therefore templates, moves items from Daily files to Note files, and creates events. It incorporates an earlier script to 'clean' or tidy up NotePlan's data files. 6 | 7 | Each time the script runs, it does a number of things, explained in each section: 8 | 9 | ### Tidy up 10 | 11 | It **tidies up** data files, by: 12 | 1. removing the time part of any `@done(...)` mentions that NotePlan automatically adds when the 'Append Completion Date' option is on. 13 | 2. removing any `@done(...)` mentions on completed Checklist items that NotePlan automatically adds when the 'Append Completion Date' option is on. (From NP v3.8) 14 | 3. removing `#waiting` or `#high` tags or `] task`) items in calendar files (as they've been copied to a new day) 16 | 5. removing any lines with just `* `or `-` or starting `#`s 17 | 6. removing header lines without any content before the next header line of the same or higher level (i.e. fewer `#`s) 18 | 7. removing any multiple consecutive blank lines. 19 | 20 | ### Moves Daily (Calendar) note items 21 | (From v1.9.2) The script can now move any Daily note entries with a **`>date`** in it to the mentioned Daily note. To do so requires turning on through the `-m` option. 22 | 23 | In more detail: 24 | - where the line is a heading, it moves the heading and all following lines until a blank line, or the next heading of the same level 25 | - where the line isn't a heading, it moves the line and any following indented lines (optionally terminated by a blank line) 26 | - the lines are inserted after a section heading (e.g. '### Tasks') as defined in the DAILY_TASKS_SECTION_NAME constant (or after header if this is blank). (To configure this constant, see below.) 27 | 28 | NB: This only operates from Daily (Calendar) notes; it therefore doesn't interfere with **linking and back-linking** between main notes. 29 | 30 | ### Files Daily (Calendar) note tasks 31 | The script can move any Daily note tasks with **a `[[Note title]]`** in it to the mentioned note, **filing** them directly after the header section. 32 | 33 | In more detail: 34 | - where the line is a heading, it moves the heading and all following lines until a blank line, or the next heading of the same level 35 | - where the line isn't a heading, it moves the line and any following indented lines (optionally terminated by a blank line) 36 | - if there is a `>YYYY-MM-DD` date specified in the line already, then carry that over, otherwise add today's date 37 | 38 | This feature can be turned on using two different options, with slightly different triggers: 39 | - `-m` (`--move`): move tasks with such a note link, _whether or not the task is complete_; 40 | - `-t` (`--movecomplete`): move only _completed tasks_. 41 | 42 | NB: This only operates from Daily (Calendar) notes; therefore it _doesn't_ interfere with linking and back-linking between regular notes. 43 | 44 | ### Templates for dates 45 | It changes any mentions of **date offset patterns** (such as `{-10d}`, `{+2w}`, `{-3m}`) into scheduled dates (e.g. `>2020-02-27`), if it can find a valid date pattern in the previous heading, previous main task if it has sub-tasks, or in the line itself. This allows for users to define simple **template sections** and copy and paste them into the note, set the due date at the start, and the other dates are then worked out for you. 46 | 47 | | For example ... | ... becomes | 48 | | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 49 | | \#\#\# Christmas Cards 25/12/2020
\* Write cards {-20d}
\* Post overseas cards {-15d}
\* Post cards to this country {-10d}
\* Store spare cards for next year {+3d} | \#\#\# Christmas Cards 25/12/2020
\* Write cards >2020-12-05
\* Post overseas cards >2020-12-10
* Post cards to this country >2020-12-15
\* Store spare cards for next year >2020-12-28 | 50 | | \* Bob's birthday on 14/09/2020
  \* Find present {-6d}
  \* Wrap & post present {-3d}
  \* Call Bob {0d} | \* Bob's birthday on 14/09/2020
  \* Find present >2020-09-08
  \* Wrap & post present >2020-09-11
  \* Call Bob >2020-09-14 | 51 | 52 | You can use this within a line to have both a **deadline** and a calculated **start date**: 53 | 54 | | For example ... | ... becomes | 55 | | ---------------------------------------------------------- | -------------------------------------------------------------------- | 56 | | * Post cards deadline 2020-12-18 {-10d} | * Post cards deadline 2020-12-18 >2020-12-08 | 57 | 58 | In more detail: 59 | 60 | - Valid **date offsets** are specified as `[+][0-9][bdwmqy]`. This allows for `b`usiness days, `d`ays, `w`eeks, `m`onths, `q`uarters or `y`ears. (Business days skip weekends. If the existing date happens to be on a weekend, it's treated as being the next working day. Public holidays aren't accounted for.) There's also the special case `{0d}` meaning on the day itself. 61 | - It ignores offsets in a section with a heading that includes a `#template` hashtag 62 | - The base date is by default of the form `YYYY-MM-DD`, not preceded by characters `0-9(<>`, all of which could confuse. 63 | - But you can add another custom date format to use by setting the `RE_DATE_FORMAT_CUSTOM` variable (see below). The dates matched by this regular expression must additionally be caapble of being correctly parsed by ruby's Date.parse() built-in command, over which I have no control. 64 | 65 | ### Create new Calendar Events 66 | NotePlan allows for time-blocking, but this only works to its in-built calendar display. With npTools you also **create events** that are visible and editable in Apple Calendar, or any other apps that use iCloud, *not just NotePlan*. To do this use the `#create_event` tag on a line (task, comment or heading) with a timeblocking command such as `3:00-3:45[AM|PM]` or `3-5pm` or `3PM` and optional location such as `at Jim's`. This allows for meeting events to be listed on a day, and also created in the calendar. In combination with the date offset patterns above, it further allows scheduling preparation time (for example) days or hours before events. 67 | 68 | The date of the event is determined in this order: 69 | 1. it will use a `>YYYY-MM-DD` mentioned in the line. NB: date offset patterns get calculated before it looks to create events. 70 | 2. if it's a calendar note, then use that date 71 | 3. otherwise use today's date. 72 | 73 | For example to create a meeting event for a certain day put this in it's daily (calendar) note: 74 | ``` 75 | ### Project X Meeting #create_event 10:30am at Jim's 76 | ``` 77 | 78 | To extend this and use the date templates, add a timed task with calendar entry to do some associated tasks 5 days before it, and the following morning: 79 | ``` 80 | ### Project X Meeting 21-12-2020 #create_event 10:30am at Jim's 81 | * write and circulate agenda {-5d} #create_event 4pm 82 | * send out actions {1d} #create_event 9am 83 | ``` 84 | 85 | ![Video showing event creation](npTools-create_event.gif) 86 | 87 | Notes on this: 88 | - If no date is given on the line, and 89 | - If no finish time is set, then the event defaults to an hour. 90 | - You can use the shortcut `3PM` or `3-5PM` when you don't need to specify the minutes -- though NotePlan itself currently doesn't recognise this syntax for time blocks. You can specify minutes as well either using `3:45` or `3.15` formats. 91 | - If am/AM or pm/PM isn't given, then the hours are assumed to be in 24-hour clock 92 | - It's best to put any `at place` location at the end of the line, as there's no easy way of telling how a location finishes, so it will use the rest of the line as the location. 93 | - Any indented following lines are copied to the event description, with leading whitespace removed. 94 | - Under the hood this uses AppleScript, and takes a few seconds per event. Once it has run succesfully the `#create_event` is changed to `#event_created` so that it won't be triggered again. 95 | - There are some calendar settings that need to be configured for this: see Installation and Configuration below. 96 | 97 | ### Extend the existing @repeat mechanism 98 | It **creates new repeats** for newly completed tasks that include a `@repeat(interval)`, on the appropriate future date. 99 | 100 | - Valid intervals are specified as `[+][0-9][bdwmqy]`. This allows for `b`usiness days, `d`ays, `w`eeks, `m`onths, `q`uarters or `y`ears. 101 | - When _interval_ is of the form `+2w` it will duplicate the task for 2 weeks after the date the _task was completed_. 102 | - When _interval_ is of the form `2w` it will duplicate the task for 2 weeks after the date the _task was last due_. If this can't be determined, then it defaults to the first option. 103 | 104 | NB: For this feature to work, you need to have the 'Append Completion Date' NotePlan setting turned on, and to have the first type of tidy up (above) happening. 105 | 106 | ## Running the Tools 107 | There are 2 ways of running the script: 108 | 1. with no arguments (`ruby npTools.rb`), it checks all note and daily files updated in the last 24 hours. This is the way to use it automatically, running one or more times each day. (This is configurable by `HOURS_TO_PROCESS` below.) 109 | 2. with passed filename pattern(s), where it works on any matching Calendar or Note files. For example, to match the Daily file from 24/3/2020 use `ruby npTools.rb 20200324.txt`. It can include wildcard *patterns* to match multiple files, for example `"202003*.txt"` to process all Daily files from March 2020. (It now needs to be in double quotes for the file pattern matching to work.) If no `.` is found in the pattern, the pattern matches all files as `"*pattern*.*"`. 110 | 111 | You can also specify the following **options**: 112 | - `-h` (`--help`) for a list of options, 113 | 114 | - `c` (`--changes HOURS`) how many hours to look back to find note changes to process, overriding the default of 24 hours (though this can be changed; see below) 115 | - `-d` (`--moveondailies`) turn on moving mentions of `>date` in a daily calendar note to the specified date 116 | - `-i` (`--skiptoday`) don't process today's file 117 | - `-f` (`--skipfile=NOTETITLE[,NOTETITLE2,etc]`) don't process specific note(s) 118 | - `-m` (`--move`) moves mentions of [[Note#Heading]] in tasks in daily calendar day notes to the [[Note]], _whether or note the task has been completed_ 119 | - `-t` (`--movecomplete`) moves mentions of [[Note#Heading]] in tasks in daily calendar day notes to the [[Note]], _but only when the task has been completed_ 120 | - `-q` (`--quiet`) suppress all output, other than error messages 121 | - `-s` (`--keepschedules`) keep the scheduled (>) dates of completed tasks 122 | - `-v` for verbose (logging) output 123 | - `-w` for more verbose (logging) output 124 | 125 | It works with all 3 storage options for storing NotePlan data: CloudKit (the default from NotePlan v3), iCloud Drive and Dropbox. 126 | 127 | **NB**: NotePlan has several options in the Markdown settings for how to mark a task, including `-`, `- [ ]', `*` and `* [ ]`. All are supported by this script. 128 | 129 | ## Installation and Configuration 130 | 1. Check you have a working Ruby installation. 131 | 2. Install two ruby gems (libraries) (`sudo gem install colorize optparse`) 132 | 3. Download and copy the script to a place where it can be found on your FILE filepath (perhaps `sudo cp npTools.rb /usr/local/bin/`) 133 | 4. Make the script executable (`chmod 755 npTools.rb`) 134 | 5. Change the following constants at the top of the script, as required: 135 | - `hours_to_process`: will process all files changed within this number of hours (default 24) 136 | - `TAGS_TO_REMOVE`: list of tags to remove. Default ["#waiting","#high"] 137 | - `DAILY_TASKS_SECTION_NAME`: the section heading name (without `#` marks) to file moved tasks in 138 | - `DATE_TIME_LOG_FORMAT`: date string format to use in logs 139 | - `DATE_TIME_APPLESCRIPT_FORMAT`: date string format to use in AppleScript for event creation -- depends on various locale settings 140 | - `CALENDAR_APP_TO_USE`: name of Calendar app to use in create_event AppleScript. Default is 'Calendar'. Can ignore if not using this for event creation. 141 | - `CALENDAR_NAME_TO_USE`: name of Calendar to create any new events in. Can ignore if not using this for event creation. 142 | - `CREATE_EVENT_TAG_TO_USE`: name of tag to use to trigger creating events. Default is `#create_event`. Can ignore if not using this for event creation. 143 | - for completeness, `NP_BASE_DIR` automatically works out where NotePlan data files are located. (If there are multiple isntallations it selects using the priority CloudKit > iCloudDrive > DropBox.) 144 | - `RE_DATE_OFFSET_FORMAT`: regular expression to find date strings in your chosen format, to use as the base date in date offset patterns. See example in the code, but don't change unless you're familiar with regular expressions. 145 | 6. Then run `ruby npTools.rb [-options]` 146 | 147 | The first time you attempt to `#create_event`, macOS (at least Catalina and Big Sur) will probably ask for permission to update your Calendar. 148 | 149 | ### Automatic running 150 | If you wish to run this automatically in the background on macOS, you can do this using the built-in `launchctl` system. (For more info on this see for example [How to Use launchd to Run Services in macOS](https://medium.com/swlh/how-to-use-launchd-to-run-services-in-macos-b972ed1e352).) 151 | 152 | Here's the configuration file `jgc.npTools.plist` that I use to automatically run `npTools.rb` three times a day: 153 | ``` 154 | 155 | 156 | 158 | 159 | 160 | Label 161 | jgc.npTools 162 | ProgramArguments 163 | 164 | /usr/bin/ruby 165 | /Users/jonathan/bin/npTools 166 | 167 | StartCalendarInterval 168 | 169 | 170 | Hour 171 | 09 172 | Minute 173 | 09 174 | 175 | 176 | Hour 177 | 12 178 | Minute 179 | 09 180 | 181 | 182 | Hour 183 | 18 184 | Minute 185 | 09 186 | 187 | 188 | StandardOutPath 189 | /tmp/jgc.npTools.stdout 190 | StandardErrorPath 191 | /tmp/jgc.npTools.stderr 192 | 193 | 194 | ``` 195 | Update the filepaths to suit your particular configuration, place this in the `~/Library/LaunchAgents` directory, and then run the following terminal command: 196 | ``` 197 | launchctl load ~/Library/LaunchAgents/jgc.npTools.plist 198 | ``` 199 | 200 | ## Problems? Suggestions? 201 | I now work on improving the Plugins, instead of these old scripts. (Though sadly I have to use JavaScript for them, not Ruby which I do have a soft spot for.) 202 | -------------------------------------------------------------------------------- /npMediaSave.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | #------------------------------------------------------------------------------- 3 | # Script to Save some Media notes into NotePlan 4 | # by Jonathan Clark 5 | # 6 | # TODO: Change YouTube processing to be fed from Zapier 7 | # v0.5.0, 10.1.2024 - add YouTube favourites (via IFTTT) 8 | # v0.4.0, 30.12.2023 - switch Spotify to be fed by Make not IFTTT 9 | # v0.3.4, 27.5.2023 - deals with date parsing errors in Instapaper, and multi-line titles in Instapaper 10 | # v0.3.3, 20.3.2021 - ? 11 | # v0.3.0, ? - now copes with multi-line tweets 12 | #------------------------------------------------------------------------------- 13 | VERSION = "0.4.0" 14 | require 'date' 15 | require 'cgi' 16 | require 'colorize' 17 | require 'optparse' # more details at https://docs.ruby-lang.org/en/2.1.0/OptionParser.html 18 | 19 | #------------------------------------------------------------------------------- 20 | # Setting variables to tweak 21 | #------------------------------------------------------------------------------- 22 | MEDIA_STRING = '### Media Consumption' # the title of the section heading to add these notes to 23 | NOTE_EXT = "md" # or "txt" 24 | IFTTT_FILEPATH = "/Users/jonathan/Dropbox/IFTTT/" 25 | IFTTT_ARCHIVE_FILEPATH = "/Users/jonathan/Dropbox/IFTTT/Archive/" 26 | MAKE_INBOX_DIR = "/Users/jonathan/Dropbox/Make/" 27 | MAKE_ARCHIVE_FILEPATH = "/Users/jonathan/Dropbox/Make/Archive/" 28 | INSTAPAPER_FILE = "Instapaper Archived Items.txt" 29 | MEDIUM_FILE = "Medium Articles.txt" 30 | SPOTIFY_FILE_GLOB = "Spotify_*.doc" # make forces .doc extension for some reason 31 | TWITTER_FILE = "My Tweets.txt" 32 | YOUTUBE_LIKES_FILE = "YouTube liked videos.txt" 33 | YOUTUBE_UPLOAD_FILE = "YouTube upload.txt" 34 | DATE_TIME_LOG_FORMAT = '%e %b %Y %H:%M'.freeze # only used in logging 35 | DATE_TIME_APPEND_FORMAT = '%Y%m%d%H%M'.freeze 36 | DATE_YYYYMMDD_FORMAT = '%Y%m%d'.freeze 37 | 38 | #------------------------------------------------------------------------------- 39 | # To use test data instead of live data, uncomment relevant definitions. 40 | # NB: assumes one per line, apart from Twitter. 41 | #------------------------------------------------------------------------------- 42 | # $spotify_ifttt_test_data = <<-END_S_DATA 43 | # February 6, 2021 at 11:11PM | Espen Eriksen Trio | In the Mountains | Never Ending January | https://ift.tt/2TRqQiB | https://ift.tt/2LptJng 44 | # February 6, 2021 at 11:56AM | Brian Doerksen | Creation Calls | Today | https://ift.tt/2qQI2Sq | https://ift.tt/3pYs1by 45 | # END_S_DATA 46 | # $spotify_make_test_data = <<-END_S_DATA 47 | # 2023-11-01T16:53:22.000Z | Sovereign Grace Music | He Will Keep You (Psalm 121) - Live | Unchanging God: Songs from the Book of Psalms, Vol. 1 (Live) | | https://i.scdn.co/image/ab67616d0000b2734fcee6fd27886f84d6efc638 48 | # 2023-11-01T18:48:39.000Z | Julian & Roman Wasserfuhr | Englishman in New York | Gravity | | https://i.scdn.co/image/ab67616d0000b273da5cb7f2949038355b094a7a 49 | # END_S_DATA 50 | 51 | 52 | # $instapaper_test_data = <<-END_I_DATA 53 | # February 6, 2021 at 05:49AM \\ Thomas Creedy: Imago Dei \\ https://ift.tt/3rJl1Bh \\ 54 | # February 6, 2021 at 06:02AM \\ Is the 'seal of the confessional' Anglican? \\ https://ift.tt/2MpMAPX \\ "Andrew Atherstone writes: The Church of England has at last published the report of the 'Seal of the Confessional' working party , more than a year after it..." 55 | # February 6, 2021 at 04:04PM \\ In what ways can we form useful relationships between notes? \\ https://ift.tt/3aT3LS3 \\ "Nick Milo Aug 8, 2020 * 7 min read Are you into personal knowledge management PKM)? Are you confused about when to use a folder versus a tag versus a link..." 56 | # END_I_DATA 57 | 58 | # $twitter_test_data = <<-END_T_DATA 59 | # February 6, 2021 at 03:56PM | A useful thread which highlights the problems of asking the wrong question. https://t.co/xEKimf9cLK | jgctweets | http://twitter.com/jgctweets/status/1353371090609909760 60 | # END_T_DATA 61 | 62 | # $youtube_liked_test_data = < 1 78 | NP_CALENDAR_DIR = "#{np_base_dir}/Calendar".freeze 79 | 80 | # Colours to use with the colorization gem 81 | # to show some possible combinations, run String.color_samples 82 | # to show list of possible modes, run puts String.modes (e.g. underline, bold, blink) 83 | CompletedColour = :light_green 84 | InfoColour = :yellow 85 | ErrorColour = :light_red 86 | # Test to see if we're running interactively or in a batch mode: 87 | # if batch mode then disable colorisation which doesn't work in logs 88 | tty_code = `tty`.chomp 89 | String.disable_colorization true if tty_code == 'not a tty' 90 | 91 | # Variables that need to be globally available 92 | time_now = Time.now 93 | $date_time_now_log_fmttd = time_now.strftime(DATE_TIME_LOG_FORMAT) 94 | $date_time_now_file_fmttd = time_now.strftime(DATE_TIME_APPEND_FORMAT) 95 | $date_now = time_now.strftime(DATE_YYYYMMDD_FORMAT) 96 | $verbose = false 97 | $npfile_count = 0 98 | 99 | #------------------------------------------------------------------------- 100 | # Helper Functions 101 | #------------------------------------------------------------------------- 102 | def main_message(message) 103 | puts message.colorize(CompletedColour) 104 | end 105 | 106 | def warning_message(message) 107 | puts message.colorize(InfoColour) 108 | end 109 | 110 | def error_message(message) 111 | puts message.colorize(ErrorColour) 112 | end 113 | 114 | def log_message(message) 115 | puts message if $verbose 116 | end 117 | 118 | def truncate_text(text, max_length = 100000, use_elipsis = false) 119 | raise ArgumentError, "max_length must be positive" unless max_length.positive? 120 | return '' if text.nil? 121 | 122 | return text if text.size <= max_length 123 | 124 | return text[0, max_length] + (use_elipsis ? '...' : '') 125 | end 126 | 127 | #------------------------------------------------------------------------- 128 | # Class definition: NPCalFile 129 | #------------------------------------------------------------------------- 130 | class NPCalFile 131 | # Define the attributes that need to be visible outside the class instances 132 | attr_reader :id, :media_header_line, :is_calendar, :is_updated, :filename, :line_count 133 | 134 | def initialize(date) 135 | # Create NPFile object from reading Calendar file of date YYYMMDD 136 | 137 | # Set the file's id 138 | $npfile_count += 1 139 | @id = $npfile_count 140 | @filename = "#{NP_CALENDAR_DIR}/#{date}.#{NOTE_EXT}" 141 | @lines = [] 142 | @line_count = 0 143 | @title = date 144 | @is_updated = false 145 | 146 | begin 147 | log_message("Reading NPCalFile for '#{@title}'") 148 | 149 | # Open file and read in all lines (finding any Done and Cancelled headers) 150 | # NB: needs the encoding line when run from launchctl, otherwise you get US-ASCII invalid byte errors (basically the 'locale' settings are different) 151 | # First, if file doesn't exist, then create it. 152 | if !File.exist?(@filename) 153 | f = File.open(@filename, 'w', encoding: 'utf-8') 154 | log_message(" - needed to CREATE file for '#{@filename}'") 155 | f.close 156 | end 157 | 158 | f = File.open(@filename, 'r', encoding: 'utf-8') 159 | n = 0 160 | # Next check whether the file doesn't have any content before trying to read it 161 | if !f.nil? 162 | f.each_line do |line| 163 | @lines[n] = line 164 | n += 1 165 | end 166 | elsif 167 | log_message(" - file '#{@filename}' exists but is empty") 168 | end 169 | f.close 170 | @line_count = @lines.size # e.g. for lines 0-2 size => 3 171 | log_message("- Finished NPCalFile init for '#{@title}' using id #{@id} with #{@line_count} lines") 172 | rescue StandardError => e 173 | error_message("ERROR: #{e.exception.message} when re-writing note file #{@filename}") 174 | end 175 | end 176 | 177 | def insert_new_line(new_line, line_number) 178 | # Insert 'line' into position 'line_number' 179 | # NB: this is insertion at the line number, so that current line gets moved to be one later 180 | n = @line_count # start iterating from the end of the array 181 | line_number = n if line_number >= n # don't go beyond current size of @lines 182 | log_message(" - insert_new_line at #{line_number} (count=#{n}) ...") 183 | @lines.insert(line_number, new_line) 184 | @line_count = @lines.size 185 | end 186 | 187 | def append_line_to_section(new_line, section_heading) 188 | # Append new_line after 'section_heading' line. 189 | # If not found, then add 'section_heading' to the end first 190 | log_message(" - append_line_to_section for '#{section_heading}' ...") 191 | n = 0 192 | added = false 193 | found_section = false 194 | while !added && (n < @line_count) 195 | line = @lines[n].chomp 196 | # if an empty line or a new header section starting, insert line here 197 | if found_section && (line.empty? || line =~ /^#+\s/) 198 | insert_new_line(new_line, n) 199 | added = true 200 | end 201 | # if this is the section header of interest, save its details. (Needs to come after previous test.) 202 | found_section = true if line =~ /^#{section_heading}/ 203 | n += 1 204 | end 205 | # log_message(" section heading not found, so adding at line #{n}, #{@line_count}") 206 | insert_new_line(section_heading, n) unless found_section # if section not yet found then add it before this line 207 | insert_new_line(new_line, n + 1) unless added # if not added so far, then now append 208 | end 209 | 210 | def rewrite_cal_file 211 | # write out this updated calendar file 212 | main_message(" > writing updated version of #{@filename}") 213 | # open file and write all the lines out 214 | begin 215 | File.open(@filename, 'w') do |f| 216 | @lines.each do |line| 217 | f.puts line 218 | end 219 | end 220 | rescue StandardError => e 221 | error_message("ERROR: #{e.exception.message} when re-writing calendar file #{filepath}") 222 | end 223 | end 224 | end 225 | 226 | #-------------------------------------------------------------------------------------- 227 | # SPOTIFY 228 | # - Saved Date (diff in Make/IFTTT) | Artist | Track name | Album | Track URL | Album art URL 229 | # Can have multiple files to process 230 | # Note: Should really go back to previous model, but concat the files first. However, this works, albeit over multiple invocations 231 | #-------------------------------------------------------------------------------------- 232 | def process_spotify 233 | spotify_filepath = IFTTT_FILEPATH + SPOTIFY_FILE 234 | # spotify_filepath = "" 235 | catch (:done) do # provide a clean way out of this 236 | if defined?($spotify_test_data) 237 | f = $spotify_test_data 238 | log_message("Using Spotify test data") 239 | elsif File.exist?(spotify_filepath) 240 | if File.empty?(spotify_filepath) 241 | warning_message("Note: Spotify file empty") 242 | throw :done 243 | else 244 | f = File.open(spotify_filepath, 'r', encoding: 'utf-8') 245 | log_message("Found Spotify file #{f.path} length #{f.size} bytes") 246 | end 247 | else 248 | warning_message("No Spotify file found") 249 | throw :done 250 | end 251 | 252 | begin 253 | # Parse each line in the file (though often only one) 254 | f.each_line do |line| 255 | parts = line.split('|') 256 | artist = parts[1].strip 257 | track_name = parts[2].strip 258 | album = parts[3].strip 259 | track_url = parts[4].strip 260 | album_art_url = parts[5].strip 261 | 262 | # IFTTT version: parse the given date-time string, then create YYYYMMDD version of it 263 | # begin 264 | # trunc_first_field = truncate_text(parts[0], 122, false) # function's limit is 128, but it seems to need fewer than that 265 | # date_YYYYMMDD = Date.parse(trunc_first_field).strftime(DATE_YYYYMMDD_FORMAT) 266 | # log_message(" Found item to save with date #{date_YYYYMMDD}:") 267 | # rescue Date::Error => e 268 | # warning_message("couldn't parse date in: #{trunc_first_field}. Will default to today instead.") 269 | # date_YYYYMMDD = $date_now 270 | # end 271 | 272 | # Make version: 273 | save_date = parts[0].strip 274 | date_YYYYMMDD = save_date.gsub('-', '')[0,8] 275 | # log_message(" Found item to save with date #{date_YYYYMMDD}:") 276 | 277 | # Format line to add 278 | line_to_add = "- fave #spotify #{artist}'s **[#{track_name}](#{track_url})** from album #{album} ![](#{album_art_url})" 279 | log_message(line_to_add) 280 | 281 | # Read in the NP Calendar file for this date 282 | this_note = NPCalFile.new(date_YYYYMMDD) 283 | 284 | # Add new lines to end of file, creating a "### Media Consumed" section before it if it doesn't exist 285 | this_note.append_line_to_section(line_to_add, MEDIA_STRING) 286 | this_note.rewrite_cal_file 287 | main_message("-> Saved new Spotify fave to #{date_YYYYMMDD}\n") 288 | end 289 | 290 | unless defined?($spotify_make_test_data) 291 | f.close 292 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 293 | archive_filename = "#{MAKE_ARCHIVE_FILEPATH}#{found_filename[0..-5]}_#{$date_time_now_file_fmttd}.txt" 294 | log_message("- Will rename file to #{archive_filename}") 295 | File.rename(spotify_filepath, archive_filename) 296 | end 297 | 298 | rescue StandardError => e 299 | error_message("ERROR: #{e.exception.message} for file #{spotify_filepath}") 300 | end 301 | end 302 | end 303 | 304 | #-------------------------------------------------------------------------------------- 305 | # INSTAPAPER 306 | #-------------------------------------------------------------------------------------- 307 | def process_instapaper 308 | instapaper_filepath = IFTTT_FILEPATH + INSTAPAPER_FILE 309 | log_message("Starting to process Instapaper file #{instapaper_filepath}") 310 | catch (:done) do # provide a clean way out of this 311 | if defined?($instapaper_test_data) 312 | f = $instapaper_test_data 313 | log_message("Using Instapaper test data") 314 | elsif File.exist?(instapaper_filepath) 315 | if File.empty?(instapaper_filepath) 316 | warning_message("Note: Instapaper file empty") 317 | throw :done 318 | else 319 | f = File.open(instapaper_filepath, 'r', encoding: 'utf-8') 320 | log_message("Found Instapaper file #{f.path} length #{f.size} bytes") 321 | end 322 | else 323 | warning_message("No Instapaper file found") 324 | throw :done 325 | end 326 | 327 | begin 328 | needs_concatenating = false 329 | previous_line = '' 330 | f.each_line do |line| 331 | # Cope with items over several lines: concatenate with next line 332 | if needs_concatenating 333 | line = previous_line + line 334 | log_message(" Concatenated -> '#{line}' with next") 335 | needs_concatenating = false 336 | end 337 | 338 | # Parse each line, splitting on \ delimiters 339 | parts = line.split(" \\ ") 340 | # If we have less than 4 parts we'll need to join this into the next line 341 | if parts.size < 4 342 | needs_concatenating = true 343 | previous_line = line.strip # remove whitespace (including the probable newline on the end) 344 | log_message(" Need to concatenate '#{line}' with next") 345 | next 346 | end 347 | log_message(" #{line} --> #{parts}") 348 | 349 | # parse the given date-time string, then create YYYYMMDD version of it 350 | begin 351 | trunc_first_field = truncate_text(parts[0], 122, false) # function's limit is 128, but it seems to need fewer than that 352 | date_YYYYMMDD = Date.parse(trunc_first_field).strftime(DATE_YYYYMMDD_FORMAT) 353 | log_message(" Found item to save with date #{date_YYYYMMDD}:") 354 | rescue Date::Error => e 355 | warning_message("couldn't parse date in: #{trunc_first_field}. Will default to today instead.") 356 | date_YYYYMMDD = $date_now 357 | end 358 | 359 | # Format line to add. Guard against possible empty fields 360 | parts[2] = '' if parts[2].nil? 361 | parts[3] = '' if parts[3].nil? 362 | line_to_add = "- #article **[#{parts[1].strip}](#{parts[2].strip})** #{parts[3].strip}" 363 | log_message(line_to_add) 364 | 365 | # Read in the NP Calendar file for this date 366 | this_note = NPCalFile.new(date_YYYYMMDD) 367 | 368 | # Add new lines to end of file, creating a ### Media section before it if it doesn't exist 369 | this_note.append_line_to_section(line_to_add, MEDIA_STRING) 370 | this_note.rewrite_cal_file 371 | main_message("-> Saved new Instapaper item to #{date_YYYYMMDD}") 372 | end 373 | 374 | unless defined?($instapaper_test_data) 375 | f.close 376 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 377 | archive_filename = "#{IFTTT_ARCHIVE_FILEPATH}#{INSTAPAPER_FILE[0..-5]}_#{$date_time_now_file_fmttd}.txt" 378 | File.rename(instapaper_filepath, archive_filename) 379 | end 380 | 381 | rescue StandardError => e 382 | error_message("ERROR: #{e.exception.message} when processing file #{INSTAPAPER_FILE}") 383 | end 384 | end 385 | end 386 | 387 | #-------------------------------------------------------------------------------------- 388 | # YOUTUBE 'likes' 389 | #-------------------------------------------------------------------------------------- 390 | def process_youtube 391 | youtube_filepath = IFTTT_FILEPATH + YOUTUBE_LIKES_FILE 392 | log_message("Starting to process youtube file #{youtube_filepath}") 393 | catch (:done) do # provide a clean way out of this 394 | if defined?($youtube_liked_test_data) 395 | f = $youtube_liked_test_data 396 | log_message("Using YouTube test data") 397 | elsif File.exist?(youtube_filepath) 398 | if File.empty?(youtube_filepath) 399 | warning_message("Note: YouTube file empty") 400 | throw :done 401 | else 402 | f = File.open(youtube_filepath, 'r', encoding: 'utf-8') 403 | end 404 | else 405 | warning_message("No YouTube file found") 406 | throw :done 407 | end 408 | 409 | begin 410 | f.each_line do |line| 411 | log_message("") # blank line 412 | # Parse each line 413 | parts = line.split(' \\ ') 414 | log_message(" #{line} --> #{parts.size} parts: #{parts}") 415 | # parse the given date-time string, then create YYYYMMDD version of it 416 | date_YYYYMMDD = Date.parse(parts[0]).strftime('%Y%m%d') 417 | log_message(" Found item to save with date #{date_YYYYMMDD}:") 418 | 419 | # Format line to add. 420 | title = parts[1].strip 421 | url = parts[2].strip if parts[2].start_with?('https://www.youtube.com/') 422 | line_to_add = "- liked video [#{title}](#{url})" 423 | log_message(line_to_add) 424 | 425 | # Read in the NP Calendar file for this date 426 | this_note = NPCalFile.new(date_YYYYMMDD) 427 | 428 | # Add new lines to end of file, creating a ### Media section before it if it doesn't exist 429 | this_note.append_line_to_section(line_to_add, MEDIA_STRING) 430 | this_note.rewrite_cal_file 431 | main_message("-> Saved new YouTube item to #{date_YYYYMMDD}") 432 | end 433 | 434 | unless defined?($youtube_liked_test_data) 435 | f.close 436 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 437 | archive_filename = "#{youtube_filepath[0..-5]}_#{$date_time_now_file_fmttd}.txt" 438 | log_message("") 439 | log_message("Finished. Will rename file to #{archive_filename}") 440 | File.rename(youtube_filepath, archive_filename) 441 | end 442 | rescue StandardError => e 443 | error_message("ERROR: #{e.exception.message} when processing file #{youtube_filepath}") 444 | end 445 | end 446 | end 447 | 448 | #-------------------------------------------------------------------------------------- 449 | # MEDIUM articles 450 | #-------------------------------------------------------------------------------------- 451 | def process_medium 452 | medium_filepath = IFTTT_FILEPATH + MEDIUM_FILE 453 | log_message("Starting to process Medium file #{medium_filepath}") 454 | catch (:done) do # provide a clean way out of this 455 | if defined?($medium_test_data) 456 | f = $medium_test_data 457 | log_message("Using Medium test data") 458 | elsif File.exist?(medium_filepath) 459 | if File.empty?(medium_filepath) 460 | warning_message("Note: Medium file empty") 461 | throw :done 462 | else 463 | f = File.open(medium_filepath, 'r', encoding: 'utf-8') 464 | end 465 | else 466 | warning_message("No Medium file found") 467 | throw :done 468 | end 469 | 470 | begin 471 | # FIXME: very slow on this line on MBA but not MM4 472 | f.each_line do |line| 473 | # Parse each line 474 | parts = line.split(" \\ ") 475 | # log_message(" #{line} --> #{parts}") 476 | # parse the given date-time string, then create YYYYMMDD version of it 477 | date_YYYYMMDD = Date.parse(parts[0]).strftime('%Y%m%d') 478 | log_message(" Found item to save with date #{date_YYYYMMDD}:") 479 | 480 | # Format line to add. Guard against possible empty fields 481 | parts[2] = '' if parts[2].nil? 482 | line_to_add = "- #article **[#{parts[1].strip}](#{parts[2].strip})**" 483 | log_message(line_to_add) 484 | 485 | # Read in the NP Calendar file for this date 486 | this_note = NPCalFile.new(date_YYYYMMDD) 487 | 488 | # Add new lines to end of file, creating a ### Media section before it if it doesn't exist 489 | this_note.append_line_to_section(line_to_add, MEDIA_STRING) 490 | this_note.rewrite_cal_file 491 | main_message("-> Saved new Medium item to #{date_YYYYMMDD}") 492 | end 493 | 494 | unless defined?($medium_test_data) 495 | f.close 496 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 497 | archive_filename = "#{medium_filepath[0..-5]}_#{$date_time_now_file_fmttd}.txt" 498 | File.rename(medium_filepath, archive_filename) 499 | end 500 | rescue StandardError => e 501 | error_message("ERROR: #{e.exception.message} when processing file #{medium_filepath}") 502 | end 503 | end 504 | end 505 | 506 | #-------------------------------------------------------------------------------------- 507 | # TWITTER 508 | #-------------------------------------------------------------------------------------- 509 | def process_twitter 510 | twitter_filepath = IFTTT_FILEPATH + TWITTER_FILE 511 | log_message("Starting to process twitter file #{twitter_filepath}") 512 | catch (:done) do # provide a clean way out of this 513 | if defined?($twitter_test_data) 514 | f = $twitter_test_data 515 | log_message("Using Twitter test data") 516 | elsif File.exist?(twitter_filepath) 517 | if File.empty?(twitter_filepath) 518 | warning_message("Note: Twitter file empty") 519 | throw :done 520 | else 521 | f = File.open(twitter_filepath, 'r', encoding: 'utf-8') 522 | log_message("Found Twitter file #{f.path} length #{f.size} bytes") 523 | end 524 | else 525 | warning_message("No Twitter file found") 526 | throw :done 527 | end 528 | 529 | begin 530 | needs_concatenating = false 531 | previous_line = '' 532 | f.each_line do |line| 533 | # Cope with tweets over several lines: concatenate with next line 534 | if needs_concatenating 535 | line = previous_line + line 536 | log_message(" Concatenated -> '#{line}' with next") 537 | needs_concatenating = false 538 | end 539 | # Parse each line 540 | parts = line.split(" | ") 541 | # If we have less than 4 parts we'll need to join this into the next line 542 | if parts.size < 4 543 | needs_concatenating = true 544 | previous_line = line.strip # remove whitespace (including the probable newline on the end) 545 | log_message(" Need to concatenate '#{line}' with next") 546 | next 547 | end 548 | # log_message(" #{line} --> #{parts}") 549 | 550 | # parse the given date-time string, then create YYYYMMDD version of it 551 | begin 552 | trunc_first_field = truncate_text(parts[0], 122, false) # function's limit is 128, but it seems to need fewer than that 553 | date_YYYYMMDD = Date.parse(trunc_first_field).strftime(DATE_YYYYMMDD_FORMAT) 554 | log_message(" Found item to save with date #{date_YYYYMMDD}:") 555 | rescue Date::Error => e 556 | warning_message("couldn't parse date in: #{trunc_first_field}. Will default to today instead.") 557 | date_YYYYMMDD = $date_now 558 | end 559 | 560 | # Format line to add 561 | line_to_add = "- @#{parts[2].strip} tweet: \"#{parts[1].strip}\" ([permalink](#{parts[3].strip}))" 562 | log_message(line_to_add) 563 | 564 | # Read in the NP Calendar file for this date 565 | this_note = NPCalFile.new(date_YYYYMMDD) 566 | 567 | # Add new lines to end of file, creating a ### Media section before it if it doesn't exist 568 | this_note.append_line_to_section(line_to_add, MEDIA_STRING) 569 | this_note.rewrite_cal_file 570 | main_message("-> Saved new Twitter item to #{date_YYYYMMDD}") 571 | needs_concatenating = false 572 | previous_line = '' 573 | end 574 | 575 | unless defined?($twitter_test_data) 576 | f.close 577 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 578 | archive_filename = "#{twitter_filepath[0..-5]}_#{$date_time_now_file_fmttd}.txt" 579 | File.rename(twitter_filepathpath, archive_filename) 580 | end 581 | rescue StandardError => e 582 | error_message("ERROR: #{e.exception.message} when processing file #{TWITTER_FILE}") 583 | end 584 | end 585 | end 586 | 587 | #======================================================================================= 588 | # Main logic 589 | #======================================================================================= 590 | 591 | # Setup program options 592 | options = {} 593 | opt_parser = OptionParser.new do |opts| 594 | opts.banner = "NotePlan media adder v#{VERSION}" # \nDetails at https://github.com/jgclark/NotePlan-tools/\nUsage: npMediaSave.rb [options]" 595 | opts.separator '' 596 | options[:instapaper] = false 597 | options[:medium] = false 598 | options[:spotify] = false 599 | options[:twitter] = false 600 | options[:verbose] = false 601 | options[:youtube] = false 602 | opts.on('-h', '--help', 'Show this help') do 603 | puts opts 604 | exit 605 | end 606 | opts.on('-i', '--instapaper', 'Add Instapaper records') do 607 | options[:instapaper] = true 608 | end 609 | opts.on('-m', '--medium', 'Add Medium records') do 610 | options[:medium] = true 611 | end 612 | opts.on('-s', '--spotify', 'Add Spotify records') do 613 | options[:spotify] = true 614 | end 615 | opts.on('-t', '--twitter', 'Add Twitter records') do 616 | options[:twitter] = true 617 | end 618 | opts.on('-v', '--verbose', 'Show information as I work') do 619 | $verbose = true 620 | end 621 | opts.on('-y', '--youtube', 'Add YouTube records') do 622 | options[:youtube] = true 623 | end 624 | end 625 | opt_parser.parse! # parse out options, leaving file patterns to process 626 | 627 | log_message("\nStarting npMediaSave v#{VERSION} at #{$date_time_now_log_fmttd}") 628 | process_instapaper if options[:instapaper] 629 | process_medium if options[:medium] 630 | process_spotify if options[:spotify] 631 | process_twitter if options[:twitter] 632 | process_youtube if options[:youtube] 633 | -------------------------------------------------------------------------------- /npTools-create_event.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jgclark/NotePlan-tools/0639ada608303e02c2790c56a295a9fb1904f98e/npTools-create_event.gif -------------------------------------------------------------------------------- /npTools.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | #------------------------------------------------------------------------------- 3 | # NotePlan Tools script 4 | # by Jonathan Clark, v2.4.1, 26.1.2023 5 | #------------------------------------------------------------------------------- 6 | # See README.md file for details, how to run and configure it. 7 | # Repository: https://github.com/jgclark/NotePlan-tools/ 8 | #------------------------------------------------------------------------------- 9 | VERSION = "2.4.1" 10 | 11 | require 'date' 12 | require 'time' 13 | require 'cgi' 14 | require 'colorize' 15 | require 'optparse' # more details at https://docs.ruby-lang.org/en/2.1.0/OptionParser.html 16 | require 'ostruct' 17 | 18 | #------------------------------------------------------------------------------- 19 | # Setting variables to tweak 20 | #------------------------------------------------------------------------------- 21 | hours_to_process = 24 # by default will process all files changed within this number of hours 22 | TAGS_TO_REMOVE = ['#waiting', '#high', '#started', '#⭐'].freeze # simple array of strings 23 | DAILY_TASKS_SECTION_NAME = '### Tasks' # set to a section heading you'd like to file tasks to in daily notes 24 | DATE_TIME_LOG_FORMAT = '%e %b %Y %H:%M'.freeze # only used in logging 25 | # DATE_TIME_APPLESCRIPT_FORMAT = '%e %b %Y %I:%M %p'.freeze # format for creating Calendar events (via AppleScript) when Region setting is 12-hour clock 26 | DATE_TIME_APPLESCRIPT_FORMAT = '%e %b %Y %H:%M:%S'.freeze # format for creating Calendar events (via AppleScript) when Region setting is 24-hour clock 27 | CALENDAR_APP_TO_USE = 'Calendar' # Name of Calendar app to use in create_event AppleScript. Default is 'Calendar'. 28 | CALENDAR_NAME_TO_USE = 'Jonathan (iCloud)' # Apple (iCal) Calendar name to create new events in (if required) 29 | CREATE_EVENT_TAG_TO_USE = '#create_event' # customise if you want a different tag 30 | NOTE_EXT = 'md' # or 'txt' 31 | 32 | #------------------------------------------------------------------------------- 33 | # Other Constants & Settings 34 | #------------------------------------------------------------------------------- 35 | DATE_TODAY_FORMAT = '%Y%m%d'.freeze # using this to identify the "today" daily note 36 | USERNAME = ENV['LOGNAME'] # pull username from environment 37 | USER_DIR = ENV['HOME'] # pull home directory from environment 38 | DROPBOX_DIR = "#{USER_DIR}/Dropbox/Apps/NotePlan/Documents".freeze 39 | ICLOUDDRIVE_DIR = "#{USER_DIR}/Library/Mobile Documents/iCloud~co~noteplan~NotePlan/Documents".freeze 40 | CLOUDKIT_DIR = "#{USER_DIR}/Library/Containers/co.noteplan.NotePlan3/Data/Library/Application Support/co.noteplan.NotePlan3".freeze 41 | np_base_dir = DROPBOX_DIR if Dir.exist?(DROPBOX_DIR) && Dir[File.join(DROPBOX_DIR, '**', '*')].count { |file| File.file?(file) } > 1 42 | np_base_dir = ICLOUDDRIVE_DIR if Dir.exist?(ICLOUDDRIVE_DIR) && Dir[File.join(ICLOUDDRIVE_DIR, '**', '*')].count { |file| File.file?(file) } > 1 43 | np_base_dir = CLOUDKIT_DIR if Dir.exist?(CLOUDKIT_DIR) && Dir[File.join(CLOUDKIT_DIR, '**', '*')].count { |file| File.file?(file) } > 1 44 | NP_NOTES_DIR = "#{np_base_dir}/Notes".freeze 45 | NP_CALENDAR_DIR = "#{np_base_dir}/Calendar".freeze 46 | 47 | #------------------------------------------------------------------------------- 48 | # Regex definitions (where they're likely to be re-used). NB need to be single quoted. 49 | #------------------------------------------------------------------------------- 50 | RE_DATE = '\d{4}[\-\.//][01]?\d[\-\.//]\d{1,2}' # built-in format for finding dates of form YYYY-MM-DD and similar 51 | RE_TIME = '\d{2}:\d{2}(?:.(?:AM|PM))?' # YYYY-MM-DD HH:MM[AM|PM] 52 | RE_DATE_TIME = RE_DATE + '\s' + RE_TIME 53 | RE_DATE_FORMAT_CUSTOM = '\d{1,2}[\-\.//][01]?\d[\-\.//]\d{4}'.freeze # regular expression of alternative format used to find dates in templates. This matches DD.MM.YYYY and similar. 54 | RE_DUE_DATE = '>' + RE_DATE # find '>2021-02-23' etc. 55 | RE_DUE_DATE_CAPTURE = '>(' + RE_DATE + ')' # find ' >2021-02-23' and return just date part 56 | RE_RESCHED_FROM_DATE = '<' + RE_DATE # find '<2021-02-23' etc. 57 | RE_DATE_INTERVAL = '[+\-]?\d+[bdwm]' 58 | RE_DATE_INTERVAL_CAPTURE = '(' + RE_DATE_INTERVAL + ')' 59 | RE_NOTE_LINK = '\[\[[^\#\]]+(\#[^\]]+)?\]\]' # find '[[note title]]' with optional #heading (not greedy) 60 | RE_NOTE_LINK_CAPTURE = '\[\[([^\#\]]+(\#[^\]]+)?)\]\]' # find '[[note title]]' (not greedy) 61 | RE_DONE_DATE_TIME = '@done\(' + RE_DATE_TIME + '\)' # find '@done(YYYY-MM-DD HH:mm)' markers 62 | RE_DONE_DATE_OPT_TIME = '@done\(' + RE_DATE + '(\s'+RE_TIME+')?\)' # find '@done(YYYY-MM-DD HH:mm)' markers (with optional time) 63 | RE_DONE_TASK_OR_CHECKLIST = '^\h*[\*\-\+]\s\[x\]\s' 64 | 65 | # Test RE_NOTE_LINK 66 | # puts 'invalid [[]] link' =~ /#{RE_NOTE_LINK}/ 67 | # puts 'invalid [[#]] link' =~ /#{RE_NOTE_LINK}/ 68 | # puts 'invalid [[#heading]] link' =~ /#{RE_NOTE_LINK}/ 69 | # puts '[[note title#heading again]]' =~ /#{RE_NOTE_LINK}/ 70 | # puts 'this is a [[note#heading]] link' =~ /#{RE_NOTE_LINK}/ 71 | 72 | # Test RE_NOTE_LINK_CAPTURE 73 | # puts 'invalid [[]] link'.match(/#{RE_NOTE_LINK_CAPTURE}/) 74 | # puts 'invalid [[#]] link'.match(/#{RE_NOTE_LINK_CAPTURE}/) 75 | # puts 'invalid [[#heading]] link'.match(/#{RE_NOTE_LINK_CAPTURE}/) 76 | # puts '[[note title#heading again]]'.match(/#{RE_NOTE_LINK_CAPTURE}/) 77 | # puts 'this is a [[note#heading]] link'.match(/#{RE_NOTE_LINK_CAPTURE}/) 78 | 79 | # Test RE_DONE_DATE_TIME 80 | # puts '@done()' =~ /#{RE_DONE_DATE_TIME}/ 81 | # puts '@done(2020-01-01)' =~ /#{RE_DATE_TIME}/ 82 | # puts '@done(2020-01-01)' =~ /#{RE_DONE_DATE_TIME}/ 83 | # puts '@done(2020-01-01 12:34)' =~ /#{RE_DATE_TIME}/ 84 | # puts '@done(2020-01-01 12:34)' =~ /#{RE_DONE_DATE_TIME}/ 85 | # puts 'with @done(2020-01-01 12:34) stuff' =~ /#{RE_DONE_DATE_TIME}/ 86 | 87 | # Colours to use with the colorization gem 88 | # to show some possible combinations, run String.color_samples 89 | # to show list of possible modes, run puts String.modes (e.g. underline, bold, blink) 90 | String.disable_colorization false 91 | CompletedColour = :light_green 92 | InfoColour = :yellow 93 | ErrorColour = :light_red 94 | # Test to see if we're running interactively or in a batch mode: 95 | # if batch mode then disable colorisation which doesn't work in logs 96 | tty_code = `tty`.chomp 97 | String.disable_colorization true if tty_code == 'not a tty' 98 | 99 | # Variables that need to be globally available 100 | time_now = Time.now 101 | time_now_fmttd = time_now.strftime(DATE_TIME_LOG_FORMAT) 102 | $verbose = 0 103 | $archive = 0 104 | $remove_rescheduled = 1 105 | $allNotes = [] # to hold all note objects 106 | $notes = [] # to hold all note objects selected for processing 107 | $date_today = time_now.strftime(DATE_TODAY_FORMAT) 108 | $npfile_count = -1 # number of NPFile objects created so far (incremented before first use) 109 | 110 | #------------------------------------------------------------------------- 111 | # Helper functions 112 | #------------------------------------------------------------------------- 113 | 114 | def main_message(message) 115 | puts message.colorize(CompletedColour) unless $quiet 116 | end 117 | 118 | def log_message(message) 119 | puts message if $verbose > 0 && !$quiet 120 | end 121 | 122 | def log_verbose_message(message) 123 | puts message if $verbose > 1 && !$quiet 124 | end 125 | 126 | def warning_message(message) 127 | puts message.colorize(InfoColour) 128 | end 129 | 130 | def error_message(message) 131 | puts message.colorize(ErrorColour) 132 | end 133 | 134 | def calc_offset_date(old_date, interval) 135 | # Calculate next review date, assuming: 136 | # - old_date is type 137 | # - interval is string of form nn[bdwmq] 138 | # - where 'b' is weekday (i.e. Monday-Friday in English) 139 | days_to_add = 0 140 | unit = interval[-1] # i.e. get last character 141 | num = interval.chop.to_i 142 | case unit 143 | when 'b' # week days 144 | # Method from Arjen at https://stackoverflow.com/questions/279296/adding-days-to-a-date-but-excluding-weekends 145 | # Avoids looping, and copes with negative intervals too 146 | current_day_of_week = old_date.strftime("%u").to_i # = day of week with Monday = 0, .. Sunday = 6 147 | dayOfWeek = num.negative? ? (current_day_of_week - 12).modulo(7) : (current_day_of_week + 6).modulo(7) 148 | num -= 1 if dayOfWeek == 6 149 | num += 1 if dayOfWeek == -6 150 | days_to_add = num + (num + dayOfWeek).div(5) * 2 151 | when 'd' 152 | days_to_add = num 153 | when 'w' 154 | days_to_add = num * 7 155 | when 'm' 156 | days_to_add = num * 30 # on average. Better to use >> operator, but it only works for months 157 | when 'q' 158 | days_to_add = num * 91 # on average 159 | when 'y' 160 | days_to_add = num * 365 # on average 161 | else 162 | error_message(" Error in calc_offset_date from #{old_date} by #{interval}") 163 | end 164 | log_verbose_message(" c_o_d: with #{old_date} interval #{interval} found #{days_to_add} days_to_add") 165 | return old_date + days_to_add 166 | end 167 | 168 | def find_daily_note(date_string) 169 | # Read in a note that we want to update. If it doesn't exist, create it. 170 | log_verbose_message(" - starting find_daily_note for #{date_string}") 171 | filename = "#{date_string}.#{NOTE_EXT}" 172 | noteToAddTo = nil # for an integer, but starting as nil 173 | 174 | # First check if it exists in existing notes read in 175 | $allNotes.each do |nn| 176 | next if nn.filename != filename 177 | 178 | noteToAddTo = nn.id 179 | log_verbose_message(" - found match via filename (id #{noteToAddTo}) ") 180 | end 181 | 182 | if noteToAddTo.nil? 183 | # now try reading in an existing daily note 184 | Dir.chdir(NP_CALENDAR_DIR) 185 | log_message(" - Looking for daily note filename #{filename}:") 186 | if File.exist?(filename) 187 | $allNotes << NPFile.new(filename) 188 | # now find the id of this most-recently-added NPFile instance 189 | noteToAddTo = $npfile_count 190 | log_verbose_message(" - read in match via filename (-> id #{noteToAddTo}) ") 191 | else 192 | # warn user it doesn't exist 193 | warning_message(" - warning: can't find matching note filename '#{filename}'") 194 | end 195 | end 196 | return noteToAddTo 197 | end 198 | 199 | def find_weekly_note(date_string) 200 | # Note: Not yet used 201 | # Read in a note that we want to update. If it doesn't exist, create it. 202 | log_verbose_message(" - starting find_weekly_note for #{date_string}") 203 | filename = "#{date_string}.#{NOTE_EXT}" 204 | noteToAddTo = nil # for an integer, but starting as nil 205 | 206 | # First check if it exists in existing notes read in 207 | $allNotes.each do |nn| 208 | next if nn.filename != filename 209 | 210 | noteToAddTo = nn.id 211 | log_verbose_message(" - found match via filename (id #{noteToAddTo}) ") 212 | end 213 | 214 | if noteToAddTo.nil? 215 | # now try reading in an existing weekly note 216 | Dir.chdir(NP_CALENDAR_DIR) 217 | log_message(" - Looking for weekly note filename #{filename}:") 218 | if File.exist?(filename) 219 | $allNotes << NPFile.new(filename) 220 | # now find the id of this most-recently-added NPFile instance 221 | noteToAddTo = $npfile_count 222 | log_verbose_message(" - read in match via filename (-> id #{noteToAddTo}) ") 223 | else 224 | # warn user it doesn't exist 225 | warning_message(" - warning: can't find matching note filename '#{filename}'") 226 | end 227 | end 228 | return noteToAddTo 229 | end 230 | 231 | def find_note(title) 232 | # Read in a note that we want to update. 233 | # Error if note can't be found, and return nil 234 | 235 | # NOTE: In NP v2.4+ there's a slight issue that there can be duplicate 236 | # note titles over different sub-folders. This will likely be improved in 237 | # the future, but for now I'll try to select the most recently-changed if 238 | # there are matching names. 239 | 240 | log_verbose_message(" - starting find_note for '#{title}'") 241 | new_note_id = nil # for an integer, but starting as nil 242 | 243 | # First check if it exists in existing notes read in 244 | mtime = Time.new(1970, 1, 1) # i.e. the earlist possible time 245 | $allNotes.each do |nn| 246 | next if nn.title != title 247 | 248 | next unless nn.modified_time > mtime 249 | 250 | new_note_id = nn.id 251 | mtime = nn.modified_time 252 | log_verbose_message(" - found existing match via title (id #{new_note_id}) last modified #{mtime}") 253 | end 254 | 255 | if new_note_id.nil? 256 | # not found, so give an error 257 | error_message(" - error: can't find matching note title '#{title}'") 258 | end 259 | return new_note_id 260 | end 261 | 262 | def osascript(script) 263 | # Run applescript 264 | # from gist https://gist.github.com/dinge/6983008 265 | log_verbose_message("About to execute this AppleScript:\n#{script}\n") 266 | system 'osascript', *script.split(/\n/).map { |line| ['-e', line] }.flatten 267 | end 268 | 269 | #------------------------------------------------------------------------- 270 | # Class definition: NPFile 271 | # NB: in this script this class covers Note *and* Daily *and* Weekly files 272 | #------------------------------------------------------------------------- 273 | class NPFile 274 | # Define the attributes that need to be visible outside the class instances 275 | attr_reader :id 276 | attr_reader :title 277 | attr_reader :cancelled_heading 278 | attr_reader :done_heading 279 | attr_reader :filename 280 | attr_reader :is_today 281 | attr_reader :is_calendar 282 | attr_reader :is_updated 283 | attr_reader :line_count 284 | attr_reader :modified_time 285 | 286 | def initialize(this_file) 287 | # Create NPFile object from reading 'this_file' file 288 | 289 | # Set variables that are visible outside the class instance 290 | $npfile_count += 1 291 | @id = $npfile_count 292 | @filename = this_file 293 | @modified_time = File.exist?(filename) ? File.mtime(this_file) : 0 294 | @title = '' 295 | @lines = [] 296 | @line_count = 0 297 | @cancelled_heading = 0 298 | @done_heading = 0 299 | @is_today = false 300 | @is_calendar = false 301 | @is_updated = false 302 | 303 | # initialise other variables (that don't need to persist with the class) 304 | n = 0 305 | 306 | # Open file and read in all lines (finding any Done and Cancelled headers) 307 | # NB: needs the encoding line when run from launchctl, otherwise you get US-ASCII invalid byte errors (basically the 'locale' settings are different) 308 | f = File.open(@filename, 'r', encoding: 'utf-8') 309 | f.each_line do |line| 310 | @lines[n] = line 311 | @done_heading = n if line =~ /^## Done$/ 312 | @cancelled_heading = n if line =~ /^## Cancelled$/ 313 | n += 1 314 | end 315 | f.close 316 | @line_count = @lines.size 317 | # Now make a title for this file: 318 | if @filename =~ /\d{8}\.(txt|md)/ 319 | # for Daily Calendar file, use the date from filename 320 | @title = @filename[0..7] 321 | @is_calendar = true 322 | @is_today = @title == $date_today 323 | 324 | elsif @filename =~ /\d{4}-W\d{2}\.(txt|md)/ 325 | # for Weekly Calendar file, use the date from filename 326 | @title = @filename[0..8] 327 | @is_calendar = true # TODO: review what this implies 328 | 329 | elsif @lines[0] =~ /^---/ 330 | # for Note file, find from frontmatter if present 331 | # look for 'title:' in frontmatter 332 | fn = 1 333 | in_frontmatter = true 334 | temp_title = '' 335 | while in_frontmatter && fn <= @line_count 336 | in_frontmatter = false if (@lines[fn] =~ /^---/) 337 | if @lines[fn] =~ /^[Tt]itle:\s+\S+/ 338 | @lines[fn].scan(/^[Tt]itle:\s+(.*)/) { |m| temp_title = m.join } 339 | end 340 | fn += 1 341 | end 342 | @title = !temp_title.empty? ? temp_title : 'temp_header' # but check it doesn't get to be blank 343 | @is_calendar = false 344 | @is_today = false 345 | 346 | else 347 | # otherwise use first line (but take off heading characters at the start and starting and ending whitespace) 348 | @title = @lines[0].gsub(/^#+\s*/, '').gsub(/\s+$/, '') 349 | @is_calendar = false 350 | @is_today = false 351 | end 352 | 353 | log_verbose_message(" Init NPFile #{@id}: #{@line_count} lines from #{this_file}, updated #{(@modified_time.to_s)[0..15]}".colorize(InfoColour)) 354 | end 355 | 356 | # def self.new2(*args) 357 | # # TODO: Use NotePlan's addNote via x-callback-url instead? 358 | # # This is a second initializer, to create a new empty file, so have to use a different syntax. 359 | # # Create empty NPFile object, and then pass to detailed initializer 360 | # object = allocate 361 | # object.create_new_note_file(*args) 362 | # object # implicit return 363 | # end 364 | 365 | # def append_new_line(new_line) 366 | # # Append 'new_line' into position 367 | # # TODO: should ideally split on '\n' and add each potential line separately 368 | # log_verbose_message(' append_new_line ...') 369 | # @lines << new_line 370 | # @line_count = @lines.size 371 | # end 372 | 373 | def create_events_from_timeblocks 374 | # Create calendar event in default calendar from an NP timeblock given in 375 | # a daily note, where #create_event is specified. 376 | # (As of NP 3.0 time blocking only works in headers and tasks, but Eduard has 377 | # said he will add to bullets as well, so I'm doing that already.) 378 | # Examples: 379 | # '* Write proposal at 12-14 #create_event' --> caledar event 12-2pm 380 | # '### Write proposal >2020-12-20 at 2pm #create_event' --> caledar event 2pm for 1 hour on that date 381 | # '- clear before meeting 2:00-2:30pm #create_event' --> caledar event 2-2:30pm 382 | log_verbose_message(' create_events_from_timeblocks ...') 383 | n = 0 384 | while n < (@done_heading.positive? ? @done_heading : @line_count) 385 | this_line = @lines[n] 386 | unless this_line =~ /#{CREATE_EVENT_TAG_TO_USE}/ 387 | n += 1 388 | next 389 | end 390 | # we have a line with one or more events to create 391 | # get date: if there's a >YYYY-MM-DD mentioned in the line, use that, 392 | # otherwise use date of calendar note. Format: YYYYMMDD, or else use today's date. 393 | event_date_s = '' 394 | if this_line =~ /#{RE_DUE_DATE}/ 395 | this_line.scan(/#{RE_DUE_DATE_CAPTURE}/) { |m| event_date_s = m.join.tr('-', '') } 396 | log_verbose_message(" - found event creation date spec: #{event_date_s}") 397 | elsif @is_calendar 398 | event_date_s = @filename[0..7] 399 | log_verbose_message(" - defaulting to create event on day: #{event_date_s}") 400 | else 401 | event_date_s = $date_today 402 | log_verbose_message(" - defaulting to create event today: #{event_date_s}") 403 | end 404 | # make title: strip off #create_event, time strings, header/task/bullet punctuation, and any location info 405 | event_title = this_line.chomp 406 | event_title.gsub!(/ #{CREATE_EVENT_TAG_TO_USE}/, '') 407 | event_title.gsub!(/^\s*[*->](\s\[.\])?\s*/, '') 408 | event_title.gsub!(/^#+\s*/, '') 409 | event_title.gsub!(/\s\d\d?(-\d\d?)?(am|pm|AM|PM)/, '') # 3PM, 9-11am etc. 410 | event_title.gsub!(/\s\d\d?:\d\d(-\d\d?:\d\d)?(am|pm|AM|PM)?/, '') # 3:00PM, 9:00-9:45am etc. 411 | event_title.gsub!(/#{RE_DUE_DATE}/, '') 412 | event_title.gsub!(/\sat\s.*$/, '') 413 | 414 | # Get times for event. 415 | # If no end time given, default to a 1-hour duration event. 416 | # NB: See https://github.com/jgclark/NotePlan-tools/issues/37 for details of an oddity with AppleScript, 417 | # which means we have to use time format of "HH:MM[ ]am|AM|pm|PM" not "HH:MM:SS" or "HH:MM" 418 | start_mins = end_mins = start_hour = end_hour = 0 419 | time_parts = [] 420 | if this_line =~ /[^\d-]\d\d?[:.]\d\d-\d\d?[:.]\d\d(am|pm)?[\s$]/i 421 | # times of form '3:00-4:00am', '3.00-3.45PM' etc. 422 | time_parts_da = this_line.scan(/[^\d-](\d\d?)[:.](\d\d)-(\d\d?)[:.](\d\d)(am|pm)?[\s$]/i) 423 | time_parts = time_parts_da[0] 424 | log_verbose_message(" - time_spec type 1: #{time_parts}") 425 | start_hour = time_parts[4] =~ /pm/i ? time_parts[0].to_i + 12 : time_parts[0].to_i 426 | start_mins = time_parts[1].to_i 427 | end_hour = time_parts[4] =~ /pm/i ? time_parts[2].to_i + 12 : time_parts[2].to_i 428 | end_mins = time_parts[3].to_i 429 | elsif this_line =~ /[^\d-]\d\d?[:.]\d\d(am|pm|AM|PM)?[\s$]/i 430 | # times of form '3:15[am|pm]' 431 | time_parts_da = this_line.scan(/[^\d-](\d\d?)[:.](\d\d)(am|pm)?[\s$]/i) 432 | time_parts = time_parts_da[0] 433 | log_message(" - time_spec type 2: #{time_parts}") 434 | start_hour = time_parts[2] =~ /pm/i ? time_parts[0].to_i + 12 : time_parts[0].to_i 435 | start_mins = time_parts[1].to_i 436 | end_hour = (start_hour + 1).modulo(24) # cope with event crossing midnight 437 | end_mins = start_mins 438 | elsif this_line =~ /[^\d-]\d\d?(am|pm)[\s$]/i 439 | # times of form '3am|PM' 440 | time_parts_da = this_line.scan(/[^\d-](\d\d?)(am|pm)[\s$]/i) 441 | time_parts = time_parts_da[0] 442 | log_verbose_message(" - time_spec type 3: #{time_parts}") 443 | start_hour = time_parts[1] =~ /pm/i ? time_parts[0].to_i + 12 : time_parts[0].to_i 444 | start_mins = 0 445 | end_hour = (start_hour + 1).modulo(24) # cope with event crossing midnight 446 | end_mins = 0 447 | elsif this_line =~ /[^\d-]\d\d?-\d\d?(am|pm)[\s$]/i 448 | # times of form '3-5am|pm' 449 | time_parts_da = this_line.scan(/[^\d-](\d\d?)-(\d\d?)(am|pm)[\s$]/i) 450 | time_parts = time_parts_da[0] 451 | log_verbose_message(" - time_spec type 4: #{time_parts}") 452 | start_hour = time_parts[2] =~ /pm/i ? time_parts[0].to_i + 12 : time_parts[0].to_i 453 | start_mins = 0 454 | end_hour = time_parts[2] =~ /pm/i ? time_parts[1].to_i + 12 : time_parts[1].to_i 455 | end_mins = 0 456 | elsif this_line =~ /[^\d-]\d\d?-\d\d?[\s$]/i 457 | # times of form '3-5', implied 24-hour clock 458 | time_parts_da = this_line.scan(/[^\d-](\d\d?)-(\d\d?)[\s$]/i) 459 | time_parts = time_parts_da[0] 460 | log_verbose_message(" - time_spec type 5: #{time_parts}") 461 | start_hour = time_parts[0].to_i 462 | start_mins = 0 463 | end_hour = time_parts[1].to_i 464 | end_mins = 0 465 | else 466 | # warn as can't find suitable time String 467 | warning_message(" - want to create '#{event_title}' event through #create_event, but cannot find suitable time spec") 468 | n += 1 469 | next 470 | end 471 | # create start and end datetime formats to use in applescript 472 | start_dt = DateTime.new(event_date_s[0..3].to_i, event_date_s[4..5].to_i, event_date_s[6..7].to_i, start_hour, start_mins, 0) 473 | end_dt = DateTime.new(event_date_s[0..3].to_i, event_date_s[4..5].to_i, event_date_s[6..7].to_i, end_hour, end_mins, 0) 474 | # deal with special case of event crossing midnight, where we need to add 1 day to end_dt 475 | if end_dt < start_dt 476 | puts " - found special case of crossing midnight:" 477 | print " #{start_dt} - #{end_dt} " 478 | end_dt += 1 479 | puts " --> #{end_dt}" 480 | end 481 | start_dt_s = start_dt.strftime(DATE_TIME_APPLESCRIPT_FORMAT) 482 | end_dt_s = end_dt.strftime(DATE_TIME_APPLESCRIPT_FORMAT) 483 | log_message(" - will create event '#{event_title}' from #{start_dt_s} to #{end_dt_s}") 484 | 485 | # use ' at X...' to set the_location (rather than that type of timeblocking) 486 | the_location = this_line =~ /\sat\s.*/ ? this_line.scan(/\sat\s(.*)/).join : '' 487 | 488 | # Copy any indented comments/notes into the_description field 489 | the_description = '' 490 | # Incrementally add lines until we find ones at the same or lower level of indent. 491 | # (similar to code from move_daily_ref_to_notes) 492 | line_indent = '' 493 | this_line.scan(/^(\s*)\*/) { |m| line_indent = m.join } 494 | log_verbose_message(" - building event description with starting indent of #{line_indent.length}") 495 | nn = n + 1 496 | while nn < @line_count 497 | line_to_check = @lines[nn] 498 | # What's the indent of this line? 499 | line_to_check_indent = '' 500 | line_to_check.scan(/^(\s*)\S/) { |m| line_to_check_indent = m.join } 501 | break if line_indent.length >= line_to_check_indent.length 502 | 503 | the_description += line_to_check.lstrip # add this line to the description, with leading whitespace removed 504 | nn += 1 505 | end 506 | 507 | # Now write the AppleScript and run it 508 | begin 509 | osascript <<-APPLESCRIPT 510 | set calendarName to "#{CALENDAR_NAME_TO_USE}" 511 | set theSummary to "#{event_title}" 512 | set theDescrption to "#{the_description}" 513 | set theLocation to "#{the_location}" 514 | set startDate to "#{start_dt_s}" 515 | set endDate to "#{end_dt_s}" 516 | set startDate to date startDate 517 | set endDate to date endDate 518 | if application "#{CALENDAR_APP_TO_USE}" is not running then 519 | launch application "#{CALENDAR_APP_TO_USE}" # hoped this would start it without a window, but not so 520 | delay 3 # pause for 3 seconds while app launches 521 | end if 522 | tell application "Calendar" 523 | tell (first calendar whose name is calendarName) 524 | make new event at end of events with properties {summary:theSummary, start date:startDate, end date:endDate, description:theDescrption, location:theLocation} 525 | end tell 526 | end tell 527 | APPLESCRIPT 528 | # Now update the line to show #event_created not #create_event 529 | @lines[n].gsub!(CREATE_EVENT_TAG_TO_USE, '#event_created') 530 | @is_updated = true 531 | n += 1 532 | rescue StandardError => e 533 | error_message("ERROR: #{e.exception.message} when calling AppleScript to create an event") 534 | end 535 | end 536 | end 537 | 538 | def clear_empty_tasks_or_headers 539 | # Clean up lines with just * or - or #s in them 540 | log_verbose_message(' remove_empty_tasks_or_headers ...') 541 | n = cleaned = 0 542 | while n < @line_count 543 | # blank any lines which just have a * or - 544 | if @lines[n] =~ /^\s*[*\-]\s*$/ 545 | @lines[n] = '' 546 | cleaned += 1 547 | end 548 | # blank any lines which just have #s at the start (and optional following whitespace) 549 | if @lines[n] =~ /^#+\s?$/ 550 | @lines[n] = '' 551 | cleaned += 1 552 | end 553 | n += 1 554 | end 555 | return unless cleaned.positive? 556 | 557 | @is_updated = true 558 | @line_count = @lines.size 559 | log_message(" - removed #{cleaned} empty lines") 560 | end 561 | 562 | def insert_new_line_at_line(new_line, line_number) 563 | # Insert 'new_line' into position 'line_number' 564 | # don't go beyond current size of @lines 565 | # Doesn't write out to file, but does update @lines and @line_count. 566 | n = line_number >= @lines.size ? @lines.size : line_number 567 | log_verbose_message(" - insert_new_line_at_line #{n}...") 568 | # break line up into separate lines (on "\n") 569 | line_a = new_line.split("\n") 570 | line_a.each do |line| 571 | @lines.insert(n, line) 572 | n += 1 573 | end 574 | @line_count = @lines.size 575 | end 576 | 577 | def prepend_line_to_section(new_line, section_heading) 578 | # Insert 'new_line' at start of a section headed 'section_heading' 579 | # If this is blank, then insert after start-of-note metadata 580 | log_message(" - prepend_line_to_section '#{section_heading}' ...") 581 | max = @lines.size 582 | line_number = max # as a fallback treat this as an append 583 | n = 0 # start iterating from the start of line of he file 584 | if section_heading.empty? 585 | # There's no section_heading to find, so insert after frontmatter 586 | in_frontmatter = false 587 | while n <= max 588 | this_line = @lines[n].chomp 589 | # if we have a blank line or the end of a YAML frontmatter section 590 | if this_line.empty? || in_frontmatter && (this_line =~ /^\.\.\./ || this_line =~ /^---/) 591 | line_number = n + 1 # point to next line 592 | break # stop looking 593 | end 594 | in_frontmatter = true if this_line =~ /^---/ 595 | # if we have a section heading or end of a YAML frontmatter section 596 | if this_line =~ /^##+\s+/ 597 | line_number = n # point to this line (inserts before it) 598 | break # stop looking 599 | end 600 | n += 1 601 | end 602 | else # we want to find the section heading 603 | while n <= max 604 | if @lines[n] =~ /^#+\s+#{section_heading}/ 605 | line_number = n + 1 # point to line after title 606 | break # stop looking 607 | end 608 | n += 1 609 | end 610 | end 611 | insert_new_line_at_line(new_line, line_number) 612 | end 613 | 614 | # TODO: split into two cases; not sure one is really needed 615 | def append_line_to_section(new_line, section_heading) 616 | # Append new_line after 'section_heading' line. 617 | # If not found, then add 'section_heading' to the end first 618 | # If 'section_heading' is blank, then append in first section after frontmatter, informally defined (i.e. doesn't have to start with ---) 619 | log_verbose_message(" - append_line_to_section for '#{section_heading}' ...") 620 | n = 0 621 | max = @lines.size 622 | line_number = max # as a fallback treat this as an append 623 | found_section = false 624 | if section_heading.empty? 625 | # There's no section_heading to find, so find end of frontmatter instead 626 | while n < max 627 | this_line = @lines[n].chomp 628 | # if we have a blank line or the end of a YAML frontmatter section 629 | if this_line.empty? && (this_line =~ /^\.\.\./ || this_line =~ /^---/) 630 | line_number = n # point to next line 631 | break # stop looking 632 | end 633 | # if we have a section heading 634 | if this_line =~ /^##+\s+/ 635 | line_number = n # point to this line (inserts before it) 636 | break # stop looking 637 | end 638 | n += 1 639 | end 640 | found_section = true # we have found the equivalent of the section heading 641 | n = line_number 642 | # log_message(" empty heading: insertion point at line #{n}") 643 | end 644 | # find the section heading 645 | added = false 646 | while !added && (n < max) 647 | line = @lines[n].chomp 648 | # if an empty line or a new header section starting, insert line here 649 | if found_section && (line.empty? || line =~ /^#+\s/) 650 | insert_new_line_at_line(new_line, n) 651 | added = true 652 | end 653 | # if this is the section header of interest, save its details. (Needs to come after previous test.) 654 | found_section = true if line =~ /^#{section_heading}/ 655 | n += 1 656 | end 657 | log_verbose_message(" final part with found_section #{found_section}, added #{added}") 658 | insert_new_line_at_line(new_line, n) unless added # if not added so far, then now append 659 | insert_new_line_at_line(section_heading, n) unless found_section # if section not yet found then add it before this line 660 | end 661 | 662 | def remove_checklist_done_markers 663 | # removes @done(...) markers in done checklist items 664 | log_verbose_message(' remove_finished_tags_dates ...') 665 | n = cleaned = 0 666 | while n < @line_count 667 | # only do something if this is a completed or cancelled task 668 | if @lines[n] =~ /\s*\+\s+\[(x|-)\]/ 669 | if @lines[n] =~ /\s#{RE_DONE_DATE_OPT_TIME}/ 670 | @lines[n].gsub!(/\s#{RE_DONE_DATE_OPT_TIME}/, '') 671 | cleaned += 1 672 | end 673 | end 674 | n += 1 675 | end 676 | return unless cleaned.positive? 677 | 678 | @is_updated = true 679 | log_message(" - removed #{cleaned} @done() marker(s) from checklist item(s)") 680 | end 681 | 682 | def remove_finished_tags_dates 683 | # removes specific tags and >dates from complete or cancelled tasks 684 | log_verbose_message(' remove_finished_tags_dates ...') 685 | n = cleaned = 0 686 | while n < @line_count 687 | # only do something if this is a completed or cancelled task 688 | if @lines[n] =~ /\[(x|-)\]/ 689 | # remove any ] tasks from calendar notes, as there will be a duplicate 716 | # (whether or not the 'Append links when scheduling' option is set or not) 717 | log_verbose_message(' remove_rescheduled ...') 718 | n = cleaned = 0 719 | while n < @line_count 720 | # Empty any [>] todo lines 721 | if @lines[n] =~ /\[>\]/ 722 | @lines.delete_at(n) 723 | @line_count -= 1 724 | n -= 1 725 | cleaned += 1 726 | end 727 | n += 1 728 | end 729 | return unless cleaned.positive? 730 | 731 | @is_updated = true 732 | log_message(" - removed #{cleaned} scheduled") 733 | end 734 | 735 | def move_daily_ref_to_daily 736 | # Moves items in daily notes with a >date to that corresponding date. 737 | # Checks whether the note exists and if not, creates one first at top level. 738 | log_verbose_message(' move_daily_ref_to_daily ...') 739 | # noteToAddTo = nil 740 | n = -1 741 | moved = 0 742 | while n < @line_count 743 | n += 1 744 | line = @lines[n] 745 | # only continue with this line if has a >date mention 746 | next unless line =~ /#{RE_DUE_DATE}/ 747 | 748 | # the following regex matches returns an array with one item, so make a string (by join) 749 | # NOTE: the '+?' gets minimum number of chars, to avoid grabbing contents of several [[notes]] in the same line 750 | yyyy_mm_dd = '' 751 | line.scan(/>(\d{4}-\d{2}-\d{2})/) { |m| yyyy_mm_dd = m.join } 752 | log_message(" - found calendar link >#{yyyy_mm_dd} in notes on line #{n + 1} of #{@line_count}") 753 | yyyymmdd = "#{yyyy_mm_dd[0..3]}#{yyyy_mm_dd[5..6]}#{yyyy_mm_dd[8..9]}" 754 | 755 | # Find the existing daily note to add to, or read in, or create 756 | noteToAddTo = find_daily_note(yyyymmdd) 757 | lines_to_output = '' 758 | 759 | # Remove the >date text by finding string points 760 | label_start = line.index('>') - 1 # remove space before it as well. TODO: could be several > so find the right one 761 | label_end = label_start + 12 762 | # also chomp off last character of line (newline) 763 | line = "#{line[0..label_start]}#{line[label_end..-2]}" 764 | 765 | is_heading = line =~ /^#+\s+.*/ ? true : false 766 | 767 | if !is_heading 768 | # If no due date is specified in rest of the line, add date from the title of the calendar file it came from 769 | if line !~ /#{RE_DUE_DATE}/ 770 | cal_date = "#{@title[0..3]}-#{@title[4..5]}-#{@title[6..7]}" 771 | log_verbose_message(" - '>#{cal_date}' to add from #{@title}") 772 | lines_to_output = line + " <#{cal_date}\n" 773 | else 774 | lines_to_output = line 775 | end 776 | # Work out indent level of current line 777 | line_indent = '' 778 | line.scan(/^(\s*)\*/) { |m| line_indent = m.join } 779 | log_verbose_message(" - starting line analysis at line #{n + 1} of #{@line_count} (indent #{line_indent.length})") 780 | 781 | # Remove this line from the calendar note 782 | @lines.delete_at(n) 783 | @line_count -= 1 784 | moved += 1 785 | 786 | # We also want to take any following indented lines 787 | # So incrementally add lines until we find ones at the same or lower level of indent 788 | while n < @line_count 789 | line_to_check = @lines[n] 790 | # What's the indent of this line? 791 | line_to_check_indent = '' 792 | line_to_check.scan(/^(\s*)\S/) { |m| line_to_check_indent = m.join } 793 | log_verbose_message(" - for '#{line_to_check.chomp}' (indent #{line_to_check_indent.length})") 794 | break if line_indent.length >= line_to_check_indent.length 795 | 796 | lines_to_output += line_to_check 797 | # Remove this line from the calendar note 798 | @lines.delete_at(n) 799 | @line_count -= 1 800 | moved += 1 801 | end 802 | else 803 | # This is a header line ... 804 | # We want to take any following lines up to the next blank line or same-level header. 805 | # So incrementally add lines until we find that break. 806 | heading_marker = '' 807 | line.scan(/^(#+)\s/) { |m| heading_marker = m.join } 808 | lines_to_output = line + "\n" 809 | @lines.delete_at(n) 810 | @line_count -= 1 811 | moved += 1 812 | log_verbose_message(" - starting header analysis at line #{n + 1}") 813 | 814 | while n < @line_count 815 | line_to_check = @lines[n] 816 | log_verbose_message(" - l_t_o checking '#{line_to_check}'") 817 | break if (line_to_check =~ /^\s*$/) || (line_to_check =~ /^#{heading_marker}\s/) 818 | 819 | lines_to_output += line_to_check 820 | # Remove this line from the calendar note 821 | log_verbose_message(" - @line_count now #{@line_count}") 822 | @lines.delete_at(n) 823 | @line_count -= 1 824 | moved += 1 825 | end 826 | end 827 | 828 | # insert updated line(s) in the daily note file in section DAILY_TASKS_SECTION_NAME (or after header if blank) 829 | $allNotes[noteToAddTo].append_line_to_section(lines_to_output, DAILY_TASKS_SECTION_NAME) 830 | 831 | # write the note file out 832 | $allNotes[noteToAddTo].rewrite_file 833 | end 834 | return unless moved.positive? 835 | 836 | @is_updated = true 837 | log_message(" - moved #{moved} lines to daily notes") 838 | end 839 | 840 | # TODO: see if Weekly notes should be included here too 841 | def move_daily_ref_to_notes(move_only_on_complete) 842 | # Move items in daily note with a [[note]] link to that note, inserting after Title, 843 | # or after the Heading if supplied in [[note#heading]]. 844 | # If move_only_on_complete is true, then only works if its a newly completed task/checklist. 845 | # Checks whether the note exists and if not, creates one first at top level. 846 | # TODO: should also check whether link is actually a date, and then do nothing. 847 | # NB: only does something with first [[note]] in a line 848 | log_verbose_message(' move_daily_ref_to_notes ...') 849 | note_link = nil 850 | note_name = nil 851 | note_heading = '' 852 | n = 0 853 | moved = 0 854 | while n < @line_count 855 | line = @lines[n] 856 | # find lines with [[note]] link mentions 857 | if line !~ /#{RE_NOTE_LINK}/ 858 | # this line doesn't match, so break out of loop and go to look at next line 859 | n += 1 860 | next 861 | end 862 | 863 | # if move_only_on_complete is set, then only proceed if this is a completed task or checklist item 864 | if move_only_on_complete && line !~ /#{RE_DONE_TASK_OR_CHECKLIST}/ 865 | # this line doesn't match, so break out of loop and go to look at next line 866 | log_message(" - skipping note link in incomplete task '#{line.chomp}'") 867 | n += 1 868 | next 869 | else 870 | log_message(" - moving note link in line '#{line.chomp}'".to_s.bold) 871 | end 872 | 873 | is_heading = line =~ /^#+\s+.*/ ? true : false 874 | 875 | # Get the first [[note]] link in a line with optional heading 876 | if line =~ /#{RE_NOTE_LINK}/ 877 | # the following regex matches returns an array with one item, so make a string (by join) 878 | line.scan(/#{RE_NOTE_LINK_CAPTURE}/) { |m| note_link = m.join } 879 | log_verbose_message(" - found note link [[#{note_link}]] in a heading on line #{n + 1} of #{@line_count}") if is_heading 880 | log_verbose_message(" - found note link [[#{note_link}]] in notes on line #{n + 1} of #{@line_count}") unless is_heading 881 | m = note_link.split('#') 882 | if m.length > 1 883 | note_name = m[0] 884 | note_heading = m[1] 885 | log_verbose_message(" = '#{note_name}' heading '#{note_heading}'") 886 | else 887 | note_name = note_link 888 | end 889 | end 890 | 891 | noteToAddTo = find_note(note_name) 892 | break if noteToAddTo.nil? 893 | 894 | # FIXME: there's also a slight bug in the in-line manipulation at end of @done() 895 | 896 | lines_to_output = '' 897 | 898 | # Remove the [[name]] text by finding first example of the string points 899 | label_start = line.index('[[') - 2 # remove space before it as well 900 | label_end = line.index(']]') + 2 901 | # also chomp off last character of line (newline) 902 | line = "#{line[0..label_start]}#{line[label_end..-2]}" 903 | 904 | if is_heading 905 | # This is a heading line. 906 | # We want to take any following lines up to the next blank line or same-level heading. 907 | # So incrementally add lines until we find that break. 908 | heading_marker = '' 909 | line.scan(/^(#+)\s/) { |m| heading_marker = m.join } 910 | lines_to_output = "#{line}\n" 911 | @lines.delete_at(n) 912 | @line_count -= 1 913 | moved += 1 914 | log_verbose_message(" - starting heading analysis at line #{n + 1}") 915 | 916 | while n < @line_count 917 | line_to_check = @lines[n] 918 | log_verbose_message(" - l_t_o checking '#{line_to_check}'") 919 | break if (line_to_check =~ /^\s*$/) || (line_to_check =~ /^#{heading_marker}\s/) 920 | 921 | lines_to_output += line_to_check 922 | # Remove this line from the calendar note 923 | log_verbose_message(" - @line_count now #{@line_count}") 924 | @lines.delete_at(n) 925 | @line_count -= 1 926 | moved += 1 927 | end 928 | else 929 | # This is not a heading line. 930 | # If no due date is specified in rest of the line, add date from the title of the calendar file it came from 931 | if line !~ /#{RE_DUE_DATE}/ 932 | cal_date = "#{@title[0..3]}-#{@title[4..5]}-#{@title[6..7]}" 933 | log_verbose_message(" - '#{cal_date}' to add from #{@title}") 934 | lines_to_output = line + " >#{cal_date}\n" 935 | else 936 | lines_to_output = line 937 | end 938 | # Work out indent level of current line 939 | line_indent = '' 940 | line.scan(/^(\s*)\*/) { |m| line_indent = m.join } 941 | log_verbose_message(" - starting line analysis at line #{n + 1} of #{@line_count} with indent '#{line_indent}' (#ine_indent.length})") 942 | # Remove this line from the calendar note 943 | @lines.delete_at(n) 944 | @line_count -= 1 945 | moved += 1 946 | 947 | # We also want to take any following indented lines 948 | # So incrementally add lines until we find ones at the same or lower level of indent 949 | while n < @line_count 950 | line_to_check = @lines[n] 951 | # What's the indent of this line? 952 | line_to_check_indent = '' 953 | line_to_check.scan(/^(\s*)\S/) { |m| line_to_check_indent = m.join } 954 | log_verbose_message(" - for '#{line_to_check.chomp}' indent='#{line_to_check_indent}' (#{line_to_check_indent.length})") 955 | break if line_indent.length >= line_to_check_indent.length 956 | 957 | lines_to_output += line_to_check 958 | # Remove this line from the calendar note 959 | @lines.delete_at(n) 960 | @line_count -= 1 961 | moved += 1 962 | end 963 | end 964 | 965 | # insert updated line(s) to the right section of the project note file 966 | # (or after header lines if no heading specified) 967 | # $allNotes[noteToAddTo].append_line_to_section(lines_to_output, note_heading) 968 | $allNotes[noteToAddTo].prepend_line_to_section(lines_to_output, note_heading) 969 | 970 | # write the note file out 971 | $allNotes[noteToAddTo].rewrite_file 972 | end 973 | return unless moved.positive? 974 | 975 | @is_updated = true 976 | log_message(" - moved #{moved} lines to notes") 977 | end 978 | 979 | def archive_lines 980 | # Shuffle @done and cancelled lines to relevant sections at end of the file 981 | # TODO: doesn't yet deal with notes with subheads in them 982 | log_verbose_message(' archive_lines ...') 983 | doneToMove = [] # NB: zero-based 984 | doneToMoveLength = [] # NB: zero-based 985 | cancToMove = [] # NB: zero-based 986 | cancToMoveLength = [] # NB: zero-based 987 | c = 0 988 | 989 | # Go through all lines between metadata and ## Done section 990 | # start, noting completed tasks 991 | n = 1 992 | searchLineLimit = @done_heading.positive? ? @done_heading : @line_count 993 | while n < searchLineLimit 994 | n += 1 995 | line = @lines[n] 996 | next unless line =~ /\*\s+\[x\]/ # TODO: change for different task markers 997 | 998 | # save this line number 999 | doneToMove.push(n) 1000 | # and look ahead to see how many lines to move -- all until blank or starting # or * 1001 | linesToMove = 0 1002 | while n < @line_count 1003 | break if (@lines[n + 1] =~ /^(#+\s+|\*\s+)/) || (@lines[n + 1] =~ /^\s*$/) # TODO: change for different task markers 1004 | 1005 | linesToMove += 1 1006 | n += 1 1007 | end 1008 | # save this length 1009 | doneToMoveLength.push(linesToMove) 1010 | end 1011 | log_verbose_message(" doneToMove: #{doneToMove} / #{doneToMoveLength}") 1012 | 1013 | # Do some done line shuffling, is there's anything to do 1014 | unless doneToMove.empty? 1015 | # If we haven't already got a Done section, make one 1016 | if @done_heading.zero? 1017 | @lines.push('') 1018 | @lines.push('## Done') 1019 | @line_count += 2 1020 | @done_heading = @line_count 1021 | end 1022 | 1023 | # Copy the relevant lines 1024 | doneInsertionLine = @cancelled_heading != 0 ? @cancelled_heading : @line_count 1025 | c = 0 1026 | doneToMove.each do |nn| 1027 | linesToMove = doneToMoveLength[c] 1028 | log_verbose_message(" Copying lines #{nn}-#{nn + linesToMove} to insert at #{doneInsertionLine}") 1029 | (nn..(nn + linesToMove)).each do |i| 1030 | @lines.insert(doneInsertionLine, @lines[i]) 1031 | @line_count += 1 1032 | doneInsertionLine += 1 1033 | end 1034 | c += 1 1035 | end 1036 | 1037 | # Now delete the original items (in reverse order to preserve numbering) 1038 | c = doneToMoveLength.size - 1 1039 | doneToMove.reverse.each do |nn| 1040 | linesToMove = doneToMoveLength[c] 1041 | log_verbose_message(" Deleting lines #{nn}-#{nn + linesToMove}") 1042 | (nn + linesToMove).downto(n) do |i| 1043 | @lines.delete_at(i) 1044 | @line_count -= 1 1045 | doneInsertionLine -= 1 1046 | @done_heading -= 1 1047 | end 1048 | c -= 1 1049 | end 1050 | end 1051 | 1052 | # Go through all lines between metadata and ## Done section 1053 | # start, noting cancelled line numbers 1054 | n = 0 1055 | searchLineLimit = @done_heading.positive? ? @done_heading : @line_count 1056 | while n < searchLineLimit 1057 | n += 1 1058 | line = @lines[n] 1059 | next unless line =~ /\*\s*\[-\]/ # TODO: change for different task markers 1060 | 1061 | # save this line number 1062 | cancToMove.push(n) 1063 | # and look ahead to see how many lines to move -- all until blank or starting # or * 1064 | linesToMove = 0 1065 | while n < @line_count 1066 | linesToMove += 1 1067 | break if (@lines[n + 1] =~ /^(#+\s+|\*\s+)/) || (@lines[n + 1] =~ /^\s*$/) # TODO: change for different task markers 1068 | 1069 | n += 1 1070 | end 1071 | # save this length 1072 | cancToMoveLength.push(linesToMove) 1073 | end 1074 | log_verbose_message(" cancToMove: #{cancToMove} / #{cancToMoveLength}") 1075 | 1076 | # Do some cancelled line shuffling, is there's anything to do 1077 | return if cancToMove.empty? 1078 | 1079 | # If we haven't already got a Cancelled section, make one 1080 | if @cancHeader.zero? 1081 | @lines.push('') 1082 | @lines.push('## Cancelled') 1083 | @line_count += 2 1084 | @cancHeader = @line_count 1085 | end 1086 | 1087 | # Copy the relevant lines 1088 | cancelledInsertionLine = @line_count 1089 | c = 0 1090 | cancToMove.each do |nn| 1091 | linesToMove = cancToMoveLength[c] 1092 | log_verbose_message(" Copying lines #{nn}-#{nn + linesToMove} to insert at #{cancelledInsertionLine}") 1093 | (nn..(nn + linesToMove)).each do |i| 1094 | @lines.insert(cancelledInsertionLine, @lines[i]) 1095 | @line_count += 1 1096 | cancelledInsertionLine += 1 1097 | end 1098 | c += 1 1099 | end 1100 | 1101 | # Now delete the original items (in reverse order to preserve numbering) 1102 | c = doneToMoveLength.size - 1 1103 | cancToMove.reverse.each do |nn| 1104 | linesToMove = doneToMoveLength[c] 1105 | log_verbose_message(" Deleting lines #{nn}-#{nn + linesToMove}") 1106 | (nn + linesToMove).downto(n) do |i| 1107 | log_verbose_message(" Deleting line #{i} ...") 1108 | @lines.delete_at(i) 1109 | @line_count -= 1 1110 | @done_heading -= 1 1111 | end 1112 | end 1113 | 1114 | # Finally mark note as updated 1115 | @is_updated = true 1116 | end 1117 | 1118 | def use_template_dates 1119 | # Take template dates and turn into real dates 1120 | 1121 | log_verbose_message(' use_template_dates ...') 1122 | date_string = '' 1123 | current_target_date = '' 1124 | calc_date = '' 1125 | last_was_template = false 1126 | n = 0 1127 | # Go through each line in the active part of the file 1128 | while n < (@done_heading.positive? ? @done_heading : @line_count) 1129 | line = @lines[n] 1130 | date_string = '' 1131 | # look for base date, of form YYYY-MM-DD and variations and whatever RE_DATE_FORMAT_CUSTOM gives 1132 | if line =~ /^#+\s/ 1133 | # clear previous settings when we get to a new heading 1134 | current_target_date = '' 1135 | last_was_template = false 1136 | end 1137 | 1138 | # Try matching for the standard YYYY-MM-DD date pattern 1139 | # (though check it's not got various characters before it, to defeat common usage in middle of things like URLs) 1140 | unless line != '' 1141 | line.scan(/[^\d(<>\/-](#{RE_DATE})/) { |m| date_string = m.join } 1142 | 1143 | if date_string != '' 1144 | # We have a date string to use for any offsets in the following section 1145 | current_target_date = date_string 1146 | log_verbose_message(" - Found CTD #{current_target_date}") 1147 | else 1148 | # Try matching for the custom date pattern, configured at the top 1149 | # (though check it's not got various characters before it, to defeat common usage in middle of things like URLs) 1150 | line.scan(/[^\d(<>\/-](#{RE_DATE_FORMAT_CUSTOM})/) { |m| date_string = m.join } 1151 | if date_string != '' 1152 | # We have a date string to use for any offsets in the following section 1153 | current_target_date = date_string 1154 | log_verbose_message(" - Found CTD #{current_target_date}") 1155 | end 1156 | end 1157 | if line =~ /#template/ 1158 | # We have a #template tag so ignore any offsets in the following section 1159 | last_was_template = true 1160 | log_verbose_message(" . Found #template in '#{line.chomp}'") 1161 | end 1162 | 1163 | # ignore line if we're in a template section (last_was_template is true) 1164 | unless last_was_template 1165 | # find lines with {+3d} or {-4w} etc. plus {0d} special case 1166 | # NB: this only deals with the first on any line; it doesn't make sense to have more than one. 1167 | date_offset_string = '' 1168 | if line =~ /\{#{RE_DATE_INTERVAL}\}/ 1169 | log_verbose_message(" - Found line '#{line.chomp}'") 1170 | line.scan(/\{(#{RE_DATE_INTERVAL_CAPTURE})\}/) { |m| date_offset_string = m.join } 1171 | # FIXME: line above seems to be returning '-18d-18d' for example. Though the code still works OK as calc_offset_date happens to parse it OK 1172 | if date_offset_string != '' 1173 | log_verbose_message(" - Found DOS #{date_offset_string} and last_was_template=#{last_was_template}") 1174 | if current_target_date != '' 1175 | begin 1176 | calc_date = calc_offset_date(Date.parse(current_target_date), date_offset_string) 1177 | rescue StandardError => e 1178 | error_message(" Error #{e.exception.message} while parsing date '#{current_target_date}' for #{date_offset_string}") 1179 | end 1180 | # Remove the offset text (e.g. {-3d}) by finding string points 1181 | label_start = line.index('{') 1182 | label_end = line.index('}') 1183 | # Create new version with inserted date 1184 | line = "#{line[0..label_start - 1]}>#{calc_date}#{line[label_end + 1..-2]}" # also chomp off last character (newline) 1185 | # then add the new date 1186 | # line += ">#{calc_date}" 1187 | @lines[n] = line 1188 | log_verbose_message(" - In line labels runs #{label_start}-#{label_end} --> '#{line.chomp}'") 1189 | @is_updated = true 1190 | elsif $verbose > 0 1191 | error_message(" Warning: have an offset date, but no current_target_date before line '#{line.chomp}'") 1192 | end 1193 | end 1194 | end 1195 | end 1196 | end 1197 | n += 1 1198 | end 1199 | end 1200 | 1201 | def process_repeats_and_done 1202 | # Process any completed (or cancelled) tasks with my extended @repeat(..) tags, 1203 | # and also remove the HH:MM portion of any @done(...) tasks. 1204 | # 1205 | # When interval is of the form +2w it will duplicate the task for 2 weeks 1206 | # after the date is was completed. 1207 | # When interval is of the form 2w it will duplicate the task for 2 weeks 1208 | # after the date the task was last due. If this can't be determined, 1209 | # then default to the first option. 1210 | # Valid intervals are [0-9][bdwmqy]. 1211 | # To work it relies on finding @done(YYYY-MM-DD HH:MM) tags that haven't yet been 1212 | # shortened to @done(YYYY-MM-DD). 1213 | # It includes cancelled tasks as well; to remove a repeat entirely, remoce 1214 | # the @repeat tag from the task in NotePlan. 1215 | log_verbose_message(' process_repeats_and_done ...') 1216 | n = cleaned = 0 1217 | # Go through each line in the active part of the file 1218 | while n < (@done_heading != 0 ? @done_heading : @line_count) 1219 | line = @lines[n] 1220 | updated_line = '' 1221 | completed_date = '' 1222 | # find lines with date-time to shorten, and capture date part of it 1223 | # i.e. @done(YYYY-MM-DD HH:MM[AM|PM]) 1224 | if line =~ /#{RE_DONE_DATE_TIME}/ 1225 | # get completed date 1226 | line.scan(/\((\d{4}-\d{2}-\d{2}) \d{2}:\d{2}(?:.(?:AM|PM))?\)/) { |m| completed_date = m.join } 1227 | updated_line = line.gsub(/\(#{RE_DATE_TIME}\)/, "(#{completed_date})") 1228 | @lines[n] = updated_line 1229 | cleaned += 1 1230 | @is_updated = true 1231 | # Test if this is one of my special extended repeats (i.e. no / in it) 1232 | if updated_line =~ /@repeat\([^\/]*\)/ 1233 | # get repeat to apply 1234 | date_interval_string = '' 1235 | updated_line.scan(/@repeat\((.*?)\)/) { |mm| date_interval_string = mm.join } 1236 | if date_interval_string[0] == '+' 1237 | # New repeat date = completed date + interval 1238 | date_interval_string = date_interval_string[1..date_interval_string.length] 1239 | new_repeat_date = calc_offset_date(Date.parse(completed_date), date_interval_string) 1240 | log_verbose_message(" Adding from completed date --> #{new_repeat_date}") 1241 | else 1242 | # New repeat date = due date + interval 1243 | # look for the due date (>YYYY-MM-DD) 1244 | due_date = '' 1245 | if updated_line =~ /#{RE_DUE_DATE}/ 1246 | updated_line.scan(/#{RE_DUE_DATE_CAPTURE}/) { |m| due_date = m.join } 1247 | # need to remove the old due date (and preceding whitespace) 1248 | updated_line = updated_line.gsub(/\s*#{RE_DUE_DATE}/, '') 1249 | else 1250 | # but if there is no due date then treat that as today 1251 | due_date = completed_date 1252 | end 1253 | new_repeat_date = calc_offset_date(Date.parse(due_date), date_interval_string) 1254 | log_verbose_message(" Adding from due date --> #{new_repeat_date}") 1255 | end 1256 | 1257 | # Create new repeat line: 1258 | updated_line_without_done = updated_line.chomp 1259 | # Remove the @done text 1260 | updated_line_without_done = updated_line_without_done.gsub(/@done\(.*\)/, '') 1261 | # Replace the * [x] text with * [ ] 1262 | updated_line_without_done = updated_line_without_done.gsub(/\[x\]/, '[ ]') 1263 | # also remove multiple >dates that stack up on repeats 1264 | updated_line_without_done = updated_line_without_done.gsub(/\s+#{RE_DUE_DATE}/, '') 1265 | # finally remove any extra trailling whitespace 1266 | updated_line_without_done.rstrip! 1267 | outline = "#{updated_line_without_done} >#{new_repeat_date}" 1268 | 1269 | # Insert this new line at current line (i.e. before the earlier repeat) 1270 | insert_new_line_at_line(outline, n) 1271 | n += 1 1272 | end 1273 | end 1274 | n += 1 1275 | end 1276 | end 1277 | 1278 | def remove_empty_heading_sections 1279 | # go backwards through the active part of the note, deleting any sections without content 1280 | log_verbose_message(' remove_empty_heading_sections ...') 1281 | cleaned = 0 1282 | n = @done_heading != 0 ? @done_heading - 1 : @line_count - 1 1283 | 1284 | # Go through each line in the file 1285 | later_header_level = this_header_level = 0 1286 | at_eof = 1 1287 | while n.positive? || n.zero? 1288 | line = @lines[n] 1289 | if line =~ /^#+\s\w/ 1290 | # this is a markdown header line; work out what level it is 1291 | line.scan(/^(#+)\s/) { |m| this_header_level = m[0].length } 1292 | log_verbose_message(puts " - #{later_header_level} / #{this_header_level}") 1293 | # if later heading is same or higher level (fewer #s) as this, 1294 | # then we can delete this line 1295 | if later_header_level == this_header_level || at_eof == 1 1296 | log_verbose_message(" - Removing empty heading line #{n} '#{line.chomp}'") 1297 | @lines.delete_at(n) 1298 | cleaned += 1 1299 | @line_count -= 1 1300 | @is_updated = true 1301 | end 1302 | later_header_level = this_header_level 1303 | elsif line !~ /^\s*$/ 1304 | # this has content but is not a header line 1305 | later_header_level = 0 1306 | at_eof = 0 1307 | end 1308 | n -= 1 1309 | end 1310 | return unless cleaned.positive? 1311 | 1312 | @is_updated = true 1313 | # @line_count = @lines.size 1314 | log_verbose_message(" - removed #{cleaned} lines of empty section(s)") 1315 | end 1316 | 1317 | def remove_multiple_empty_lines 1318 | # go backwards through the active parts of the note, deleting any blanks at the end 1319 | log_verbose_message(' remove_multiple_empty_lines ...') 1320 | cleaned = 0 1321 | n = (@done_heading != 0 ? @done_heading - 1 : @line_count - 1) 1322 | last_was_empty = false 1323 | while n.positive? 1324 | line_to_test = @lines[n] 1325 | if line_to_test =~ /^\s*$/ && last_was_empty 1326 | @lines.delete_at(n) 1327 | cleaned += 1 1328 | end 1329 | last_was_empty = line_to_test =~ /^\s*$/ ? true : false 1330 | n -= 1 1331 | end 1332 | return unless cleaned.positive? 1333 | 1334 | @is_updated = true 1335 | @line_count = @lines.size 1336 | log_verbose_message(" - removed #{cleaned} empty lines") 1337 | end 1338 | 1339 | def rewrite_file 1340 | # write out this update file 1341 | main_message(" > writing updated version of " + @filename) 1342 | # open file and write all the lines out 1343 | filepath = if @is_calendar 1344 | "#{NP_CALENDAR_DIR}/#{@filename}" 1345 | else 1346 | "#{NP_NOTES_DIR}/#{@filename}" 1347 | end 1348 | begin 1349 | File.open(filepath, 'w') do |f| 1350 | @lines.each do |line| 1351 | f.puts line 1352 | end 1353 | end 1354 | rescue StandardError => e 1355 | error_message("ERROR: #{e.exception.message} when re-writing note file #{filepath}") 1356 | end 1357 | end 1358 | end 1359 | 1360 | #======================================================================================= 1361 | # Main logic 1362 | #======================================================================================= 1363 | 1364 | # Setup program options 1365 | options = OpenStruct.new 1366 | options.archive = false # default off at the moment as feature isn't complete 1367 | options.move_daily_to_note = false # default off now we have the next option 1368 | options.move_daily_to_note_when_complete = false 1369 | options.move_on_dailies = false 1370 | options.remove_checklist_done_markers = false 1371 | options.remove_rescheduled = true 1372 | options.skipfile = '' 1373 | options.skiptoday = false 1374 | options.quiet = false 1375 | options.verbose = 0 1376 | opt_parser = OptionParser.new do |opts| 1377 | opts.banner = "NotePlan tools v#{VERSION}\nDetails at https://github.com/jgclark/NotePlan-tools/\nUsage: npTools.rb [options] [file-pattern]" 1378 | opts.separator '' 1379 | # Remove option from use until ready 1380 | # opts.on('-a', '--archive', "Archive completed tasks into the ## Done section.") do 1381 | # options.archive = true 1382 | # end 1383 | opts.on('-c', '--changes HOURS', Integer, "How many hours to look back to find note changes to process") do |n| 1384 | hours_to_process = n 1385 | end 1386 | opts.on('-d', '--moveondailies', "Move Daily items with >date to that Daily note") do 1387 | options.move_on_dailies = true 1388 | end 1389 | opts.on('-f', '--skipfile=TITLE[,TITLE2,etc]', Array, "Don't process specific file(s)") do |skipfile| 1390 | options.skipfile = skipfile 1391 | end 1392 | opts.on('-h', '--help', 'Show this help summary') do 1393 | puts opts 1394 | exit 1395 | end 1396 | opts.on('-i', '--skiptoday', "Don't touch today's daily note file") do 1397 | options.skiptoday = true 1398 | end 1399 | opts.on('-m', '--move', "Move Daily items with [[Note#Heading]] reference to that Note", 1400 | "This is triggered whether or not the task is complete.") do 1401 | options.move_daily_to_note = true 1402 | end 1403 | opts.on('-q', '--quiet', 'Suppress all output, apart from error messages. Overrides -v or -w.') do 1404 | options.quiet = true 1405 | end 1406 | opts.on('-r', '--removechecklistdonemarkers', 'Remove @done() markers from checklist items') do 1407 | options.remove_checklist_done_markers = true 1408 | end 1409 | opts.on('-s', '--keepscheduled', 'Keep the re-scheduled (>) dates of completed tasks') do 1410 | options.remove_rescheduled = false 1411 | end 1412 | opts.on('-t', '--movecomplete', "Move Daily items with [[Note#Heading]] reference to that Note on completion") do 1413 | options.move_daily_to_note_when_complete = true 1414 | end 1415 | opts.on('-v', '--verbose', 'Show information as I work') do 1416 | options.verbose = 1 1417 | end 1418 | opts.on('-w', '--moreverbose', 'Show more information as I work') do 1419 | options.verbose = 2 1420 | end 1421 | end 1422 | opt_parser.parse!(ARGV) # parse out options, leaving file patterns to process 1423 | $quiet = options.quiet 1424 | $verbose = $quiet ? 0 : options.verbose # if quiet, then verbose has to be 0 1425 | $archive = options.archive 1426 | $remove_rescheduled = options.remove_rescheduled 1427 | 1428 | #-------------------------------------------------------------------------------------- 1429 | # Start by reading all Notes files in 1430 | # (This is needed to have a list of all note titles that we might be moving tasks to.) 1431 | 1432 | # NOTE: Would like to just work this out on the fly, but there's no way at the moment of 1433 | # looking up note titles from filenames, without reading them all in :-( 1434 | 1435 | begin 1436 | Dir.chdir(NP_NOTES_DIR) 1437 | Dir.glob(File.join('[!@]*/**/*.{md,txt}')).each do |this_file| 1438 | next if File.zero?(this_file) # ignore if this file is empty 1439 | 1440 | $allNotes << NPFile.new(this_file) 1441 | end 1442 | rescue StandardError => e 1443 | error_message("ERROR: #{e.exception.message} when reading in all notes files") 1444 | end 1445 | log_verbose_message("Read in all Note files: #{$npfile_count} found\n") 1446 | 1447 | if ARGV.count.positive? 1448 | # We have a file pattern given, so find that (starting in the notes directory), and use it 1449 | main_message("\nStarting npTools at #{time_now_fmttd} for files matching pattern(s) #{ARGV}.") 1450 | begin 1451 | ARGV.each do |pattern| 1452 | # if pattern has a '.' in it assume it is a full filename ... 1453 | # ... otherwise treat as close to a regex term as possible with Dir.glob 1454 | glob_pattern = pattern =~ /\./ ? pattern : '[!@]**/*' + pattern + '*.{md,txt}' 1455 | log_message(" Looking for note filenames matching glob_pattern #{glob_pattern}:") 1456 | Dir.glob(glob_pattern).each do |this_file| 1457 | log_message(" - #{this_file}") 1458 | next if File.zero?(this_file) # ignore if this file is empty 1459 | 1460 | # Note has already been read in; so now just find which one to point to, by matching filename 1461 | $allNotes.each do |this_note| 1462 | # copy the $allNotes item into $notes array 1463 | if this_file == this_note.filename 1464 | $notes << this_note 1465 | log_verbose_message(" -> found at $allNotes ID #{this_note.id}") 1466 | end 1467 | end 1468 | end 1469 | end 1470 | 1471 | # Now look for matches in Calendar files 1472 | Dir.chdir(NP_CALENDAR_DIR) 1473 | ARGV.each do |pattern| 1474 | # if pattern has a '.' in it assume it is a full filename ... 1475 | # ... otherwise treat as close to a regex term as possible with Dir.glob 1476 | glob_pattern = pattern =~ /\./ ? pattern : '*' + pattern + '*.{md,txt}' 1477 | log_message(" Looking for calendar note filenames matching glob_pattern #{glob_pattern}:") 1478 | Dir.glob(glob_pattern).each do |this_file| 1479 | log_message(" - #{this_file}") 1480 | # read in file unless this file is empty 1481 | next if File.zero?(this_file) 1482 | 1483 | this_note = NPFile.new(this_file) 1484 | $allNotes << this_note 1485 | # copy the $allNotes item into $notes array 1486 | $notes << this_note 1487 | end 1488 | end 1489 | rescue StandardError => e 1490 | error_message("ERROR: #{e.exception.message} when reading in files matching pattern #{pattern}") 1491 | end 1492 | 1493 | else 1494 | # Read metadata for all Note files, and find those altered in the last 24 hours 1495 | main_message("\nStarting npTools at #{time_now_fmttd} for all NP files altered in last #{hours_to_process} hours.") 1496 | begin 1497 | $allNotes.each do |this_note| 1498 | next unless this_note.modified_time > (time_now - hours_to_process * 60 * 60) 1499 | 1500 | # copy this relevant $allNotes item into $notes array to process 1501 | log_verbose_message(" Found relevant project file '#{this_note.filename}'") 1502 | $notes << this_note 1503 | end 1504 | rescue StandardError => e 1505 | error_message("ERROR: #{e.exception.message} when finding recently changed files") 1506 | end 1507 | 1508 | # Also read metadata for all Calendar files, and find those altered in the last 24 hours 1509 | begin 1510 | Dir.chdir(NP_CALENDAR_DIR) 1511 | Dir.glob(['{[!@]**/*,*}.{txt,md}']).each do |this_file| 1512 | # log_verbose_message(" Checking Calendar file #{this_file}, updated #{File.mtime(this_file)}, size #{File.size(this_file)}") 1513 | next if File.zero?(this_file) # ignore if this file is empty 1514 | # if modified time (mtime) in the last 24 hours 1515 | next unless File.mtime(this_file) > (time_now - hours_to_process * 60 * 60) 1516 | 1517 | log_verbose_message(" Found relevant Calendar file #{this_file}, updated #{File.mtime(this_file)}, size #{File.size(this_file)}") 1518 | this_note = NPFile.new(this_file) 1519 | $allNotes << this_note 1520 | # copy the $allNotes item into $notes array 1521 | $notes << this_note 1522 | end 1523 | rescue StandardError => e 1524 | error_message("ERROR: #{e.exception.message} when finding recently changed files") 1525 | end 1526 | end 1527 | 1528 | #-------------------------------------------------------------------------------------- 1529 | if $notes.count.positive? # if we have some files to work on ... 1530 | log_message("Processing #{$notes.count} files:") 1531 | # For each NP file to process, do the following: 1532 | $notes.sort! { |a, b| a.title <=> b.title } 1533 | $notes.each do |note| 1534 | if note.is_today && options.skiptoday 1535 | log_message(" (Skipping #{note.title.to_s.bold} due to --skiptoday option)") 1536 | next 1537 | end 1538 | if options.skipfile.include? note.title 1539 | log_message(" (Skipping#{ note.title.to_s.bol}' due to --skipfile option)") 1540 | next 1541 | end 1542 | log_message(" Processing file id #{note.id}: " + note.title.to_s.bold) 1543 | note.clear_empty_tasks_or_headers 1544 | # note.remove_empty_heading_sections 1545 | note.move_daily_ref_to_notes(options.move_daily_to_note_when_complete) if note.is_calendar && (options.move_daily_to_note || options.move_daily_to_note_when_complete) 1546 | note.remove_finished_tags_dates 1547 | note.remove_checklist_done_markers if options.remove_checklist_done_markers 1548 | note.remove_rescheduled if note.is_calendar 1549 | note.process_repeats_and_done 1550 | note.remove_multiple_empty_lines 1551 | note.move_daily_ref_to_daily if note.is_calendar && options.move_on_dailies 1552 | note.use_template_dates # unless note.is_calendar 1553 | # note.create_events_from_timeblocks 1554 | note.archive_lines if $archive 1555 | # If there have been changes, write out the file 1556 | note.rewrite_file if note.is_updated 1557 | end 1558 | else 1559 | error_message(" Warning: No matching files found.\n") 1560 | end 1561 | -------------------------------------------------------------------------------- /tidyClippings.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | #------------------------------------------------------------------------------- 3 | # Script to tidy and clean up HTML text clipped into markdown files 4 | # by Jonathan Clark, v1.3.x, 9.7.2022 5 | #------------------------------------------------------------------------------- 6 | # TODO: sort out what to do with no H1. 7 | #------------------------------------------------------------------------------- 8 | VERSION = "1.3.10" 9 | require 'date' 10 | require 'cgi' 11 | require 'colorize' 12 | require 'optparse' # more details at https://docs.ruby-lang.org/en/2.1.0/OptionParser.html 13 | require 'ostruct' 14 | 15 | #------------------------------------------------------------------------------- 16 | # Setting variables to tweak 17 | #------------------------------------------------------------------------------- 18 | LIVE = true 19 | 20 | NOTE_EXT = "md" # or "txt" 21 | # FILEPATH = "/Users/jonathan/Dropbox/IFTTT/Clips" # or ... 22 | INPUT_FILEPATH = "/Users/jonathan/Library/Mobile Documents/iCloud~is~workflow~my~workflows/Documents" 23 | # ARCHIVE_FILEPATH = "/Users/jonathan/Dropbox/IFTTT/Archive" # or ... 24 | ARCHIVE_FILEPATH = "#{INPUT_FILEPATH}/tidyClippingsOriginals" 25 | DATE_ISO_FORMAT = '%Y-%m-%d'.freeze 26 | DATE_TIME_HUMAN_FORMAT = '%e %b %Y %H:%M'.freeze 27 | DATE_TIME_LOG_FORMAT = '%Y%m%d%H%M'.freeze # only used in logging 28 | IGNORE_SECTION_TITLES = ['IgnoreMe', 'Resources', 'New Resources', 'Menu', 'Primary Menu', 'Archive', 'Meta', 'Subscribe', 'Post navigation', 'Shared', 'Share this', 'Share this post', 'How we use cookies', 'Skip to content', 'Like this', 'Leave a Reply', '_Related', 'Related Posts', 'Related Articles', 'Related Resources', 'More by this author', 'Ways to Follow', 'My recent publications', 'Other Publications', 'Publications', 'Follwers', 'Site Map', 'Solid Joys', 'Look at the Book', 'Join The Conversation', 'About Us', 'Follow Us', 'Events', 'Ministries', 'Blog Archive', 'Blogroll', 'Think', 'More like this', 'From Across The Blog', 'Author info', 'Authors', 'Join the Discussion', 'Popular Articles in This Series'] 29 | 30 | #------------------------------------------------------------------------------- 31 | # Other Constants 32 | #------------------------------------------------------------------------------- 33 | USERNAME = ENV['LOGNAME'] # pull username from environment 34 | USER_DIR = ENV['HOME'] # pull home directory from environment 35 | # NP_DROPBOX_DIR = "#{USER_DIR}/Dropbox/Apps/NotePlan/Documents".freeze 36 | # NP_ICLOUDDRIVE_DIR = "#{USER_DIR}/Library/Mobile Documents/iCloud~co~noteplan~NotePlan/Documents".freeze 37 | NP_CLOUDKIT_DIR = "#{USER_DIR}/Library/Containers/co.noteplan.NotePlan3/Data/Library/Application Support/co.noteplan.NotePlan3".freeze 38 | # TodaysDate = Date.today # can't work out why this needs to be a 'constant' to work -- something about visibility, I suppose 39 | # Note: simplifying next line, as it seems to take an *age* to run 40 | NP_BASE_DIR = NP_CLOUDKIT_DIR if Dir.exist?(NP_CLOUDKIT_DIR) # && Dir[File.join(NP_CLOUDKIT_DIR, '**', '*')].count { |file| File.file?(file) } > 1 41 | NP_CALENDAR_DIR = "#{NP_BASE_DIR}/Calendar".freeze 42 | 43 | # Colours to use with the colorization gem 44 | # to show some possible combinations, run String.color_samples 45 | # to show list of possible modes, run puts String.modes (e.g. underline, bold, blink) 46 | String.disable_colorization false 47 | CompletedColour = :light_green 48 | InfoColour = :yellow 49 | ErrorColour = :light_red 50 | # Test to see if we're running interactively or in a batch mode: 51 | # if batch mode then disable colorisation which doesn't work in logs 52 | tty_code = `tty`.chomp 53 | String.disable_colorization true if tty_code == 'not a tty' 54 | 55 | # Variables that need to be globally available 56 | time_now = Time.now 57 | $date_time_now_human_fmttd = time_now.strftime(DATE_TIME_HUMAN_FORMAT) 58 | $date_time_now_log_fmttd = time_now.strftime(DATE_TIME_LOG_FORMAT) 59 | $date_now = time_now.strftime(DATE_ISO_FORMAT) 60 | $verbose = false 61 | $npfile_count = 0 62 | 63 | #------------------------------------------------------------------------- 64 | # Helper Functions 65 | #------------------------------------------------------------------------- 66 | def main_message(message) 67 | puts message.colorize(CompletedColour) 68 | end 69 | 70 | def info_message(message) 71 | puts message.colorize(InfoColour) 72 | end 73 | 74 | def error_message(message) 75 | puts message.colorize(ErrorColour) 76 | end 77 | 78 | def log_message(message) 79 | puts message if $verbose 80 | end 81 | 82 | def truncate_text(text, max_length = 100000, use_elipsis = false) 83 | raise ArgumentError, "max_length must be positive" unless max_length.positive? 84 | return '' if text.nil? 85 | 86 | return text if text.size <= max_length 87 | 88 | return text[0, max_length] + (use_elipsis ? '...' : '') 89 | end 90 | 91 | #------------------------------------------------------------------------- 92 | 93 | # simplify HTML and Markdown in the line we receive 94 | def cleanup_line(line) 95 | orig_line = line 96 | 97 | # replace lines with '***' or '* * *' or similar with '---' 98 | line = '---' if line =~ /\*\s*\*\s*\*/ 99 | 100 | # replace HTML entity elements with ASCII equivalents 101 | line.gsub!(/&/, '&') 102 | line.gsub!(/ /, ' ') 103 | line.gsub!(/ _place_holder;/, ' ') 104 | line.gsub!(/—/, '--') 105 | line.gsub!(/&lsquot;/, "\'") 106 | line.gsub!(/&ldquot;/, "\"") 107 | line.gsub!(/&rsquot;/, "\'") 108 | line.gsub!(/&rdquot;/, "\"") 109 | line.gsub!(/"/, "\"") 110 | line.gsub!(/</, "<") 111 | line.gsub!(/>/, ">") 112 | line.gsub!(/…/, "...") 113 | line.gsub!(/ /, " ") 114 | line.gsub!(/%20/, " ") 115 | 116 | # replace smart quotes with dumb ones 117 | line.gsub!(/“/, '"') 118 | line.gsub!(/”/, '"') 119 | line.gsub!(/‘/, '\'') 120 | line.gsub!(/’/, '\'') 121 | line.gsub!(/'/, '\'') 122 | line.gsub!(/’/, '\'') 123 | # replace en dash with markdwon equivalent 124 | line.gsub!(/—/, '--') 125 | 126 | # replace '\.' with '.' 127 | line.gsub!(/\\\./, '.') 128 | 129 | # replace opening '* ' with '- ' 130 | line.gsub!(/^(\s*)\*\s/, '\1- ') 131 | 132 | # replace '## [' with '[' (Desiring God) 133 | line.gsub!(/## \[/, '[') 134 | 135 | # drop base64 image lines 136 | line = '' if line =~ /!\[\]\(data:image\/gif;base64/ 137 | 138 | # Remove some lines which aren't interesting 139 | line = '' if line =~ /^Previous article/i || line =~ /^Next article/i 140 | line = '' if line =~ /^\[Share\]/i 141 | line = '' if line =~ /^\##\s*\[View All\]/i # for Crossway 142 | line = '' if line =~ /^\\-\s*$/ 143 | line = '' if line =~ /^\*\*Comments policy:\*\*/ 144 | line = '' if line == "#popclipped" 145 | line = '' if line == "#clipped" 146 | line = '' if line == "[Donate](/donate)" 147 | line = '' if line == "Submit" 148 | line = '' if line == "[News & Updates](/posts)" 149 | line = '' if line == "Advanced Search" 150 | 151 | # for Think Theology 152 | line = '' if line == "# Think" # Not working, and I don't know why. Instead knocking out all headings which start 'Think' 153 | line = '' if line == "Date from:" 154 | line = '' if line == "Date to:" 155 | line = '' if line =~ /\[\s*Prev article\s*\]/ 156 | line = '' if line =~ /\[\s*Next article\s*\]/ 157 | 158 | # for Stocki 159 | # replace odd things in Stocki '**_ ... _**' with simpler '_..._' 160 | line.gsub!(/\*\*_/, '_') 161 | line.gsub!(/_\*\*/, '_') 162 | 163 | # replace a line just surrounded by **...** with an H4 instead 164 | # (needs to come after **_..._** test above) 165 | line.gsub!(/^\s*\*\*(.*)\*\*\s*$/, '#### \1') 166 | 167 | # replace asterisk lists with dash lists (to stop NP thinking they are tasks) 168 | line.gsub!(/^(\s*)\*\s/, '\1- ') 169 | 170 | # replace line starting ' - ####' with just heading markers 171 | line.gsub!(/^\s*\-\s+####\s+/, '#### ') 172 | 173 | # trim the end of the line 174 | line.rstrip! 175 | 176 | # write out if changed 177 | log_message(" cl-> #{line}") if orig_line != line 178 | 179 | return line 180 | end 181 | 182 | def help_identify_sections(line) 183 | orig_line = line 184 | # Fix headings that are missing heading markers 185 | line = '# Menu' if line =~ /^#Menu/i 186 | line = '## Join the Conversation' if line =~ /^Join the Conversation$/i 187 | line = '## About Us' if line =~ /^# ABOUT US/i 188 | line = '## Follow Us' if line =~ /^FOLLOW US$/i 189 | 190 | # Change some heading levels 191 | line = '### Labels' if line =~ /^## Labels$/i 192 | 193 | # Ignore sections without heading text 194 | # v1.3.5 added this, but stops heidelblog.net working 195 | # line = '## IgnoreMe' if line =~ /^#+\s*$/ 196 | # v1.3.10 trying simplication 197 | line = '' if line =~ /^#+\s*$/ 198 | 199 | # log if changed 200 | log_message(" his-> #{line}") if orig_line != line 201 | 202 | return line 203 | end 204 | 205 | def help_identify_metadata(line) 206 | orig_line = line 207 | 208 | # wordpress possible way of detecting source URL 209 | line.gsub!(/^\[Leave a comment\]\((.*?)\/#respond\)/, 'poss_source: \1') 210 | 211 | # wordpress possible way of detecting author 212 | line = 'poss_author: ' + line if line =~ /^\[.*\]\(.*\/author\/.+\)/ 213 | 214 | # wordpress possible way of detecting publish date 215 | line = 'poss_date: ' + line if line =~ /\[.*\]\(.*\/\d{4}\/\d{2}\/\d{2}\/\)/ 216 | 217 | # DesiringGod.org specifics 218 | # line = "site: https://www.desiringgod.org/" if line =~ /\/about-us/ 219 | 220 | # typepad.com specifics 221 | line = 'source: ' + line if line =~ /^https:\/\/.*\.typepad.com\/.*/ 222 | 223 | # blogspot.com specifics 224 | line.gsub!('# ', 'site: ') if line =~ /^#\s.*\(https:\/\/.*\.blogspot\.com\// # first H1 is site title not post title 225 | 226 | # Psephizo specifics 227 | line = "author: Ian Paul" if line == "scholarship. serving. ministry." # not always correct, but typically not given if there isn't a guest author 228 | # line = "site: www.psephizo.com" if line == "[ Psephizo ](https://www.psephizo.com/)" # covered by later [Home](...) 229 | line = '## ' + line if line =~ /^Categories .*https:\/\/www.psephizo.com\// 230 | # TODO: decide if it's OK to change some comment lines in Psephizo that are [...](https://www.psephizo.com/life-ministry/why-does-embracing-justice-matter/...) -- ie source 231 | 232 | # log if changed 233 | log_message(" him-> #{line}") if orig_line != line 234 | 235 | return line 236 | end 237 | 238 | #=========================================================================== 239 | # Main logic 240 | #=========================================================================== 241 | 242 | # Setup program options 243 | options = OpenStruct.new 244 | opt_parser = OptionParser.new do |opts| 245 | opts.banner = "Tidy web clippings v#{VERSION}\nUsage: tidyClippings.rb [options] [file-pattern]" 246 | opts.separator '' 247 | opts.on('-h', '--help', 'Show this help') do 248 | puts opts 249 | exit 250 | end 251 | options[:verbose] = false 252 | opts.on('-v', '--verbose', 'Show information as I work') do 253 | options[:verbose] = true 254 | end 255 | end 256 | opt_parser.parse!(ARGV) # parse out options, leaving file patterns to process 257 | $verbose = options.verbose 258 | 259 | #------------------------------------------------------------------------- 260 | # Read .txt files in the directory 261 | #------------------------------------------------------------------------- 262 | begin 263 | Dir.chdir(INPUT_FILEPATH) 264 | glob_pattern = ARGV.count.positive? ? '*' + ARGV[0] + '*.txt' : '*.txt' 265 | main_message("Starting to tidy web clippings for #{glob_pattern} at #{$date_time_now_human_fmttd}.") 266 | Dir.glob(glob_pattern).each do |this_file| 267 | main_message("- file '#{this_file}'") 268 | 269 | # initialise other variables (that don't need to persist with the class) 270 | lines = [] 271 | author = nil 272 | poss_author = nil 273 | doc_date = nil 274 | clip_date = File.birthtime(this_file) # = creation date (when it arrives in my filesystem) 275 | poss_date = nil 276 | site = nil 277 | tags = [] 278 | title = nil 279 | source = nil 280 | poss_source = nil 281 | 282 | # Open file and read in all lines -- the first pass 283 | # NB: needs the encoding line when run from launchctl, otherwise you get US-ASCII invalid byte errors (basically the 'locale' settings are different) 284 | n = 0 285 | f = File.open(this_file, 'r', encoding: 'utf-8') 286 | f.each_line do |line| 287 | line_in = line.clone.rstrip # needs a proper clone, not just a reference 288 | # log_message(" #{n}: #{line_in}") 289 | lines[n] = line 290 | 291 | # Fix all sorts of things in the line 292 | line = cleanup_line(line) 293 | # Tweak lines to standardise section headings etc. 294 | line = help_identify_sections(line) 295 | # See what we can do to help identify metadata, and change accordingly 296 | line = help_identify_metadata(line) 297 | 298 | # For first line only ignore H1 which is just a blog title 299 | if n == 0 && line =~ /^#\s+\[.+\]\(.+\)/ 300 | line.gsub!(/^#/, 'site:') 301 | end 302 | if line != line_in 303 | log_message(" #{n}~ #{line}") 304 | lines[n] = line 305 | end 306 | 307 | n += 1 308 | end 309 | f.close 310 | info_message(" After first pass, #{n} lines") 311 | 312 | last_line = "" 313 | ignore_before = 0 314 | ignore_after = 99999 315 | ignore_this_section = false 316 | last_heading_level = 6 # start low 317 | this_heading_level = 6 # start low 318 | re_ignore_sections = "^#+\\s+(#{IGNORE_SECTION_TITLES.join('|')})" # lines that start with MD headings then any of those sections 319 | n = -1 # line number in lines array 320 | max_lines = lines.size # need to track this; for some reason testing for lines.size in the while loop doesn't work 321 | 322 | # Go through lines again 323 | while n < max_lines 324 | n += 1 325 | line = lines[n] 326 | # first check for H1 lines (that shouldn't be ignored) 327 | if line =~ /^#\s/ && line !~ /#{re_ignore_sections}/i 328 | # this is the first H1 so use this as a title and starting line 329 | if ignore_before.zero? 330 | ignore_before = n 331 | line.scan(/^#\s+(.*)/) { |m| title = m.join } 332 | info_message(" #{n}: found title (from H1): '#{title}'") 333 | else 334 | # this is a subsequent H1 then probably ignore after this 335 | ignore_after = n - 1 336 | this_heading = '' # needed to access variable set in following loop 337 | line.scan(/^#\s+(.*)/) { |m| this_heading = m.join } 338 | info_message(" #{n}: found subsequent H1: ignore after this '#{this_heading}'") 339 | end 340 | end 341 | 342 | # If a new section, should we ignore it? 343 | if line =~ /^#+\s+/ 344 | this_heading_level = line.index(' ') # = how many chars before first space? 345 | if line =~ /#{re_ignore_sections}/i 346 | log_message(" #{n}: will ignore new section '#{line}'") 347 | ignore_this_section = true 348 | elsif this_heading_level <= last_heading_level 349 | log_message(" #{n}: new section '#{line}' #{this_heading_level} (<= #{last_heading_level}) stopping ignore") 350 | ignore_this_section = false 351 | last_heading_level = this_heading_level 352 | else 353 | log_message(" #{n}: new section '#{line}' #{this_heading_level} (> #{last_heading_level}) : ignore_this_section = #{ignore_this_section}") 354 | # last_heading_level = this_heading_level 355 | end 356 | end 357 | 358 | # if this is the Comments section then ignore after this (unless we already have found an ignore point) 359 | if line =~ /^#+\s+(No comments|Comments|\d*\s*Comments on\s|\d*\s*thoughts on\s)/i && ignore_after == 99999 360 | ignore_after = n - 1 361 | ignore_this_section = true 362 | info_message("#{n}: found Comments section: will ignore after line #{n}") 363 | end 364 | 365 | if ignore_this_section || n > ignore_after # TODO: test me 366 | # We're in an ignore section, so blank this line 367 | # log_message(" #{n}/#{lines.size}: ignored") if n < 250 368 | last_line = line 369 | lines[n] = "" 370 | line = "" 371 | 372 | else 373 | # log_message(" #{n}: #{truncate_text(line, 70, true)}") if n < 100 374 | 375 | # insert blank line before heading 376 | # if !last_line.empty? && line =~ /^#+\s+/ 377 | # log_message(" #{n} inserted: empty line before heading '#{line}'") 378 | # lines.insert(n, "") 379 | # last_line = "" 380 | # n -= 1 381 | # redo # i.e. redo this particular line, now that we've added the blank line 382 | # end 383 | 384 | # remove blank line after heading 385 | # if last_line =~ /^#+\s+/ && line.empty? 386 | # log_message(" #{n} removed: empty line after heading '#{last_line}'") 387 | # last_line = line 388 | # line = "" 389 | # lines.delete_at(n) 390 | # max_lines -= 1 391 | # n += 1 392 | # next 393 | # end 394 | 395 | #----------------------------------------------------------- 396 | # save out any other metadata we can spot 397 | # [...](../authors?/..) 398 | if line =~ /\[[^\]]*?\]\([^)]*?\/authors?\/.*?\)/i 399 | line.scan(/\[([^\]]*?)\]\([^)]*?\/authors?\/.*?\)/i) { |m| poss_author = m.join } 400 | info_message(" #{n}: found poss_author/1: #{poss_author}") 401 | end 402 | # 'by|© X Y' but not 2022 (or too long, as a random line starting 'By ...') 403 | if line =~ /^[\s*-]*(?:by|©):?\s*[^\d]{4}.*/i && line.size < 40 404 | line.scan(/^[\s*-]*(?:by|©):?\s*([^\d]{4}.*)/i) { |m| author = m.join } # .join turns ['a'] to 'a' 405 | info_message(" #{n}: found author/1: #{author}") 406 | end 407 | # 'by [X Y](...)' 408 | if line =~ /^\s*by[:\s]+\[.+\]/i 409 | line.scan(/^\s*by[:\s]+\[(.+)\]/i) { |m| author = m.join } # .join turns ['a'] to 'a' 410 | info_message(" #{n}: found author/2: #{author}") 411 | line.scan(/^\s*by[:\s]+\[.+\]\((https:\/\/[^\/]+\/)/i) { |m| site = m.join } 412 | info_message(" #{n}: found site/1: #{site}") 413 | end 414 | # ... blogger.com/profile ... at ... 415 | if line =~ /https:\/\/www\.blogger\.com\/profile\/.*\s*at\s*.*?https:\/\/[^\)\s$]+/ 416 | line.scan(/\[([^\]]+)\].*\s*at\s*.*?(https:\/\/[^\)\s$]+)/) do |m| 417 | poss_author = m[0] 418 | source = m[1] 419 | end 420 | info_message(" #{n}: found poss_author/2: #{poss_author}") 421 | info_message(" #{n}: found source/1: #{source}") 422 | end 423 | # Posted by ... at ... 424 | if line =~ /Posted\sby\s*.*\s*at\s*.*?(https:\/\/[^\)\s$]+)/ # 425 | line.scan(/Posted\sby\s*(.*)\s*at\s*.*?(https:\/\/[^\)\s$]+)/) do |m| 426 | poss_author = m[0] 427 | source = m[1] 428 | end 429 | info_message(" #{n}: found poss_author/3: #{poss_author}") 430 | info_message(" #{n}: found source/2: #{source}") 431 | end 432 | # /category/...) TODO: align with /label/ and /tag/ below? 433 | if line =~ /\/category\/.*?\//i 434 | line.scan(/\/category\/([^\/]+)\//i) { |m| tags << m.join } 435 | info_message(" #{n}: found tags (from category): #{tags}") 436 | end 437 | # /label/...) 438 | if line =~ /\/label\/[^\)\/]+[\)\/]/i 439 | line.scan(/\/label\/([^\)\/]+)[\)\/]/i) { |m| tags << m.join } 440 | info_message(" #{n}: found tags (from label): #{tags}") 441 | end 442 | # /tag/...) 443 | if line =~ /\/tag\/[^\)\/]+[\)\/]/i 444 | line.scan(/\/tag\/([^\)\/]+)[\)\/]/i) { |m| tags << m.join } 445 | info_message(" #{n}: found tags: #{tags}") 446 | end 447 | if line =~ /https:\/\/[^\/]+\/wp-content\// 448 | line.scan(/(https:\/\/[^\/]+)\/wp-content\//) { |m| site = m.join } 449 | info_message(" #{n}: found site/2: #{site}") 450 | end 451 | # [Home](...) 452 | if line =~ /[^!]\[Home\]\(.*\)/i # TODO: test me 453 | line.scan(/\[Home\]\((.*)\)/i) { |m| site = m.join} 454 | info_message(" #{n}: found site/3: #{site}") 455 | end 456 | # [View web version](...) 457 | if line =~ /^\[View web version\]\(.+\)/i 458 | line.scan(/^\[View web version\]\((.+)\)/i) { |m| source = m.join } 459 | info_message(" #{n}: found source/3: #{source}") 460 | end 461 | # https://www.facebook.com/sharer/sharer.php?u=... 462 | if line =~ /https:\/\/www\.facebook\.com\/sharer\/sharer\.php\?u=.+\)/i 463 | line.scan(/https:\/\/www\.facebook\.com\/sharer\/sharer\.php\?u=(.+)\)/i) { |m| source = m.join } 464 | info_message(" #{n}: found source/4: #{source}") 465 | end 466 | # [Leave a comment](.../#respond) 467 | if line =~ /\[Leave a Comment\]\([^)]+?\/#respond\)/i 468 | line.scan(/\[Leave a Comment\]\(([^)]+)\/#respond\)/i) { |m| source = m.join } 469 | info_message(" #{n}: found source/5: #{source}") 470 | end 471 | 472 | # Stocki specific 473 | author = 'Steve Stockman' if line =~ /^source: https:\/\/stocki.typepad.com\//i 474 | 475 | 476 | #----------------------------------------------------------- 477 | # save any already fielded metadata 478 | # (after the previous set to give higher priority to its results) 479 | if line =~ /^\s*title[:\s]+.+/i # TODO: finish me 480 | line.scan(/^\s*title[:\s]+(.*)/i) { |m| title = m.join } 481 | info_message(" #{n}: found title: #{title}") 482 | end 483 | if line =~ /^\s*date[:\s]+.+/i 484 | line.scan(/\s*date[:\s]+(.*)/i) { |m| doc_date = m.join } 485 | info_message(" #{n}: found date: #{doc_date}") 486 | end 487 | if line =~ /^\s*poss_date[:\s]+.+/i 488 | line.scan(/\s*poss_date[:\s]+(.*)/) { |m| poss_date = m.join } 489 | info_message(" #{n}: found poss_date: #{poss_date}") 490 | end 491 | if line =~ /^\s*(author|by):\s*.+/i 492 | line.scan(/^\s*(?:author|by):\s*(.+)/i) { |m| author = m.join } 493 | info_message(" #{n}: found author: #{author}") 494 | end 495 | if line =~ /^\s*poss_author[:\s]+.+/i 496 | line.scan(/^\s*poss_author[:\s]+(.*)/i) { |m| poss_author = m.join } 497 | info_message(" #{n}: found poss_author: #{poss_author}") 498 | end 499 | if line =~ /^\s*tags?[:\s]+.+/i # tag: field etc. 500 | line.scan(/^Tags?[:\s]+\[?([^\]]+)/i) { |m| tags << m.join } # add to array, having first turned ['a'] to 'a' 501 | info_message(" #{n}: found tags: #{tags}") 502 | end 503 | if line =~ /^\s*(category|categories)[:\s]+/i # category: field etc. 504 | line.scan(/^\s*(?:category|categories)[:\s]+\[?([^\]]+)/i) { |m| tags << m.join } # ?: stops first (...) being a capturing group 505 | info_message(" #{n}: found tags (from category field): #{tags}") 506 | end 507 | if line =~ /^site:\s+.+/i # TODO: test me 508 | line.scan(/^site:\s+(.*)/i) { |m| site = m.join} 509 | info_message(" #{n}: found site: #{site}") 510 | end 511 | if line =~ /^source:\s+.+/i 512 | line.scan(/^source:\s+(.*)/i) { |m| source = m.join } 513 | info_message(" #{n}: found source: #{source}") 514 | end 515 | if line =~ /^poss_source:\s+.+/i 516 | line.scan(/^poss_source:\s+(.*)/i) { |m| poss_source = m.join } 517 | info_message(" #{n}: found poss_source: #{poss_source}") 518 | end 519 | 520 | #----------------------------------------------------------- 521 | # try just parsing line for a valid date string 522 | begin 523 | # look for at least "...4.3.22..." or "4 Mar 22" 524 | # or one of the month names 525 | if !line.nil? && (line.count('0-9') >= 3 || line =~ /(^|\s)(Jan(uary)?|Feb(uary)?|Mar(ch)?|Apr(il)?|May|Jun(e)?|Jul(y)?|Aug(ust)?|Sep(tember)?|Oct(ober)?|Nov(ember)?|Dec(ember)?)\s/) 526 | trunc_text = truncate_text(line, 122, false) # function's limit is 128, but it seems to need fewer than that 527 | line_date = Date.parse(trunc_text).to_s 528 | # only keep if it's before or equal to today 529 | if line_date < $date_now 530 | poss_date = line_date 531 | info_message(" #{n}: found poss date: #{poss_date} from '#{truncate_text(line, 30)}'") 532 | end 533 | end 534 | rescue Date::Error => e 535 | # log_message("didn't like that: #{e}") 536 | end 537 | end # if in ignore_section 538 | 539 | last_line = line 540 | end # for each line 541 | info_message(" After second pass, #{lines.size} lines left") 542 | log_message(" Will ignore before l.#{ignore_before} & after l.#{ignore_after}") 543 | 544 | #------------------------------------------------------------------------- 545 | # Form the frontmatter section 546 | #------------------------------------------------------------------------- 547 | fm_title = title || "" 548 | fm_author = author || poss_author || "" 549 | fm_clip_date = clip_date || $date_time_now_log_fmttd # fallback to current date 550 | fm_doc_date = doc_date || poss_date || "?" 551 | fm_tags = tags.join(', ') || "" 552 | fm_source = source || poss_source || site || "" 553 | fm_generated = "#{$date_time_now_log_fmttd} by tidyClippings v#{VERSION}" 554 | frontmatter = "---\ntitle: #{fm_title}\nauthor: #{fm_author}\ndate: #{fm_doc_date}\nclipped: #{fm_clip_date}\ntags: [#{fm_tags}]\nsource: #{fm_source}\ngenerated: #{fm_generated}\n---\n\n" 555 | 556 | info_message(frontmatter) 557 | 558 | #------------------------------------------------------------------------- 559 | # write out this updated file, as a markdown file, with frontmatter prepended 560 | #------------------------------------------------------------------------- 561 | Dir.chdir('/tmp') unless LIVE 562 | 563 | # first simplify filename itself 564 | new_filename = "#{cleanup_line(this_file[0..-5]).lstrip}.#{NOTE_EXT}" # take off .txt and put on .md 565 | 566 | # open file and write all the lines out, 567 | # though ignoring any before the 'ignore_before' line, and after the 'ignore_after' line 568 | # also only write out 1 empty line in a row 569 | # file mode 'w' = write-only, truncates existing file 570 | last_fl = "" 571 | File.open(new_filename, 'w') do |ff| 572 | ff.puts frontmatter 573 | n = 0 574 | lines.each do |fl| 575 | ff.puts fl if n >= ignore_before && n <= ignore_after && !(last_fl.empty? && fl.empty?) 576 | n += 1 577 | last_fl = fl 578 | end 579 | end 580 | main_message(" -> written updated version to '#{new_filename}'") 581 | 582 | # Now rename file to same as above but _YYYYMMDDHHMM on the end 583 | archive_filename = "#{ARCHIVE_FILEPATH}/#{this_file}" 584 | File.rename(this_file, archive_filename) if LIVE 585 | 586 | break # TODO: remove me 587 | end # second pass 588 | rescue SystemCallError => e 589 | error_message("ERROR: on rename? #{e.exception.full_message}") 590 | rescue StandardError => e 591 | error_message("ERROR: #{e.exception.full_message}") 592 | end 593 | --------------------------------------------------------------------------------