├── .github └── FUNDING.yml ├── .gitignore ├── .scalafmt.conf ├── .travis.yml ├── LICENSE ├── README.md ├── build.sbt ├── highlights.png ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt ├── scripts ├── release.sh └── windows.bat └── src ├── main └── scala │ └── notionfys │ ├── App.scala │ ├── Cli.scala │ ├── Console.scala │ ├── FS.scala │ ├── Highlights.scala │ ├── Main.scala │ ├── Notion.scala │ └── Program.scala └── test ├── resources ├── My Clippings.txt ├── issue_3_clippings.txt └── win.txt └── scala └── notionfys └── HighlightsSpec.scala /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [yannick-cw] 4 | custom: ['paypal.me/yagla'] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # sbt 2 | dist/* 3 | target/ 4 | lib_managed/ 5 | src_managed/ 6 | project/boot/ 7 | project/plugins/project/ 8 | .history 9 | .cache 10 | .lib/ 11 | documents/ 12 | 13 | # Java 14 | *.class 15 | 16 | # Logs 17 | *.log 18 | log/ 19 | logs/ 20 | 21 | # Temporary files 22 | *~ 23 | *.bak 24 | *.old 25 | *.tmp 26 | nohup.out 27 | tmp/ 28 | 29 | # vim 30 | .*.sw[a-z] 31 | 32 | # IntelliJ 33 | .idea/ 34 | .idea_modules/ 35 | *.iml 36 | *.ipr 37 | *.iws 38 | 39 | # Eclipse 40 | .cache 41 | .classpath 42 | .project 43 | .scala_dependencies 44 | .settings/ 45 | *.sc 46 | 47 | # Mac OS X 48 | ._* 49 | .DS_Store 50 | 51 | # Windows 52 | Desktop.ini 53 | Thumbs.db 54 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = "2.2.1" 2 | 3 | maxColumn = 100 4 | align = more 5 | rewrite.rules = [RedundantBraces, RedundantParens, SortModifiers, PreferCurlyFors] 6 | rewrite.redundantBraces.stringInterpolation = true 7 | binPack.literalArgumentLists = true 8 | includeCurlyBraceInSelectChains = true 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.13.1 5 | 6 | jdk: 7 | - openjdk11 8 | 9 | jobs: 10 | include: 11 | - os: linux 12 | - os: osx 13 | - os: windows 14 | language: shell 15 | env: NATIVE_IMAGE_PATH="C:/Program Files/GraalVM/graalvm-ce-java11-21.0.0.2/bin/native-image.cmd" 16 | 17 | 18 | before_install: 19 | - |- 20 | case $TRAVIS_OS_NAME in 21 | windows) 22 | choco install graalvm sbt 23 | eval $(powershell -NonInteractive -Command 'write("export PATH=`"" + ([Environment]::GetEnvironmentVariable("PATH","Machine") + ";" + [Environment]::GetEnvironmentVariable("PATH","User")).replace("\","/").replace("C:","/c").replace(";",":") + ":`$PATH`"")') 24 | ;; 25 | esac 26 | 27 | install: 28 | - |- 29 | case $TRAVIS_OS_NAME in 30 | windows) 31 | gu.cmd install native-image 32 | ;; 33 | esac 34 | 35 | script: 36 | - |- 37 | case $TRAVIS_OS_NAME in 38 | windows) 39 | sbt test 40 | ./scripts/windows.bat 41 | ;; 42 | *) 43 | sbt test 44 | sbt "nativeImage" 45 | ;; 46 | esac 47 | 48 | before_deploy: 49 | - ./scripts/release.sh 50 | 51 | deploy: 52 | provider: releases 53 | skip_cleanup: true 54 | api_key: 55 | secure: hh2IBMCwL3F6Ku3ThlahCX72SiHMlobIwH6uqfbMZzBCgOyDwFUsad8L4ktCqZABY5sUzfzIx3Pe6aGy4owfjqVm10nODOU4HA6kcCJrnHPkonySh6wASqjOiw9rHOty6p2Sn7Urqx1Ox/CHGgKMGKCcxBujwcN/4uPu1vCeTCEE1V5rFLaLmhT8bHFV2JjCpEEkRQmoPgTfJwJsNKnblw4A0z3weD7BNuiBXmQLBkdP1fUna+a1++WkWEIAybjqFEtVinfuJVQWCaL2MAfS5qCefrga7/nmP0AAKJPVc65YpGC1osTibm00vLnqE7wgXZT38xVjIdwzlnGBjnsoIJGiVLBAPDlg+kUcfuXQoi94JQQ9/0A0vADhra1WQeaf7a2m/bhpmHvq6VnY2X3nLqOZVkylkiE019y0xZ+qQc7uOT8b+zX3zWMrqrUJ/d/u2a2Iotd81WCdVuSY1fVX7GT+rqfDshwWPvSu9FNcrMSC11Nv9hIB6i8BfPKJo8l0SIJeEgyaXnNaznf+vO1KNJYYBijKEZmeVdHtbvUOH7ze60KnlgVTF/wmcE2JVJTtpRBQL1SDH2KkF8I+L1CfwdhylsNRoW93sMJBLFJG2lHEfZlAd4WV7ekKnuigHYI+pd5tDE5a+A1ASnm1vwR096Bz//1crtBJCjuawH/hikQ= 56 | file: release/* 57 | file_glob: true 58 | on: 59 | tags: true 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Author name here (c) 2019 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following 13 | disclaimer in the documentation and/or other materials provided 14 | with the distribution. 15 | 16 | * Neither the name of Author name here nor the names of other 17 | contributors may be used to endorse or promote products derived 18 | from this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # notionfy: 📓 to 🗒 -> ♥ 2 | 3 | ### Sync your Kindle highlights to [Notion](https://www.notion.so/) 4 | 5 | This script reads the `clippings.txt` from your kindle and syncs all the highlight to a selected notion page. 6 | When rerunning it only appends new clippings. 7 | 8 | ### Install 9 | 10 | ### Linux 11 | 12 | Download the [notionfy](https://github.com/yannick-cw/notionfy/releases/tag/0.3.x) linux zip, unzip it and place notionfy in your `PATH` or run it directly with `./notionfy`. Don't forget to make it executable with `chmod +x notionfy`. 13 | 14 | 15 | #### Mac 16 | On Mac you can use homebrew 17 | ``` 18 | brew install yannick-cw/homebrew-tap/notionfy 19 | ``` 20 | Alternatively: 21 | 22 | Download the [notionfy](https://github.com/yannick-cw/notionfy/releases/tag/0.3.x) mac zip, unzip it and place notionfy in your `PATH` or run it directly with `./notionfy`. Don't forget to make it executable with `chmod +x notionfy`. 23 | 24 | On Mac you may also give it permission to run in `System Preferences -> Security & Privacy` 25 | 26 | #### Windows 27 | 28 | 1. Download the zip for the latest windows [release](https://github.com/yannick-cw/notionfy/releases/download/0.3.x/notionfy.exe) file to .e.g `Downloads` 29 | 2. Open power shell (or any shell) 30 | 4. Change Directory to the exe's path, e.g.: `cd .\Downloads` 31 | 5. `.\notionfy.exe` 32 | Should give you the outcome 33 | ``` 34 | Usage: ..... 35 | ``` 36 | That means it works so far. 37 | Now run it with your configuration: 38 | 39 | ``` 40 | .\notionfy.exe --token "TOKEN_HERE" --page "PAGE_ID_HERE" --kindle "D:/" 41 | ``` 42 | 43 | Where token is token form the cookie and page id from the url of the page you want to add the snippets. When I connect my kindle to a windows machine it is mounted as `D:/` so check under what path you kindle is mounted and add that instead of `D:/` 44 | 45 | ### Setup 46 | 47 | 1. Get the `token_v2` token from https://www.notion.so/ 48 | 49 | - when using chrome [here](https://developers.google.com/web/tools/chrome-devtools/storage/cookies) is some info on how to read a cookie 50 | 51 | 2. Create a new, empty page and copy the id 52 | 53 | - e.g. `https://www.notion.so/Kindle-Highlights-5129b8f88a414b8e893469b2d95daac8` 54 | - take `5129b8f88a414b8e893469b2d95daac8` 55 | 56 | 3. Connect you kindle to your machine and get the path to the kindle (on Mac this is `/Volumes/Kindle`) 57 | 4. run `notionfy` with: 58 | 59 | ```bash 60 | notionfy -n "notion_token" -p "parent_page_id" -k "kindle_path" 61 | ``` 62 | 63 | 5. See the highlights added to notion page 64 | 65 | ![Highlights](./highlights.png) 66 | 67 | 68 | ## Changelog 69 | - `0.2.0` 70 | - Fixing changing Notion api which made syncing impossible 71 | - Moving to Scala codebase 72 | - `0.2.3` 73 | - updating graalVm version 74 | - `0.2.6` 75 | - adjusting to notion api change 76 | - `0.3.0` 77 | - Now adds the reference of where the highlight is as first line to the content in Notion 78 | - `0.3.1` 79 | - Api Limit changed to 100 for Notion, currently only a limited (< 100) number of highlights is supported until pagination is added 80 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import Dependencies.Libraries 2 | 3 | name := """notionfys""" 4 | 5 | organization in ThisBuild := "notionfys" 6 | 7 | scalaVersion in ThisBuild := "2.13.1" 8 | 9 | version in ThisBuild := "0.3.1" 10 | 11 | enablePlugins(GraalVMNativeImagePlugin) 12 | 13 | mappings in (Compile, packageDoc) := Seq() 14 | 15 | enablePlugins(NativeImagePlugin) 16 | 17 | nativeImageOptions ++= List( 18 | "--no-fallback", 19 | "--allow-incomplete-classpath", 20 | "--report-unsupported-elements-at-runtime", 21 | "--initialize-at-build-time", 22 | "--enable-https", 23 | "-J-Xmx8g" 24 | ) 25 | 26 | graalVMNativeImageOptions ++= Seq( 27 | "--no-fallback", 28 | "--allow-incomplete-classpath", 29 | "--report-unsupported-elements-at-runtime", 30 | "--initialize-at-build-time", 31 | "--enable-https", 32 | "-J-Xmx8g" 33 | ) 34 | 35 | val nativeImagePath = sys.env.get("NATIVE_IMAGE_PATH") 36 | .map(path => Seq(graalVMNativeImageCommand := path)) 37 | .getOrElse(Seq()) 38 | 39 | lazy val commonSettings = Seq( 40 | organizationName := "notionfys", 41 | scalafmtOnCompile := true, 42 | libraryDependencies ++= Seq( 43 | Libraries.cats, 44 | Libraries.catsEffect, 45 | Libraries.catsMtl, 46 | Libraries.osLib, 47 | Libraries.atto, 48 | Libraries.decline, 49 | Libraries.circe, 50 | Libraries.circeGeneric, 51 | Libraries.sttp, 52 | Libraries.sttpCirce, 53 | Libraries.scalaTest % Test, 54 | Libraries.scalaCheck % Test, 55 | compilerPlugin(Libraries.kindProjector), 56 | compilerPlugin(Libraries.betterMonadicFor) 57 | ) 58 | ) ++ nativeImagePath 59 | 60 | lazy val root = 61 | (project in file(".")) 62 | .settings(Compile / mainClass := Some("notionfys.Main")) 63 | .settings(commonSettings) 64 | -------------------------------------------------------------------------------- /highlights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yannick-cw/notionfy/8a1f5b638142216e49f7e9429fe5244c29c063b8/highlights.png -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | 5 | object Versions { 6 | val cats = "2.0.0" 7 | val catsEffect = "2.0.0" 8 | 9 | // Test 10 | val scalaTest = "3.0.8" 11 | val scalaCheck = "1.14.2" 12 | 13 | val sttp = "2.1.0-RC1" 14 | 15 | val circe = "0.13.0" 16 | 17 | // Compiler 18 | val kindProjector = "0.10.3" 19 | val betterMonadicFor = "0.3.0" 20 | } 21 | 22 | object Libraries { 23 | lazy val cats = "org.typelevel" %% "cats-core" % Versions.cats 24 | lazy val catsEffect = "org.typelevel" %% "cats-effect" % Versions.catsEffect 25 | lazy val catsMtl = "org.typelevel" %% "cats-mtl-core" % "0.7.1" 26 | 27 | lazy val circe = "io.circe" %% "circe-core" % Versions.circe 28 | lazy val circeGeneric = "io.circe" %% "circe-generic" % Versions.circe 29 | 30 | lazy val sttp = "com.softwaremill.sttp.client" %% "core" % Versions.sttp 31 | lazy val sttpCirce = "com.softwaremill.sttp.client" %% "circe" % Versions.sttp 32 | 33 | lazy val atto = "org.tpolecat" %% "atto-core" % "0.7.0" 34 | 35 | lazy val osLib = "com.lihaoyi" %% "os-lib" % "0.6.3" 36 | 37 | lazy val decline = "com.monovore" %% "decline-effect" % "1.2.0" 38 | 39 | // Test 40 | lazy val scalaTest = "org.scalatest" %% "scalatest" % Versions.scalaTest 41 | lazy val scalaCheck = "org.scalacheck" %% "scalacheck" % Versions.scalaCheck 42 | 43 | // Compiler 44 | lazy val kindProjector = "org.typelevel" %% "kind-projector" % Versions.kindProjector 45 | lazy val betterMonadicFor = "com.olegpy" %% "better-monadic-for" % Versions.betterMonadicFor 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.3.4 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.2.1") 2 | addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.8") 3 | addSbtPlugin("org.scalameta" % "sbt-native-image" % "0.1.1") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.0") 5 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Exit on failure 4 | set -e 5 | 6 | mkdir release 7 | 8 | if [ $TRAVIS_OS_NAME = windows ]; then 9 | mv target/graalvm-native-image/notionfys.exe release/notionfy.exe 10 | else 11 | mv target/native-image/notionfys release/notionfy 12 | cd release 13 | zip "notionfy_$TRAVIS_OS_NAME.zip" notionfy 14 | tar -zcvf "notionfy_$TRAVIS_OS_NAME.tar.gz" notionfy 15 | rm -f notionfy 16 | fi 17 | -------------------------------------------------------------------------------- /scripts/windows.bat: -------------------------------------------------------------------------------- 1 | call "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvars64.bat" 2 | 3 | sbt "graalvm-native-image:packageBin" 4 | 5 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/App.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import cats.data.Kleisli 4 | import cats.effect.IO 5 | import sttp.model.StatusCode 6 | 7 | object App { 8 | val AppM = Kleisli 9 | type AppM[A] = Kleisli[IO, Args, A] 10 | 11 | sealed trait Error extends Throwable 12 | 13 | case class FileNotFoundErr(path: os.Path) extends Error 14 | case class HttpResponseError(code: StatusCode, body: String) extends Error 15 | case class DecodingError(errMsg: String, body: String) extends Error 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Cli.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import cats.data.ValidatedNel 4 | import cats.data.NonEmptyList 5 | import cats.data.Validated 6 | import scala.util.Try 7 | import java.{util => ju} 8 | import com.monovore.decline._ 9 | import cats.implicits._ 10 | 11 | case class Args( 12 | token: String, 13 | page: ju.UUID, 14 | kindle: os.Path, 15 | verbose: Boolean = false 16 | ) 17 | 18 | object Cli { 19 | def parseArgs: Opts[Args] = 20 | ( 21 | Opts.option[String]( 22 | "token", 23 | short = "n", 24 | metavar = "id", 25 | help = "Your notion token, found in the token_v2 cookie when you open notion in the browser" 26 | ), 27 | Opts 28 | .option[String]( 29 | "page", 30 | short = "p", 31 | metavar = "id", 32 | help = 33 | "Id of the page to which the highlights should be added, you find it in the url if you open the page in the browser" 34 | ) 35 | .mapValidated(parseToUUID), 36 | Opts 37 | .option[String]( 38 | "kindle", 39 | short = "k", 40 | metavar = "Path", 41 | help = "Path to your kindle, e.g. on Mac /Volumes/Kindle" 42 | ) 43 | .mapValidated( 44 | p => 45 | Validated 46 | .fromTry(Try(os.Path(p))) 47 | .bimap(_ => NonEmptyList.one(s"$p is not an absolute path"), identity) 48 | ), 49 | Opts.flag("verbose", help = "turn on verbose logging").orFalse 50 | ).mapN(Args) 51 | 52 | private def parseToUUID(rawId: String): ValidatedNel[String, ju.UUID] = 53 | Validated 54 | .fromTry(Try(ju.UUID.fromString(rawId.zipWithIndex.flatMap { 55 | case (c, p) if p == 7 || p == 11 || p == 15 || p == 19 => List(c, '-') 56 | case (c, _) => List(c) 57 | }.mkString))) 58 | .bimap(_ => NonEmptyList.one(s"$rawId was is not parsebble to UUID"), identity) 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Console.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import App._ 4 | import cats.effect.IO 5 | 6 | trait Console[F[_]] { 7 | def verboseLog(msg: String): F[Unit] 8 | def log(msg: String): F[Unit] 9 | } 10 | 11 | object Console extends Console[AppM] { 12 | 13 | override def log(msg: String): App.AppM[Unit] = AppM.liftF(IO(println(msg))) 14 | 15 | override def verboseLog(msg: String): AppM[Unit] = 16 | AppM 17 | .ask[IO, Args] 18 | .flatMap(args => if (args.verbose) AppM.liftF(IO(println(msg))) else AppM.pure(())) 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/FS.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import App._ 4 | import cats.effect.IO 5 | import cats.implicits.{catsSyntaxApply, catsSyntaxIfM} 6 | 7 | trait FS[F[_]] { 8 | def readF(p: os.Path): F[String] 9 | } 10 | 11 | object FS extends FS[AppM] { 12 | private def checkFileExists(p: os.Path): IO[Unit] = 13 | IO(os.exists(p)) 14 | .ifM(ifTrue = IO.unit, ifFalse = IO.raiseError(FileNotFoundErr(p))) 15 | 16 | def readF(p: os.Path): AppM[String] = AppM.liftF(checkFileExists(p) *> IO(os.read(p))) 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Highlights.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import cats.Applicative 4 | import atto._ 5 | import Atto._ 6 | import cats.implicits._ 7 | import cats.data.NonEmptyList 8 | 9 | trait Highlights[F[_]] { 10 | def parseKindleHighlights(f: String): F[List[Highlight]] 11 | } 12 | 13 | object Highlights { 14 | 15 | case class ContentPart(title: String, reference: String, content: String) 16 | case class TagPart(title: String, tags: NonEmptyList[String]) 17 | 18 | def apply[F[_]: Applicative] = new Highlights[F] { 19 | def parseKindleHighlights(f: String): F[List[Highlight]] = 20 | Applicative[F].pure( 21 | pp.parseOnly(f.filterNot(_ == '\r')).either.getOrElse(List.empty).filter(_.content.nonEmpty) 22 | ) 23 | 24 | val whitespace = char(' ') 25 | val sepP = string("==========") 26 | val title = many1(notChar('\n')).map(_.toList.mkString) 27 | val reference = many1(notChar('\n')).map(_.toList.mkString) 28 | val lines = many(notChar('\n')).map(_.mkString) 29 | val tagP = many1(letter | digit) 30 | 31 | val tags = (char('.') ~> tagP.map(_.toList.mkString)) 32 | .sepBy1(whitespace | char(',') | char(',') <~ whitespace) 33 | 34 | val eol = char('\n') 35 | 36 | val contentPartParser: Parser[ContentPart] = 37 | (title <~ eol, reference <~ eol <~ lines <~ eol, lines <~ eol).mapN(ContentPart) 38 | 39 | val tagPartParser: Parser[TagPart] = 40 | (title <~ eol <~ lines <~ eol <~ lines <~ eol, tags <~ eol).mapN(TagPart) 41 | 42 | val segmentP: Parser[Either[(TagPart, ContentPart), ContentPart]] = 43 | (((tagPartParser <~ sepP <~ eol) ~ contentPartParser || contentPartParser) <~ sepP <~ eol) 44 | 45 | val pp = many(segmentP.map { 46 | case Right(c) => Highlight(c.title, c.reference, c.content, List.empty) 47 | case Left((t, c)) => Highlight(c.title, c.reference, c.content, t.tags.toList) 48 | }) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Main.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import cats.effect._ 4 | import cats.implicits._ 5 | import cats.mtl.implicits._ 6 | import com.monovore.decline._ 7 | import com.monovore.decline.effect._ 8 | import App._ 9 | import scala.util.control.NonFatal 10 | import sttp.client.DeserializationError 11 | import sttp.client.HttpError 12 | 13 | case class Highlight(title: String, reference: String, content: String, tags: List[String]) { 14 | def newContent: String = reference + "\n" + content 15 | } 16 | 17 | object Main 18 | extends CommandIOApp( 19 | name = "notionfy", 20 | header = "Sync your Kindle highlights to Notion", 21 | version = "0.3.1" 22 | ) { 23 | 24 | implicit val F: FS[AppM] = FS 25 | implicit val N: Notion[AppM] = Notion 26 | implicit val H: Highlights[AppM] = Highlights[AppM] 27 | implicit val C: Console[AppM] = Console 28 | 29 | private def errorMsg(errShort: String, errLong: String, args: Args): IO[ExitCode] = 30 | IO( 31 | println( 32 | s"\n\nUnfortunately syncing failed, you can run with --verbose to see more details about what is going on. Please report the output to https://github.com/yannick-cw/notionfys, so I can fix it. Here is the error:\n\n${if (args.verbose) 33 | errLong 34 | else errShort}" 35 | ) 36 | ).as(ExitCode.Error) 37 | 38 | private def errorMsg(err: Throwable, args: Args): IO[ExitCode] = 39 | errorMsg( 40 | err.getMessage, 41 | err.getLocalizedMessage ++ "\n" ++ formatTrace(err.getStackTrace), 42 | args 43 | ) 44 | 45 | private def formatTrace(trace: Array[StackTraceElement]) = trace.mkString("\n") 46 | 47 | override def main: Opts[IO[ExitCode]] = 48 | Cli.parseArgs 49 | .map( 50 | args => 51 | Program 52 | .updateNotion[AppM] 53 | .run(args) 54 | .as(println("Welcome")) 55 | .as(ExitCode.Success) 56 | .recoverWith { 57 | case HttpError(b) => 58 | errorMsg( 59 | "Http request to Notion failed", 60 | "Http request to Notion failed with:\n" ++ b, 61 | args 62 | ) 63 | case err @ DeserializationError(b, e) => 64 | errorMsg( 65 | e.toString, 66 | e.toString ++ "\nwith Stacktrace:\n" ++ formatTrace(err.getStackTrace) 67 | ++ "\nwith response body:\n" ++ b, 68 | args 69 | ) 70 | case FileNotFoundErr(p) => 71 | val notFoundErr = s"Did not find the kindle clippings file under the path $p" 72 | errorMsg(notFoundErr, notFoundErr, args) 73 | case HttpResponseError(code, body) => 74 | errorMsg( 75 | errShort = s"Http request failed with status code: $code", 76 | errLong = s"Http request failed with status code: $code and body: $body", 77 | args = args 78 | ) 79 | case DecodingError(msg, body) => 80 | errorMsg( 81 | errShort = s"Http request failed while decoding the response with: $msg", 82 | errLong = 83 | s"Http request failed while decoding the response with: $msg and body: $body", 84 | args = args 85 | ) 86 | case NonFatal(err) => errorMsg(err, args) 87 | } 88 | ) 89 | } 90 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Notion.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import java.util.UUID 4 | 5 | import App._ 6 | import sttp.client._ 7 | import sttp.client.circe._ 8 | import io.circe._ 9 | import io.circe.generic.semiauto._ 10 | import io.circe.syntax._ 11 | import cats.effect.IO 12 | import java.{util => ju} 13 | 14 | import cats.MonadError 15 | 16 | trait Notion[F[_]] { 17 | def addHighlight(userId: ju.UUID): Highlight => F[Unit] 18 | 19 | def getHighlights: F[(List[Highlight], ju.UUID)] 20 | } 21 | 22 | object Notion extends Notion[AppM] { 23 | implicit val backend: SttpBackend[Identity, Nothing, NothingT] = HttpURLConnectionBackend() 24 | private val notionUrl = "https://www.notion.so/api/v3" 25 | private val noIdErr: AppM[ju.UUID] = MonadError[AppM, Throwable].raiseError( 26 | new RuntimeException("Did not find a notion userId in pageChunkResponse") 27 | ) 28 | 29 | private def randomId = ju.UUID.randomUUID 30 | 31 | def addHighlight(userId: ju.UUID): Highlight => AppM[Unit] = 32 | highlight => 33 | for { 34 | Args(token, page, _, _) <- AppM.ask[IO, Args] 35 | (headerId, contentId, separatorId) = (randomId, randomId, randomId) 36 | titlePart = List( 37 | addSegment(headerId, page, userId, "sub_header"), 38 | addAfter(headerId, page), 39 | addContent(headerId, highlight.title) 40 | ) 41 | contentPart = List( 42 | addSegment(contentId, page, userId, "text"), 43 | addAfter(contentId, page), 44 | addContent(contentId, highlight.newContent) 45 | ) 46 | separator = List( 47 | addSegment(separatorId, page, userId, "divider"), 48 | addAfter(separatorId, page) 49 | ) 50 | reqBody = Transaction(titlePart ++ contentPart ++ separator) 51 | _ <- reqRes[Transaction, Unit](reqBody, "submitTransaction", token) 52 | } yield () 53 | 54 | private def addAfter(opId: ju.UUID, parentId: ju.UUID) = 55 | Operation( 56 | id = parentId, 57 | path = List("content"), 58 | command = "listAfter", 59 | table = "block", 60 | args = ObjArgs(opId) 61 | ) 62 | 63 | private def addContent(opId: ju.UUID, content: String) = 64 | Operation( 65 | id = opId, 66 | path = List("properties", "title"), 67 | command = "set", 68 | table = "block", 69 | args = ArrayArgs(List(List(content))) 70 | ) 71 | 72 | private def addSegment(opId: ju.UUID, parentId: ju.UUID, userId: ju.UUID, `type`: String) = 73 | Operation( 74 | id = opId, 75 | path = List.empty, 76 | command = "set", 77 | table = "block", 78 | args = ObjArgs(id = opId, createdBy = userId, parentId = parentId, `type` = `type`) 79 | ) 80 | 81 | def getHighlights: AppM[(List[Highlight], ju.UUID)] = 82 | for { 83 | Args(token, page, _, _) <- AppM.ask[IO, Args] 84 | reqBody = PageChunkRequest(page) 85 | pageChunkResponse <- reqRes[PageChunkRequest, PageChunkResponse]( 86 | reqBody, 87 | "loadPageChunk", 88 | token 89 | ) 90 | highlights = extractHighlights(pageChunkResponse, page) 91 | userId <- pageChunkResponse.recordMap.block 92 | .get(page) 93 | .flatMap(_.value.created_by_id) 94 | .fold(noIdErr)(AppM.pure(_)) 95 | } yield (highlights, userId) 96 | 97 | private def reqRes[A: Encoder, Res: Decoder](body: A, path: String, token: String) = 98 | AppM.liftF(for { 99 | res <- IO( 100 | basicRequest 101 | .post(uri"$notionUrl/$path") 102 | .cookie("token_v2", token) 103 | .body(body) 104 | .response(asJson[Res]) 105 | .send() 106 | ) 107 | resV <- IO.fromEither(res.body.left.map { 108 | case HttpError(body) => HttpResponseError(res.code, body) 109 | case DeserializationError(body, err) => DecodingError(err.getMessage, body) 110 | }) 111 | } yield resV) 112 | 113 | private def extractHighlights(response: PageChunkResponse, pageId: ju.UUID): List[Highlight] = 114 | for { 115 | contents <- response.recordMap.block.get(pageId).toList.flatMap(_.value.content) 116 | (title, content) <- contents 117 | .flatMap(cId => response.recordMap.block.get(cId).toList.map(_.value)) 118 | .grouped(3) 119 | .toList 120 | .collect { 121 | case List( 122 | Value(Some("sub_header"), Some(propsT), _, _), 123 | Value(Some("text"), Some(propsH), _, _), 124 | Value(Some("divider"), None, _, _) 125 | ) => 126 | (propsT, propsH) 127 | } 128 | titleA <- title.title.toList.flatMap(_.headOption.toList.flatMap(_.headOption.toList)) 129 | contentA <- content.title.toList 130 | .flatMap(_.headOption.toList.flatMap(_.headOption.toList)) 131 | } yield Highlight(titleA, "", contentA, List.empty) 132 | } 133 | 134 | sealed trait Arg 135 | 136 | object Arg { 137 | implicit val encoder: Encoder[Arg] = Encoder.instance { 138 | case arr: ArrayArgs => arr.asJson 139 | case obj: ObjArgs => obj.asJson 140 | } 141 | } 142 | 143 | case class ArrayArgs(path: List[List[String]]) extends Arg 144 | 145 | object ArrayArgs { 146 | implicit val encoder: Encoder[ArrayArgs] = Encoder[List[List[String]]].contramap(_.path) 147 | } 148 | 149 | case class ObjArgs( 150 | id: ju.UUID, 151 | version: Option[Int], 152 | alive: Option[String], 153 | created_by: Option[ju.UUID], 154 | parent_id: Option[ju.UUID], 155 | parent_table: Option[String], 156 | `type`: Option[String] 157 | ) extends Arg 158 | 159 | object ObjArgs { 160 | implicit val encoder: Encoder[ObjArgs] = deriveEncoder[ObjArgs].mapJson(_.dropNullValues) 161 | 162 | def apply(id: ju.UUID): ObjArgs = ObjArgs(id, None, None, None, None, None, None) 163 | 164 | def apply(id: ju.UUID, createdBy: ju.UUID, parentId: ju.UUID, `type`: String): ObjArgs = 165 | ObjArgs( 166 | id, 167 | Some(1), 168 | Some("True"), 169 | Some(createdBy), 170 | Some(parentId), 171 | Some("block"), 172 | Some(`type`) 173 | ) 174 | } 175 | 176 | case class Operation(id: ju.UUID, path: List[String], command: String, table: String, args: Arg) 177 | 178 | object Operation { 179 | implicit val encoder: Encoder[Operation] = deriveEncoder 180 | } 181 | 182 | case class Transaction(operations: List[Operation]) 183 | 184 | object Transaction { 185 | implicit val encoder: Encoder[Transaction] = deriveEncoder 186 | } 187 | 188 | // Response 189 | case class Properties(title: Option[List[List[String]]]) 190 | 191 | object Properties { 192 | implicit val decoder: Decoder[Properties] = deriveDecoder 193 | } 194 | 195 | case class Value( 196 | `type`: Option[String], 197 | properties: Option[Properties], 198 | content: Option[List[ju.UUID]], 199 | created_by_id: Option[UUID] 200 | ) 201 | 202 | object Value { 203 | implicit val decoder: Decoder[Value] = deriveDecoder 204 | } 205 | 206 | case class Block(value: Value) 207 | 208 | object Block { 209 | implicit val decoder: Decoder[Block] = deriveDecoder 210 | } 211 | 212 | case class RecordMap(block: Map[ju.UUID, Block]) 213 | 214 | object RecordMap { 215 | implicit val decoder: Decoder[RecordMap] = deriveDecoder 216 | } 217 | 218 | case class PageChunkResponse(recordMap: RecordMap) 219 | 220 | object PageChunkResponse { 221 | implicit val decoder: Decoder[PageChunkResponse] = deriveDecoder 222 | } 223 | 224 | // Request 225 | case class Stack(table: String, id: ju.UUID, index: Int) 226 | 227 | object Stack { 228 | implicit val encoder: Encoder[Stack] = deriveEncoder 229 | } 230 | 231 | case class Cursor(stack: List[List[Stack]]) 232 | 233 | object Cursor { 234 | implicit val encoder: Encoder[Cursor] = deriveEncoder 235 | } 236 | 237 | case class PageChunkRequest( 238 | pageId: ju.UUID, 239 | limit: Int, 240 | cursor: Cursor, 241 | chunkNumber: Int, 242 | verticalColumns: Boolean 243 | ) 244 | 245 | object PageChunkRequest { 246 | implicit val encoder: Encoder[PageChunkRequest] = deriveEncoder 247 | 248 | def apply(pageId: ju.UUID): PageChunkRequest = 249 | PageChunkRequest( 250 | pageId = pageId, 251 | limit = 100, 252 | cursor = Cursor(stack = List(List(Stack(table = "block", id = pageId, index = 0)))), 253 | chunkNumber = 0, 254 | verticalColumns = false 255 | ) 256 | } 257 | -------------------------------------------------------------------------------- /src/main/scala/notionfys/Program.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import cats.mtl.ApplicativeAsk 4 | import cats.Monad 5 | import cats.implicits._ 6 | 7 | object Program { 8 | def updateNotion[F[_]: Monad]( 9 | implicit FS: FS[F], 10 | Notion: Notion[F], 11 | Highlights: Highlights[F], 12 | Console: Console[F], 13 | Ask: ApplicativeAsk[F, Args] 14 | ): F[Unit] = 15 | for { 16 | kindlePath <- Ask.reader(_.kindle) 17 | _ <- Console.verboseLog(s"Reading highlights from Kindle at $kindlePath....") 18 | kindleFile <- FS.readF(kindlePath / "documents" / "My Clippings.txt") 19 | _ <- Console.verboseLog("Done reading kindle highlights") 20 | kindleHighlights <- Highlights.parseKindleHighlights(kindleFile) 21 | _ <- Console.verboseLog( 22 | s"\nFound Highlights locally:\n${kindleHighlights.map(_.title).mkString("\n")}\n" 23 | ) 24 | _ <- Console.verboseLog("Parsed kindle highlights to internal format") 25 | _ <- Console.verboseLog("Fetching highlights from Notion...") 26 | (currentHighlights, userId) <- Notion.getHighlights 27 | _ <- Console.verboseLog("Fetched current highlights from Notion") 28 | _ <- Console.verboseLog( 29 | s"\nFound Highlights in Notion:\n${currentHighlights.map(_.title).mkString("\n")}\n" 30 | ) 31 | newHighlights = kindleHighlights.filterNot( 32 | h => 33 | currentHighlights.exists( 34 | cH => cH.title == h.title && (cH.content == h.newContent || cH.content == h.content) 35 | ) 36 | ) 37 | _ <- Console.verboseLog( 38 | s"\nSyncing new highlights:\n${newHighlights.map(_.title).mkString("\n")}" 39 | ) 40 | _ <- Console.verboseLog("....\n") 41 | _ <- newHighlights.traverse(Notion.addHighlight(userId)) 42 | _ <- Console.log("Done syncing new highlights to Notion") 43 | _ <- Console.log("Shutting down") 44 | } yield () 45 | } 46 | -------------------------------------------------------------------------------- /src/test/resources/My Clippings.txt: -------------------------------------------------------------------------------- 1 | Tools of Titans (Timothy Ferriss) 2 | - Ihre Markierung bei Position 6801-6805 | Hinzugefügt am Mittwoch, 9. Oktober 2019 11:13:06 3 | 4 | brainpicking 5 | ========== 6 | Der Rithmatist: Roman (German Edition) (Sanderson, Brandon) 7 | - Ihr Lesezeichen auf Seite 307 | Position 3793 | Hinzugefügt am Mittwoch, 16. Oktober 2019 11:34:47 8 | 9 | .tag1,.tag2,.tag3 10 | ========== 11 | Der Rithmatist: Roman (German Edition) (Sanderson, Brandon) 12 | - Ihr Lesezeichen auf Seite 307 | Position 3793 | Hinzugefügt am Mittwoch, 16. Oktober 2019 11:34:47 13 | 14 | What 15 | ========== 16 | Thinking with Types (Sandy Maguire) 17 | - Ihre Markierung bei Position 224-225 | Hinzugefügt am Mittwoch, 6. November 2019 06:10:25 18 | 19 | .fp .is .good 20 | ========== 21 | Thinking with Types (Sandy Maguire) 22 | - Ihre Markierung bei Position 224-225 | Hinzugefügt am Mittwoch, 6. November 2019 06:10:25 23 | 24 | Monads are fun 25 | ========== 26 | -------------------------------------------------------------------------------- /src/test/resources/issue_3_clippings.txt: -------------------------------------------------------------------------------- 1 | Indistractable (Nir Eyal;) 2 | - Your Highlight on page 69 | Location 1051-1052 | Added on Friday, October 11, 2019 8:50:35 AM 3 | 4 | The Fogg Behavior Model states that for a behavior (B) to occur, three things must be present at the same time: motivation (M), ability (A), and a trigger (T). More succinctly, B = MAT. 5 | ========== 6 | Indistractable (Nir Eyal;) 7 | - Your Highlight on page 71 | Location 1084-1085 | Added on Friday, October 11, 2019 9:02:49 AM 8 | 9 | Is this trigger serving me, or am I serving it? 10 | ========== 11 | 12 | -------------------------------------------------------------------------------- /src/test/resources/win.txt: -------------------------------------------------------------------------------- 1 | Das Jesus-Video: Thriller (German Edition) (Eschbach, Andreas) 2 | - Ihr Lesezeichen bei Position 1420 | Hinzugefügt am Samstag, 3. Januar 2015 19:14:54 3 | 4 | 5 | a========== 6 | Blindsight (Peter Watts) 7 | - Ihre Markierung bei Position 1446-1446 | Hinzugefügt am Sonntag, 7. Mai 2017 22:26:04 8 | 9 | chaff 10 | ========== 11 | Tools of Titans (Timothy Ferriss) 12 | - Ihre Notiz bei Position 6805 | Hinzugefügt am Mittwoch, 9. Oktober 2019 11:13:06 13 | 14 | brainpicking 15 | ========== 16 | Tools of Titans (Timothy Ferriss) 17 | - Ihre Markierung bei Position 6813-6813 | Hinzugefügt am Mittwoch, 9. Oktober 2019 11:14:03 18 | 19 | What is the best or most 20 | ========== 21 | Der Rithmatist: Roman (German Edition) (Sanderson, Brandon) 22 | - Ihr Lesezeichen auf Seite 307 | Position 3793 | Hinzugefügt am Mittwoch, 16. Oktober 2019 11:34:47 23 | 24 | 25 | ========== 26 | 27 | -------------------------------------------------------------------------------- /src/test/scala/notionfys/HighlightsSpec.scala: -------------------------------------------------------------------------------- 1 | package notionfys 2 | 3 | import org.scalatest.{FlatSpec, Matchers} 4 | import cats.Id 5 | 6 | class HighlightsSpec extends FlatSpec with Matchers { 7 | "Program" should "find all highlights" in { 8 | val clippingsFile = os.read(os.resource / "My Clippings.txt") 9 | val parsedHighlights: List[Highlight] = Highlights[Id].parseKindleHighlights(clippingsFile) 10 | parsedHighlights should contain.theSameElementsAs(expectedHighlights) 11 | } 12 | 13 | it should "parse issue3 file" in { 14 | val clippingsFile = os.read(os.resource / "issue_3_clippings.txt") 15 | val parsedHighlights: List[Highlight] = Highlights[Id].parseKindleHighlights(clippingsFile) 16 | parsedHighlights should contain.theSameElementsAs(issue3Highlights) 17 | } 18 | 19 | def expectedHighlights = List( 20 | Highlight( 21 | title = "Tools of Titans (Timothy Ferriss)", 22 | reference = 23 | "- Ihre Markierung bei Position 6801-6805 | Hinzugefügt am Mittwoch, 9. Oktober 2019 11:13:06", 24 | content = "brainpicking", 25 | List.empty 26 | ), 27 | Highlight( 28 | title = "Der Rithmatist: Roman (German Edition) (Sanderson, Brandon)", 29 | reference = 30 | "- Ihr Lesezeichen auf Seite 307 | Position 3793 | Hinzugefügt am Mittwoch, 16. Oktober 2019 11:34:47", 31 | content = "What", 32 | List("tag1", "tag2", "tag3") 33 | ), 34 | Highlight( 35 | title = "Thinking with Types (Sandy Maguire)", 36 | reference = 37 | "- Ihre Markierung bei Position 224-225 | Hinzugefügt am Mittwoch, 6. November 2019 06:10:25", 38 | content = "Monads are fun", 39 | tags = List("fp", "is", "good") 40 | ) 41 | ) 42 | 43 | def issue3Highlights = 44 | List( 45 | Highlight( 46 | title = "Indistractable (Nir Eyal;)", 47 | reference = 48 | "- Your Highlight on page 69 | Location 1051-1052 | Added on Friday, October 11, 2019 8:50:35 AM", 49 | content = 50 | "The Fogg Behavior Model states that for a behavior (B) to occur, three things must be present at the same time: motivation (M), ability (A), and a trigger (T). More succinctly, B = MAT.", 51 | List.empty 52 | ), 53 | Highlight( 54 | title = "Indistractable (Nir Eyal;)", 55 | reference = 56 | "- Your Highlight on page 71 | Location 1084-1085 | Added on Friday, October 11, 2019 9:02:49 AM", 57 | content = "Is this trigger serving me, or am I serving it?", 58 | List.empty 59 | ) 60 | ) 61 | } 62 | --------------------------------------------------------------------------------