├── .github └── workflows │ ├── publish.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── pinboard_to_sqlite ├── __init__.py └── cli.py ├── poetry.lock ├── pyproject.toml ├── pytest.ini └── tests ├── __init__.py ├── posts.json └── test_pinboard_to_sqlite.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | on: 3 | release: 4 | types: [created] 5 | jobs: 6 | publish: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v1 10 | - name: Set up Python 11 | uses: actions/setup-python@v1 12 | with: 13 | python-version: "3.x" 14 | - name: Install dependencies 15 | run: | 16 | python -m pip install --upgrade pip 17 | python -m pip install poetry==1.0.0b3 18 | - name: Build and publish 19 | env: 20 | PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 21 | run: | 22 | python -m poetry publish --build -u __token__ -p $PYPI_TOKEN 23 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | on: [push] 3 | jobs: 4 | test: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | max-parallel: 4 8 | matrix: 9 | python-version: [3.6, 3.7, 3.8] 10 | fail-fast: false 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Set up Python ${{ matrix.python-version }} 14 | uses: actions/setup-python@v1 15 | with: 16 | python-version: ${{ matrix.python-version }} 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | python -m pip install poetry==1.0.0b3 21 | python -m poetry install 22 | - name: Test with pytest 23 | run: | 24 | python -m poetry run pytest 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | auth.json 2 | pinboard_to_sqlite.egg-info 3 | *.db 4 | .coverage 5 | .vscode/ 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 3 | Version 2, December 2004 4 | 5 | Copyright (C) 2004 Jacob Kaplan-Moss 6 | 7 | Everyone is permitted to copy and distribute verbatim or modified 8 | copies of this license document, and changing it is allowed as long 9 | as the name is changed. 10 | 11 | DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE 12 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 13 | 14 | 0. You just DO WHAT THE FUCK YOU WANT TO. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Save data from Pinboard to a SQLite database. 2 | 3 | Inspired by (and using libraries from) [Simon Willison's Dogsheep 4 | project](https://github.com/dogsheep). You're probably going to want to run 5 | [Datasette](https://github.com/simonw/datasette) on the resulting db. 6 | 7 | ## How to install 8 | 9 | ``` 10 | $ pip install pinboard-to-sqlite 11 | ``` 12 | 13 | ## Authentication 14 | 15 | Run: 16 | 17 | ``` 18 | $ pinboard-to-sqlite auth 19 | ``` 20 | 21 | This will direct you to https://pinboard.in/settings/password to find your API 22 | token, which you'll then paste into the terminal. This'll get saved in an 23 | `auth.json` file, which subsequent commands will pick up. 24 | 25 | To save to a different file, see the `-a` / `--auth` flag. 26 | 27 | ## Fetching posts 28 | 29 | Run: 30 | 31 | ``` 32 | $ pinboard-to-sqlite posts pinboard.db 33 | ``` 34 | 35 | Where `pinboard.db` is the name of the database you'd like to save posts to. 36 | Note that the API this uses has a rate limit of once per minute, so don't run 37 | this command more than once per minute (I don't know why you would). This 38 | doesn't seem to be enforced fairly loosely, but be careful anyway. 39 | -------------------------------------------------------------------------------- /pinboard_to_sqlite/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobian/pinboard-to-sqlite/e848a1e1a1a72411f6e50fd2776b4abfcf10b036/pinboard_to_sqlite/__init__.py -------------------------------------------------------------------------------- /pinboard_to_sqlite/cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | import os 3 | import json 4 | import sqlite_utils 5 | import requests 6 | import dateutil.parser 7 | 8 | 9 | @click.group() 10 | @click.version_option() 11 | def cli(): 12 | "Save data from Pinboard to a SQLite database" 13 | 14 | 15 | @cli.command() 16 | @click.option( 17 | "-a", 18 | "--auth", 19 | type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), 20 | default="auth.json", 21 | help="Path to save token to", 22 | show_default=True, 23 | ) 24 | def auth(auth): 25 | "Save authentication credentials to a JSON file" 26 | click.echo("Find your API token here: https://pinboard.in/settings/password") 27 | click.echo( 28 | "Paste the whole thing including your username " 29 | "(e.g. yourname:xxxyyyzzz) below." 30 | ) 31 | click.echo() 32 | pinboard_token = click.prompt("API Token") 33 | auth_data = json.load(open(auth)) if os.path.exists(auth) else {} 34 | auth_data["pinboard_token"] = pinboard_token 35 | json.dump(auth_data, open(auth, "w")) 36 | 37 | 38 | @cli.command() 39 | @click.argument( 40 | "database", 41 | required=True, 42 | type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), 43 | ) 44 | @click.option( 45 | "-a", 46 | "--auth", 47 | type=click.Path(file_okay=True, dir_okay=False, allow_dash=False, exists=True), 48 | default="auth.json", 49 | help="Path to read auth token from", 50 | show_default=True, 51 | ) 52 | @click.option( 53 | "--since", 54 | is_flag=True, 55 | default=False, 56 | help="Pull new posts since last saved post in DB", 57 | ) 58 | @click.option("--since-date", metavar="DATE", help="Pull new posts since DATE") 59 | def posts(database, auth, since, since_date): 60 | if since and since_date: 61 | raise click.UsageError("use either --since or --since-date, not both") 62 | 63 | token = json.load(open(auth))["pinboard_token"] 64 | params = {"format": "json", "auth_token": token} 65 | 66 | db = sqlite_utils.Database(database) 67 | 68 | if since and db["posts"].exists: 69 | since_date = db.conn.execute("SELECT max(time) FROM posts;").fetchone()[0] 70 | if since_date: 71 | params["fromdt"] = ( 72 | dateutil.parser.parse(since_date) 73 | .replace(microsecond=0, tzinfo=None) 74 | .isoformat() 75 | + "Z" 76 | ) 77 | 78 | posts = requests.get(f"https://api.pinboard.in/v1/posts/all", params=params).json() 79 | _save_posts(db, posts) 80 | 81 | 82 | def _save_posts(db, posts): 83 | # Convert/coerce some fields 84 | for post in posts: 85 | post["shared"] = post["shared"] == "yes" 86 | post["toread"] = post["toread"] == "yes" 87 | post["time"] = dateutil.parser.parse(post["time"]) 88 | post["tags"] = json.dumps(post["tags"].split()) 89 | 90 | db["posts"].upsert_all( 91 | posts, 92 | pk="hash", 93 | column_order=[ 94 | "hash", 95 | "href", 96 | "description", 97 | "extended", 98 | "meta", 99 | "time", 100 | "shared", 101 | "toread", 102 | "tags", 103 | ], 104 | ) 105 | 106 | 107 | # TODO: notes? 108 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | [[package]] 2 | category = "dev" 3 | description = "Atomic file writes." 4 | name = "atomicwrites" 5 | optional = false 6 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 7 | version = "1.3.0" 8 | 9 | [[package]] 10 | category = "dev" 11 | description = "Classes Without Boilerplate" 12 | name = "attrs" 13 | optional = false 14 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 15 | version = "19.3.0" 16 | 17 | [package.extras] 18 | azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] 19 | dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] 20 | docs = ["sphinx", "zope.interface"] 21 | tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] 22 | 23 | [[package]] 24 | category = "main" 25 | description = "Python package for providing Mozilla's CA Bundle." 26 | name = "certifi" 27 | optional = false 28 | python-versions = "*" 29 | version = "2019.9.11" 30 | 31 | [[package]] 32 | category = "main" 33 | description = "Universal encoding detector for Python 2 and 3" 34 | name = "chardet" 35 | optional = false 36 | python-versions = "*" 37 | version = "3.0.4" 38 | 39 | [[package]] 40 | category = "main" 41 | description = "Composable command line interface toolkit" 42 | name = "click" 43 | optional = false 44 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 45 | version = "7.0" 46 | 47 | [[package]] 48 | category = "main" 49 | description = "Extends click.Group to invoke a command without explicit subcommand name" 50 | name = "click-default-group" 51 | optional = false 52 | python-versions = "*" 53 | version = "1.2.2" 54 | 55 | [package.dependencies] 56 | click = "*" 57 | 58 | [[package]] 59 | category = "dev" 60 | description = "Cross-platform colored terminal text." 61 | marker = "sys_platform == \"win32\"" 62 | name = "colorama" 63 | optional = false 64 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 65 | version = "0.4.1" 66 | 67 | [[package]] 68 | category = "dev" 69 | description = "Code coverage measurement for Python" 70 | name = "coverage" 71 | optional = false 72 | python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4" 73 | version = "4.5.4" 74 | 75 | [[package]] 76 | category = "main" 77 | description = "Internationalized Domain Names in Applications (IDNA)" 78 | name = "idna" 79 | optional = false 80 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 81 | version = "2.8" 82 | 83 | [[package]] 84 | category = "dev" 85 | description = "Read metadata from Python packages" 86 | marker = "python_version < \"3.8\"" 87 | name = "importlib-metadata" 88 | optional = false 89 | python-versions = ">=2.7,!=3.0,!=3.1,!=3.2,!=3.3" 90 | version = "0.23" 91 | 92 | [package.dependencies] 93 | zipp = ">=0.5" 94 | 95 | [package.extras] 96 | docs = ["sphinx", "rst.linker"] 97 | testing = ["packaging", "importlib-resources"] 98 | 99 | [[package]] 100 | category = "dev" 101 | description = "More routines for operating on iterables, beyond itertools" 102 | name = "more-itertools" 103 | optional = false 104 | python-versions = ">=3.4" 105 | version = "7.2.0" 106 | 107 | [[package]] 108 | category = "dev" 109 | description = "Core utilities for Python packages" 110 | name = "packaging" 111 | optional = false 112 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 113 | version = "19.2" 114 | 115 | [package.dependencies] 116 | pyparsing = ">=2.0.2" 117 | six = "*" 118 | 119 | [[package]] 120 | category = "dev" 121 | description = "plugin and hook calling mechanisms for python" 122 | name = "pluggy" 123 | optional = false 124 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 125 | version = "0.13.0" 126 | 127 | [package.dependencies] 128 | [package.dependencies.importlib-metadata] 129 | python = "<3.8" 130 | version = ">=0.12" 131 | 132 | [package.extras] 133 | dev = ["pre-commit", "tox"] 134 | 135 | [[package]] 136 | category = "dev" 137 | description = "library with cross-python path, ini-parsing, io, code, log facilities" 138 | name = "py" 139 | optional = false 140 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 141 | version = "1.8.0" 142 | 143 | [[package]] 144 | category = "dev" 145 | description = "Python parsing module" 146 | name = "pyparsing" 147 | optional = false 148 | python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" 149 | version = "2.4.4" 150 | 151 | [[package]] 152 | category = "dev" 153 | description = "pytest: simple powerful testing with Python" 154 | name = "pytest" 155 | optional = false 156 | python-versions = ">=3.5" 157 | version = "5.2.2" 158 | 159 | [package.dependencies] 160 | atomicwrites = ">=1.0" 161 | attrs = ">=17.4.0" 162 | colorama = "*" 163 | more-itertools = ">=4.0.0" 164 | packaging = "*" 165 | pluggy = ">=0.12,<1.0" 166 | py = ">=1.5.0" 167 | wcwidth = "*" 168 | 169 | [package.dependencies.importlib-metadata] 170 | python = "<3.8" 171 | version = ">=0.12" 172 | 173 | [package.extras] 174 | testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] 175 | 176 | [[package]] 177 | category = "dev" 178 | description = "Pytest plugin for measuring coverage." 179 | name = "pytest-cov" 180 | optional = false 181 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 182 | version = "2.8.1" 183 | 184 | [package.dependencies] 185 | coverage = ">=4.4" 186 | pytest = ">=3.6" 187 | 188 | [package.extras] 189 | testing = ["fields", "hunter", "process-tests (2.0.2)", "six", "virtualenv"] 190 | 191 | [[package]] 192 | category = "main" 193 | description = "Extensions to the standard Python datetime module" 194 | name = "python-dateutil" 195 | optional = false 196 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" 197 | version = "2.8.1" 198 | 199 | [package.dependencies] 200 | six = ">=1.5" 201 | 202 | [[package]] 203 | category = "main" 204 | description = "Python HTTP for Humans." 205 | name = "requests" 206 | optional = false 207 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 208 | version = "2.22.0" 209 | 210 | [package.dependencies] 211 | certifi = ">=2017.4.17" 212 | chardet = ">=3.0.2,<3.1.0" 213 | idna = ">=2.5,<2.9" 214 | urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26" 215 | 216 | [package.extras] 217 | security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] 218 | socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] 219 | 220 | [[package]] 221 | category = "main" 222 | description = "Python 2 and 3 compatibility utilities" 223 | name = "six" 224 | optional = false 225 | python-versions = ">=2.6, !=3.0.*, !=3.1.*" 226 | version = "1.13.0" 227 | 228 | [[package]] 229 | category = "main" 230 | description = "CLI tool and Python utility functions for manipulating SQLite databases" 231 | name = "sqlite-utils" 232 | optional = false 233 | python-versions = "*" 234 | version = "1.12.1" 235 | 236 | [package.dependencies] 237 | click = "*" 238 | click-default-group = "*" 239 | tabulate = "*" 240 | 241 | [package.extras] 242 | docs = ["sphinx-rtd-theme", "sphinx-autobuild"] 243 | test = ["pytest", "black"] 244 | 245 | [[package]] 246 | category = "main" 247 | description = "Pretty-print tabular data" 248 | name = "tabulate" 249 | optional = false 250 | python-versions = "*" 251 | version = "0.8.5" 252 | 253 | [package.extras] 254 | widechars = ["wcwidth"] 255 | 256 | [[package]] 257 | category = "main" 258 | description = "HTTP library with thread-safe connection pooling, file post, and more." 259 | name = "urllib3" 260 | optional = false 261 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" 262 | version = "1.25.6" 263 | 264 | [package.extras] 265 | brotli = ["brotlipy (>=0.6.0)"] 266 | secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] 267 | socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"] 268 | 269 | [[package]] 270 | category = "dev" 271 | description = "Measures number of Terminal column cells of wide-character codes" 272 | name = "wcwidth" 273 | optional = false 274 | python-versions = "*" 275 | version = "0.1.7" 276 | 277 | [[package]] 278 | category = "dev" 279 | description = "Backport of pathlib-compatible object wrapper for zip files" 280 | marker = "python_version < \"3.8\"" 281 | name = "zipp" 282 | optional = false 283 | python-versions = ">=2.7" 284 | version = "0.6.0" 285 | 286 | [package.dependencies] 287 | more-itertools = "*" 288 | 289 | [package.extras] 290 | docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] 291 | testing = ["pathlib2", "contextlib2", "unittest2"] 292 | 293 | [metadata] 294 | content-hash = "ff9e881ddb1b4df318f2769832befee112a91da015d5ae86758132c253259d23" 295 | python-versions = "^3.6" 296 | 297 | [metadata.hashes] 298 | atomicwrites = ["03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", "75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"] 299 | attrs = ["08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", "f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"] 300 | certifi = ["e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", "fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"] 301 | chardet = ["84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", "fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"] 302 | click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"] 303 | click-default-group = ["d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904"] 304 | colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"] 305 | coverage = ["08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", "0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", "141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", "19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", "23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", "245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", "331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", "386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", "3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", "60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", "63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", "6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", "6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", "7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", "826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", "93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", "9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", "af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", "bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", "bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", "c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", "dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", "df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", "e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", "e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", "e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", "eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", "eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", "ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", "efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", "fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", "ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025"] 306 | idna = ["c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", "ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"] 307 | importlib-metadata = ["aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", "d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af"] 308 | more-itertools = ["409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", "92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"] 309 | packaging = ["28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", "d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"] 310 | pluggy = ["0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", "fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"] 311 | py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"] 312 | pyparsing = ["4acadc9a2b96c19fe00932a38ca63e601180c39a189a696abce1eaab641447e1", "61b5ed888beab19ddccab3478910e2076a6b5a0295dffc43021890e136edf764"] 313 | pytest = ["27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6", "58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4"] 314 | pytest-cov = ["cc6742d8bac45070217169f5f72ceee1e0e55b0221f54bcf24845972d3a47f2b", "cdbdef4f870408ebdbfeb44e63e07eb18bb4619fae852f6e760645fa36172626"] 315 | python-dateutil = ["73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"] 316 | requests = ["11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", "9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"] 317 | six = ["1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", "30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"] 318 | sqlite-utils = ["64d20aa680468a5ed101fd4173ebd3aa8ddeafbe008d7a9f91934c2b8e21d846"] 319 | tabulate = ["d0097023658d4dea848d6ae73af84532d1e86617ac0925d1adf1dd903985dac3"] 320 | urllib3 = ["3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", "9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"] 321 | wcwidth = ["3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", "f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"] 322 | zipp = ["3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", "f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"] 323 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pinboard-to-sqlite" 3 | version = "1.2.0" 4 | description = "Save data from Pinboard to a SQLite database" 5 | authors = ["Jacob Kaplan-Moss "] 6 | license = "wtfpl" 7 | readme = "README.md" 8 | repository = "https://github.com/jacobian/pinboard-to-sqlite/" 9 | 10 | [tool.poetry.scripts] 11 | pinboard-to-sqlite = "pinboard_to_sqlite.cli:cli" 12 | 13 | [tool.poetry.dependencies] 14 | python = "^3.6" 15 | sqlite-utils = "^1.12.1" 16 | requests = "^2.22.0" 17 | click = "^7.0" 18 | python-dateutil = "^2.8.1" 19 | 20 | [tool.poetry.dev-dependencies] 21 | pytest = "^5.2" 22 | pytest-cov = "^2.8.1" 23 | 24 | [build-system] 25 | requires = ["poetry>=0.12"] 26 | build-backend = "poetry.masonry.api" 27 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --cov=pinboard_to_sqlite/ --cov-report=term -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacobian/pinboard-to-sqlite/e848a1e1a1a72411f6e50fd2776b4abfcf10b036/tests/__init__.py -------------------------------------------------------------------------------- /tests/posts.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "href": "https://example.com/", 4 | "description": "An Example", 5 | "extended": "example description", 6 | "meta": "m1", 7 | "hash": "h1", 8 | "time": "2019-11-06T22:50:31Z", 9 | "shared": "yes", 10 | "toread": "no", 11 | "tags": "fish sheep goats" 12 | }, 13 | { 14 | "href": "https://example.com/2", 15 | "description": "Example 2", 16 | "extended": "", 17 | "meta": "m2", 18 | "hash": "h2", 19 | "time": "2018-11-06T22:50:31Z", 20 | "shared": "no", 21 | "toread": "yes", 22 | "tags": "elephants goats lizards" 23 | } 24 | ] 25 | -------------------------------------------------------------------------------- /tests/test_pinboard_to_sqlite.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import json 3 | import pinboard_to_sqlite.cli 4 | import sqlite_utils 5 | import pathlib 6 | 7 | 8 | @pytest.fixture 9 | def db(): 10 | return sqlite_utils.Database(memory=True) 11 | 12 | 13 | @pytest.fixture 14 | def posts(): 15 | p = pathlib.Path(__file__).parent / "posts.json" 16 | return json.load(open(p)) 17 | 18 | 19 | def test_posts(db, posts): 20 | pinboard_to_sqlite.cli._save_posts(db, posts) 21 | assert ["posts"] == db.table_names() 22 | assert set(p["href"] for p in posts) == set(r["href"] for r in db["posts"].rows) 23 | 24 | 25 | def test_posts_tags_json(db, posts): 26 | pinboard_to_sqlite.cli._save_posts(db, posts) 27 | assert ["fish", "sheep", "goats"] == json.loads(db["posts"].get("h1")["tags"]) 28 | --------------------------------------------------------------------------------