├── .github ├── FUNDING.yml └── workflows │ ├── black.yml │ └── python-test.yml ├── .gitignore ├── .ruff.toml ├── COPYING.GPL-3.0+ ├── LICENSE.md ├── README.md ├── docs ├── CNAME ├── _config.yml ├── create-credentials.png ├── demo.png ├── download-json.png └── index.md ├── gmi ├── lieer ├── __init__.py ├── gmailieer.py ├── local.py ├── nobar.py ├── remote.py └── resume.py ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── conftest.py └── test_local.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: gauteh 2 | custom: ["https://www.paypal.me/gauteh"] 3 | -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- 1 | # This is the official black github action per https://black.readthedocs.io/en/stable/integrations/github_actions.html 2 | # The workflow will fail if `black --check --diff` finds files that need to be formatted 3 | 4 | name: Lint 5 | 6 | on: [push, pull_request] 7 | 8 | jobs: 9 | lint: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: psf/black@stable -------------------------------------------------------------------------------- /.github/workflows/python-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python tests 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | tags: 10 | - v* 11 | pull_request: 12 | branches: [ master ] 13 | 14 | permissions: {} 15 | 16 | jobs: 17 | tests: 18 | 19 | runs-on: ubuntu-latest 20 | strategy: 21 | matrix: 22 | python-version: ["3.9", "3.10", "3.11", "3.12"] 23 | 24 | steps: 25 | - uses: actions/checkout@v3 26 | with: 27 | persist-credentials: false 28 | - name: Set up Python ${{ matrix.python-version }} 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | - name: Install notmuch 33 | run: | 34 | sudo apt-get install notmuch 35 | sudo apt-get install libnotmuch-dev 36 | - name: Install dependencies 37 | run: | 38 | python -m pip install --upgrade pip 39 | pip install pytest ruff 40 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 41 | - name: Lint with ruff 42 | run: | 43 | ruff --output-format=github . 44 | - name: Test with pytest 45 | run: | 46 | pytest -v tests 47 | 48 | - name: Build wheels 49 | run: | 50 | python setup.py sdist 51 | 52 | - name: Upload wheels 53 | if: matrix.python-version == '3.10' 54 | uses: actions/upload-artifact@v2 55 | with: 56 | name: wheels 57 | path: dist 58 | 59 | release: 60 | name: Release 61 | runs-on: ubuntu-latest 62 | if: "startsWith(github.ref, 'refs/tags/')" 63 | needs: [ tests ] 64 | steps: 65 | - uses: actions/download-artifact@v4.1.7 66 | with: 67 | name: wheels 68 | - uses: conda-incubator/setup-miniconda@v2 69 | with: 70 | python-version: '3.10' 71 | miniforge-version: latest 72 | miniforge-variant: Mambaforge 73 | - name: Publish to PyPi 74 | env: 75 | TWINE_USERNAME: __token__ 76 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 77 | run: | 78 | pip install --upgrade twine 79 | twine upload --skip-existing * 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | client_secret.json 3 | .venv 4 | .gmailieer.json 5 | mail 6 | build 7 | dist 8 | *.egg-info 9 | 10 | # Emacs 11 | .dir-locals.el 12 | -------------------------------------------------------------------------------- /.ruff.toml: -------------------------------------------------------------------------------- 1 | # Exclude a variety of commonly ignored directories. 2 | exclude = [ 3 | ".bzr", 4 | ".direnv", 5 | ".eggs", 6 | ".git", 7 | ".git-rewrite", 8 | ".hg", 9 | ".mypy_cache", 10 | ".nox", 11 | ".pants.d", 12 | ".pytype", 13 | ".ruff_cache", 14 | ".svn", 15 | ".tox", 16 | ".venv", 17 | "__pypackages__", 18 | "_build", 19 | "buck-out", 20 | "build", 21 | "dist", 22 | "node_modules", 23 | "venv", 24 | ] 25 | 26 | # Same as Black. 27 | line-length = 88 28 | indent-width = 4 29 | 30 | # Assume Python 3.9 -- sync with GitHub testing matrix 31 | target-version = "py39" 32 | 33 | [lint] 34 | # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. 35 | # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or 36 | # McCabe complexity (`C901`) by default. 37 | select = ["E4", "E7", "E9", "F", "I", "UP", "S", "C4", "EXE", "SIM", "PERF"] 38 | ignore = [ 39 | "E501", # Line too long 40 | "E741", # Ambiguous variable name: `l` 41 | "F403", # `from .gmailieer import *` used; unable to detect undefined names 42 | "S101", # Use of `assert` detected 43 | ] 44 | 45 | # Allow fix for all enabled rules (when `--fix`) is provided. 46 | fixable = ["ALL"] 47 | unfixable = [] 48 | 49 | # Allow unused variables when underscore-prefixed. 50 | dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" 51 | 52 | [format] 53 | # Like Black, use double quotes for strings. 54 | quote-style = "double" 55 | 56 | # Like Black, indent with spaces, rather than tabs. 57 | indent-style = "space" 58 | 59 | # Like Black, respect magic trailing commas. 60 | skip-magic-trailing-comma = false 61 | 62 | # Like Black, automatically detect the appropriate line ending. 63 | line-ending = "auto" 64 | -------------------------------------------------------------------------------- /COPYING.GPL-3.0+: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | {project} Copyright (C) {year} {fullname} 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | “This program” is Lieer. 3 | 4 | Copyright (c) 2017 Gaute Hope 5 | 6 | This program is free software: you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or 9 | (at your option) any later version. 10 | 11 | This program is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License for more details. 15 | 16 | You should have received a copy of the GNU General Public License 17 | along with this program. If not, see . 18 | 19 | See COPYING.GPL-3.0+ 20 | 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | docs/index.md -------------------------------------------------------------------------------- /docs/CNAME: -------------------------------------------------------------------------------- 1 | lieer.gaute.vetsj.com -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | name: Lieer 2 | theme: jekyll-theme-minimal 3 | -------------------------------------------------------------------------------- /docs/create-credentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gauteh/lieer/5c9ca43d3bc040bdb012dbfc1ca4cf15b1451dca/docs/create-credentials.png -------------------------------------------------------------------------------- /docs/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gauteh/lieer/5c9ca43d3bc040bdb012dbfc1ca4cf15b1451dca/docs/demo.png -------------------------------------------------------------------------------- /docs/download-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gauteh/lieer/5c9ca43d3bc040bdb012dbfc1ca4cf15b1451dca/docs/download-json.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Lieer 2 | 3 | 4 | 5 | This program can pull, and send, email and labels (and changes to labels) from 6 | your GMail account and store them locally in a maildir with the labels 7 | synchronized with a [notmuch](https://notmuchmail.org/) database. The changes 8 | to tags in the notmuch database may be pushed back remotely to your GMail 9 | account. 10 | 11 | ## Disclaimer 12 | 13 | Lieer will not and can not: 14 | 15 | * Add or delete messages on your remote account (except syncing the `trash` or `spam` label to messages, and those messages will eventually be [deleted](https://support.google.com/mail/answer/7401?co=GENIE.Platform%3DDesktop&hl=en)) 16 | * Modify messages other than their labels 17 | 18 | While Lieer has been used to successfully synchronize millions of messages and tags by now, it comes with **NO WARRANTIES**. 19 | 20 | ## Requirements 21 | 22 | * Python 3 23 | * Notmuch 0.30+ for `notmuch2` Python bindings 24 | * `google_api_python_client` (sometimes `google-api-python-client`) 25 | * `google_auth_oauthlib` 26 | * `tqdm` (optional - for progress bar) 27 | 28 | ## Installation 29 | 30 | After cloning the repository Lieer can be installed through pip by using the command ```pip install .``` 31 | # Usage 32 | 33 | This assumes your root mail folder is in `~/.mail` and that this folder is _already_ set up with notmuch. 34 | 35 | 1. Make a directory for the lieer storage and state files (local repository). 36 | 37 | ```sh 38 | $ cd ~/.mail 39 | $ mkdir account.gmail 40 | $ cd account.gmail/ 41 | ``` 42 | 43 | All commands should be run from the local mail repository unless otherwise specified. 44 | 45 | 2. Ignore the `.json` files in notmuch. Any tags listed in `new.tags` will be added to newly pulled messages (but see [Caveats](#caveats)). Process tags on new messages directly after running gmi, or run `notmuch new` to trigger the `post-new` hook for [initial tagging](https://notmuchmail.org/initial_tagging/). The `new.tags` are not ignored by default if you do not remove them, but you can prevent custom tags from being pushed to the remote by using e.g. `gmi set --ignore-tags-local new`. In your notmuch config file (usually `~/.notmuch-config`): 46 | 47 | ``` 48 | [new] 49 | tags=new 50 | ignore=/.*[.](json|lock|bak)$/ 51 | ``` 52 | 53 | 3. Initialize the mail storage: 54 | 55 | ```sh 56 | $ gmi init your.email@gmail.com 57 | ``` 58 | 59 | `gmi init` will now open your browser and request limited access to your e-mail. 60 | 61 | > The access token is stored in `.credentials.gmailieer.json` in the local mail repository. If you wish, you can specify [your own api key](#using-your-own-api-key) that should be used. 62 | 63 | 4. You're now set up, and you can do the initial pull. 64 | 65 | > Use `gmi -h` or `gmi command -h` to get more usage information. 66 | 67 | ## Pull 68 | 69 | will pull down all remote changes since last time, overwriting any local tag 70 | changes of the affected messages. 71 | 72 | ```sh 73 | $ gmi pull 74 | ``` 75 | 76 | the first time you do this, or if a full synchronization is needed it will take longer. You can try to use the `--resume` option if you get stuck on getting the metadata and have to abort (this will cause local changes made in the interim to be ignored in the next push). 77 | 78 | ## Push 79 | 80 | will push up all changes since last push, conflicting changes will be ignored 81 | unless `-f` is specified. these will be overwritten with the remote changes at 82 | the next `pull`. 83 | 84 | ```sh 85 | $ gmi push 86 | ``` 87 | 88 | ## Normal synchronization routine 89 | 90 | ```sh 91 | $ cd ~/.mail/account.gmail 92 | $ gmi sync 93 | ``` 94 | 95 | This effectively does a `push` followed by a `pull`. Any conflicts detected 96 | with the remote in `push` will not be pushed. After the next `pull` has been 97 | run the conflicts should be resolved, overwriting the local changes with the 98 | remote changes. You can force the local changes to overwrite the remote changes 99 | by using `push -f`. 100 | 101 | > Note: If changes are being made on the remote, on a message that is currently being synced with `lieer`, the changes may be overwritten or merged in weird ways. 102 | 103 | See below for more [caveats](#caveats). 104 | 105 | ## Sending 106 | 107 | Lieer may be used as a simple stand-in for the `sendmail` MTA. A typical configuration for a MUA send command might be: 108 | 109 | ```sh 110 | gmi send -t -C ~/.mail/account.gmail 111 | ``` 112 | 113 | For example, you can enable git send-email with `git config sendemail.sendmailcmd "gmi send -t -C 114 | $HOME/.mail/account.gmail"`. 115 | 116 | Like the real sendmail program, the raw message is read from `stdin`. 117 | 118 | Most sendmail implementations allow passing additional recipients in additional 119 | arguments. However, the GMail API only supports the `-t` (`--read-recipients`) mode of 120 | sendmail, without additional recipients. 121 | 122 | We try to support valid combinations from MUAs that make use of recipients 123 | passed as arguments. Additional recipients are ignored, but validated. The 124 | following combinations are OK: 125 | 126 | - When `-t` is passed, we need to check for the CLI-passed recipients to be 127 | equal or a subset of the ones passed in the headers. 128 | 129 | - When `-t` is not passed, all header-passed recipients need to be provided in 130 | the CLI as well. 131 | 132 | This avoids silently not sending mail to some recipients (pretending we did), 133 | or sending mail to recipients we didn't want to send to again. 134 | 135 | One of the implication of `-t` is you have to keep `Bcc:` header in your 136 | message when passing it to `sendmail`. It is not enough to just put the 137 | additional recipient on the command line. For `mutt`, this means setting 138 | `write_bcc` option. 139 | 140 | Lieer will try to associate the sent message with the existing thread if it has 141 | an `In-Reply-To` header. According to the [Gmail 142 | API](https://developers.google.com/gmail/api/v1/reference/users/messages/send#request-body) 143 | the `Subject:` header must also match, but this does not seem to be necessary 144 | (at least not where just `Re:` has been prepended). 145 | 146 | > If the email address in the `From:` header does not match exactly the one of 147 | > your account, it seems like GMail resets the from to your account _address_ 148 | > only. 149 | 150 | Note that the following flags are ignored for `sendmail` compatibility: 151 | 152 | - `-f` (ignored, set envelope `From:` yourself) 153 | - `-o` (ignored) 154 | - `-i` (always implied, not bothered by single `.`'s) 155 | 156 | There are instructions for using this in your email client (for example Emacs) in the [wiki](https://github.com/gauteh/lieer/wiki/GNU-Emacs-and-Lieer). 157 | 158 | # Settings 159 | 160 | Lieer can be configured using `gmi set`. Use without any options to get a list of the current settings as well as the current history ID and notmuch revision. 161 | 162 | **`Account`** is the GMail account the repository is synced with. Configured during setup with [`gmi init`](#usage). 163 | 164 | **`historyId`** is the latest synced GMail revision. Anything since this ID will be fetched on the next [`gmi pull`](#pull) (partial). 165 | 166 | **`lastmod`** is the latest synced Notmuch database revision. Anything changed after this revision will be pushed on [`gmi push`](#ush). 167 | 168 | **`Timeout`** is the timeout in seconds used for the HTTP connection to GMail. `0` means the forever or system error/timeout, [whichever occurs first](https://github.com/gauteh/lieer/issues/83#issuecomment-396487919). 169 | 170 | **`File extension`** is an optional argument to include the specified extension in local file names (e.g., `mbox`) which can be useful for indexing them with third-party programs. 171 | 172 | *Important:* If you change this setting after synchronizing, the best case scenario is that all files will appear to not have being pulled down and will be re-downloaded (and duplicated with a different extension in the maildir). There might also be changes to tags. You should in theory be able to change it by renaming all files, but since this will update the lastmod you will get a check on all files. 173 | 174 | **`Drop non existing labels`** can be used to silently ignore errors where GMail gives us a label identifier which is not associated with a label. See [Caveats](#caveats). 175 | 176 | **`Replace slash with dot`** is used to replace the sub-label separator (`/`) with a dot (`.`). I think this is easier to work with. *Important*: See note below on [changing this setting after initial sync](#changing-ignored-tags-and-translation-after-initial-sync). 177 | 178 | **`Ignore tags (local)`** can be used to specify a list of tags which should not be synced from local to remote (e.g. [`new`](#usage)). In addition to the user-configured tags these tags are ignored: `'attachment', 'encrypted', 'signed', 'passed', 'replied', 'muted', 'mute', 'todo', 'Trash', 'voicemail'`. Some are special tags in notmuch and some are unsupported by GMail. See [Caveats](#caveats) below for more explanations. *Note:* This setting expects [_translated_ tags](#translation-between-labels-and-tags). 179 | 180 | *Important*: See note below on [changing this setting after initial sync](#changing-ignored-tags-and-translation-after-initial-sync). 181 | 182 | **`Ignore tags (remote)`** can be used to specify a list of tags (labels) which should not be synced from remote (GMail) to local. By default the [`CATEGORY_*` type](https://developers.google.com/gmail/api/guides/labels) labels which are mapped to the Personal/Promotions/etc tabs in the GMail interface are ignored. You can specify that no label should ignored by doing: `gmi set --ignore-tags-remote ""`. *Note:* This setting expects [_*un*translated_ tags](#translation-between-labels-and-tags). 183 | 184 | *Important*: See note below on [changing this setting after initial sync](#changing-ignored-tags-and-translation-after-initial-sync). 185 | 186 | **`Local Trash Tag (local)`** can be used to set the local tag to which the remote GMail 'TRASH' label is translated. 187 | 188 | *Important*: See note below on [changing this setting after initial sync](#changing-ignored-tags-and-translation-after-initial-sync). 189 | 190 | **`Translation List Overlay`** can be used to add or change entries in the translation mapping between local and remote tags. Argument is a comment-separated list with an even number of items. This is interpreted as a list of pairs of (remote, local), where each pair is added to the tag translation overwriting any existing translation for that tag if any. For example, 191 | `--translation-list-overlay CATEGORY_FORUMS,my_forum_tag` will translate Google's CATEGORY_FORUMS tag to my_forum_tag.') 192 | 193 | *Important*: See note below on [changing this setting after initial sync](#changing-ignored-tags-and-translation-after-initial-sync). 194 | 195 | ## Changing ignored tags and translation after initial sync 196 | 197 | If you change the [ignored tags](#settings) after the initial sync this will not update already synced messages. This means that if a change is made locally on an already synced message the previously ignored remote labels may be deleted. Conversely, if a change occurs remotely on a message which previously which has local tags that were ignored before, these ignored tags may be deleted. 198 | 199 | The best way to deal with this is to do a full push or pull after having changed one of the settings. **Do not change both `--ignore-tags-locally` and `--ignore-tags-remote` at the same time.** 200 | 201 | Before changing either setting make sure you are fully synchronized. After changing e.g. `--ignore-tags-remote` do first a dry-run and then a real run of full `gmi pull -f --dry-run`. This will fetch the full tag list for all messages and overwrite the local tags of all your messages with the remote labels. 202 | 203 | When changing the opposite setting: `--ignore-tags-local`, do a full push (dry-run first): `gmi push -f --dry-run`. 204 | 205 | The same goes for the options `--replace-slash-with-dot` and `--local-trash-tag`. I prefer to do `gmi pull -f --dry-run` after changing this option. This will overwrite the local tags with the remote labels. 206 | 207 | 208 | # Translation between labels and tags 209 | 210 | We translate some of the GMail labels to other tags. The default map of labels to tags are: 211 | 212 | ```py 213 | 'INBOX' : 'inbox', 214 | 'SPAM' : 'spam', 215 | 'TRASH' : 'trash', 216 | 'UNREAD' : 'unread', 217 | 'STARRED' : 'flagged', 218 | 'IMPORTANT' : 'important', 219 | 'SENT' : 'sent', 220 | 'DRAFT' : 'draft', 221 | 'CHAT' : 'chat', 222 | 223 | 'CATEGORY_PERSONAL' : 'personal', 224 | 'CATEGORY_SOCIAL' : 'social', 225 | 'CATEGORY_PROMOTIONS' : 'promotions', 226 | 'CATEGORY_UPDATES' : 'updates', 227 | 'CATEGORY_FORUMS' : 'forums', 228 | ``` 229 | 230 | The 'trash' local tag can be replaced using the `--local-trash-tag` option. 231 | 232 | # Using your own OAuth 2 Client 233 | 234 | Lieer ships with a set of OAuth 2 credentials that is shared openly, this shares API quota, but [cannot be used to access data](https://github.com/gauteh/lieer/pull/9) unless access is gained to your private `access_token` or `refresh_token`. 235 | 236 | The minor risk of using these public OAuth 2 credentials is that a malicious application running on your computer could pretend to be the lieer application, and gain access to your Gmail account. Since that already involves a malicious application running on your computer, it could likely just read the existing `.credentials.gmailieer.json` file directly, making this attack not particularly interesting. 237 | 238 | In any case, you can generate your own OAuth 2 credentials instead. This requires you have a Google account and access to Google Cloud Platform. 239 | 240 | 1. [Create a project on GCP](https://cloud.google.com/resource-manager/docs/creating-managing-projects), or go to an existing one. 241 | 2. [Enable access to the Gmail API](https://console.developers.google.com/flows/enableapi?apiid=gmail) for that GCP project 242 | 3. [Configure the OAuth consent screen](https://console.cloud.google.com/apis/credentials/consent), which involves naming your application (e.g. "Lieer Gmail Access") and setting other app-related metadata, most of which will be shown when you visit the OAuth 2 consent screen to give Lieer access to your Gmail account. 243 | - The important piece is to make sure to configure the `https://mail.google.com/` scope for your application, otherwise Lieer won't be able to access the Gmail API on your behalf. 244 | - You **don't** need to verify your application, you'll just get a scary warning about unverified applications when you go through the OAuth 2 flow, which you can safely ignore. 245 | 4. [Create the OAuth 2 credentials](https://console.cloud.google.com/apis/credentials) by selecting 'Create Credentials' > 'OAuth client ID'. 246 | - For `Application type`, select `Desktop app` 247 | - For `Name`, name it `Lieer` or `gmi client` or something you'll recognize 248 | 249 | ![A screenshot of the GCP OAuth 2 credential creation dialog](/docs/create-credentials.png) 250 | 251 | 5. [Download the credentials](https://console.cloud.google.com/auth/clients) by clicking the little download icon next to your newly created OAuth 2 client ID 252 | - This will download the `client_secret.json` file you need for Lieer. 253 | 254 | ![A screenshot of a mouse hovering over the "Download JSON" icon](/docs/download-json.png) 255 | 256 | Store the `client_secret.json` file somewhere safe and specify it to `gmi auth -c`. You can do this on a repository that is already initialized, possibly using `-f` to force reauthorizing with the new client secrets. 257 | 258 | # Privacy policy 259 | 260 | Lieer downloads e-mail and labels to your local computer. No data is sent elsewhere. 261 | 262 | Lieers use and transfer to any other app of information received from Google 263 | APIs will adhere to [Google API Services User Data Policy](https://developers.google.com/terms/api-services-user-data-policy#additional_requirements_for_specific_api_scopes), 264 | including the Limited Use requirements 265 | 266 | # Caveats 267 | 268 | * By default, notmuch adds the tags `inbox` and `unread` to all newly imported message. While setting `new.tags` in the notmuch config should change the initial tagging of messages, it seems as though `new.tags` can also be set in the notmuch database and this overrides the config file during Lieer pulls. To ensure `new.tags` are applied correctly, set them directly in the notmuch database using `notmuch config set --database new.tags ` in addition to your local config file. 269 | 270 | * The GMail API does not let you sync `muted` messages. Until [this Google 271 | bug](https://issuetracker.google.com/issues/36759067) is fixed, the `mute` and `muted` tags are not synchronized with the remote. 272 | 273 | * The [`todo`](https://github.com/gauteh/lieer/issues/52) and [`voicemail`](https://github.com/gauteh/lieer/issues/74) labels seem to be reserved and will be ignored. 274 | 275 | * The `draft` and `sent` labels are read only: They are synced from GMail to local notmuch tags, but not back (if you change them via notmuch). 276 | 277 | * [Only one of the tags](https://github.com/gauteh/lieer/issues/26) `inbox`, `spam`, and `trash` may be added to an email. For 278 | the time being, `trash` will be preferred over `spam`, and `spam` over `inbox`. 279 | 280 | * `Trash` (capital `T`) is reserved and not allowed, use `trash` (lowercase, see above) to bin messages remotely. 281 | 282 | * `archive` or `arxiv` are reserved and not allowed; see [issue/109](https://github.com/gauteh/lieer/issues/109) and [issue/171](https://github.com/gauteh/lieer/issues/171). To archive e-mails remove the `inbox` tag. 283 | 284 | * Sometimes GMail provides a label identifier on a message for a label that does not exist. If you encounter this [issue](https://github.com/gauteh/lieer/issues/48) you can get around it by using `gmi set --drop-non-existing-labels` and re-try to pull. The labels will now be ignored, and if this message is ever synced back up the unmapped label ID will be removed. You can list labels with `gmi pull -t`. 285 | 286 | * Sometimes GMail [indicates that there are more changes](https://github.com/gauteh/lieer/issues/120) when doing a partial pull, but an empty set is returned. The default is to fail, but you can ignore empty history by setting: `gmi set --ignore-empty-history`. 287 | 288 | * You [cannot add any new files](https://github.com/gauteh/lieer/issues/54) (files starting with `.` will be ignored) to the lieer repository. Lieer uses the directory content an index of local files. Lieer does not push new messages to your account (note that if you send messages with GMail, GMail automatically adds the message to your mailbox). 289 | 290 | * Make sure that you use the same domain for you GMail account as you initially created your account with: usually `@gmail.com`, but sometimes `@googlemail.com`. Otherwise you might get a [`Delegation denied` error](https://github.com/gauteh/lieer/issues/88). 291 | 292 | 293 | # Development 294 | 295 | Github actions are configured to check for python code formatted by [black](https://black.readthedocs.io/en/stable/integrations/github_actions.html). 296 | -------------------------------------------------------------------------------- /gmi: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # 3 | # Copyright © 2020 Gaute Hope 4 | # 5 | # This file is part of Lieer. 6 | # 7 | # Lieer is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | from lieer import Gmailieer 21 | 22 | if __name__ == '__main__': 23 | g = Gmailieer () 24 | g.main () 25 | 26 | -------------------------------------------------------------------------------- /lieer/__init__.py: -------------------------------------------------------------------------------- 1 | from .gmailieer import * 2 | -------------------------------------------------------------------------------- /lieer/gmailieer.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # 3 | # Copyright © 2020 Gaute Hope 4 | # Author: Gaute Hope / 2017-03-05 5 | # 6 | # This file is part of Lieer. 7 | # 8 | # Lieer is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # This program is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with this program. If not, see . 20 | 21 | import argparse 22 | import os 23 | import sys 24 | 25 | import googleapiclient 26 | import googleapiclient.errors 27 | import notmuch2 28 | 29 | from .local import Local 30 | from .remote import Remote 31 | 32 | 33 | class Gmailieer: 34 | cwd = None 35 | 36 | def main(self): 37 | parser = argparse.ArgumentParser("gmi") 38 | self.parser = parser 39 | 40 | common = argparse.ArgumentParser(add_help=False) 41 | common.add_argument("-C", "--path", type=str, default=None, help="path") 42 | 43 | common.add_argument( 44 | "-c", 45 | "--credentials", 46 | type=str, 47 | default=None, 48 | help="optional credentials file for google api", 49 | ) 50 | 51 | common.add_argument( 52 | "-s", 53 | "--no-progress", 54 | action="store_true", 55 | default=False, 56 | help="Disable progressbar (always off when output is not TTY)", 57 | ) 58 | 59 | common.add_argument( 60 | "-q", 61 | "--quiet", 62 | action="store_true", 63 | default=False, 64 | help="Produce less output (implies -s)", 65 | ) 66 | 67 | common.add_argument( 68 | "-v", 69 | "--verbose", 70 | action="store_true", 71 | default=False, 72 | help="print list of changes", 73 | ) 74 | 75 | subparsers = parser.add_subparsers(help="actions", dest="action") 76 | subparsers.required = True 77 | 78 | # pull 79 | parser_pull = subparsers.add_parser( 80 | "pull", 81 | help="pull new e-mail and remote tag-changes", 82 | description="pull", 83 | parents=[common], 84 | ) 85 | 86 | parser_pull.add_argument( 87 | "-t", 88 | "--list-labels", 89 | action="store_true", 90 | default=False, 91 | help="list all remote labels (pull)", 92 | ) 93 | 94 | parser_pull.add_argument( 95 | "--limit", 96 | type=int, 97 | default=None, 98 | help="Maximum number of messages to pull (soft limit, GMail may return more), note that this may upset the tally of synchronized messages.", 99 | ) 100 | 101 | parser_pull.add_argument( 102 | "-d", 103 | "--dry-run", 104 | action="store_true", 105 | default=False, 106 | help="do not make any changes", 107 | ) 108 | 109 | parser_pull.add_argument( 110 | "-f", 111 | "--force", 112 | action="store_true", 113 | default=False, 114 | help="Force a full synchronization to be performed", 115 | ) 116 | 117 | parser_pull.add_argument( 118 | "-r", 119 | "--resume", 120 | action="store_true", 121 | default=False, 122 | help="Resume previous incomplete synchronization if possible (this might cause local changes made in the interim to be ignored when pushing)", 123 | ) 124 | 125 | parser_pull.set_defaults(func=self.pull) 126 | 127 | # push 128 | parser_push = subparsers.add_parser( 129 | "push", parents=[common], description="push", help="push local tag-changes" 130 | ) 131 | 132 | parser_push.add_argument( 133 | "--limit", 134 | type=int, 135 | default=None, 136 | help="Maximum number of messages to push, note that this may upset the tally of synchronized messages.", 137 | ) 138 | 139 | parser_push.add_argument( 140 | "-d", 141 | "--dry-run", 142 | action="store_true", 143 | default=False, 144 | help="do not make any changes", 145 | ) 146 | 147 | parser_push.add_argument( 148 | "-f", 149 | "--force", 150 | action="store_true", 151 | default=False, 152 | help="Push even when there has been remote changes (might overwrite remote tag-changes)", 153 | ) 154 | 155 | parser_push.set_defaults(func=self.push) 156 | 157 | # send 158 | parser_send = subparsers.add_parser( 159 | "send", 160 | parents=[common], 161 | description="Read a MIME message from STDIN and send.", 162 | help="send a MIME message read from STDIN.", 163 | ) 164 | 165 | parser_send.add_argument( 166 | "-d", 167 | "--dry-run", 168 | action="store_true", 169 | default=False, 170 | help="do not actually send message", 171 | ) 172 | 173 | # Ignored arguments for sendmail compatibility 174 | if "-oi" in sys.argv: 175 | sys.argv.remove("-oi") 176 | 177 | if "-i" in sys.argv: 178 | sys.argv.remove("-i") 179 | 180 | parser_send.add_argument( 181 | "-i", 182 | action="store_true", 183 | default=None, 184 | help="Ignored: always implied, allowed for sendmail compatibility.", 185 | dest="i3", 186 | ) 187 | parser_send.add_argument( 188 | "-t", 189 | "--read-recipients", 190 | action="store_true", 191 | default=False, 192 | dest="read_recipients", 193 | help="Read recipients from message headers. This is always done by GMail. If this option is not specified, the same addresses (as those in the headers) must be specified as additional arguments.", 194 | ) 195 | 196 | parser_send.add_argument( 197 | "-f", 198 | type=str, 199 | help="Ignored: has no effect, allowed for sendmail compatibility.", 200 | dest="i1", 201 | ) 202 | 203 | parser_send.add_argument( 204 | "recipients", 205 | nargs="*", 206 | default=[], 207 | help="Recipients to send this message to (these are essentially ignored, but they are validated against the header fields.)", 208 | ) 209 | 210 | parser_send.set_defaults(func=self.send) 211 | 212 | # sync 213 | parser_sync = subparsers.add_parser( 214 | "sync", 215 | parents=[common], 216 | description="sync", 217 | help="sync changes (flags have same meaning as for push and pull)", 218 | ) 219 | 220 | parser_sync.add_argument( 221 | "--limit", 222 | type=int, 223 | default=None, 224 | help="Maximum number of messages to sync, note that this may upset the tally of synchronized messages.", 225 | ) 226 | 227 | parser_sync.add_argument( 228 | "-d", 229 | "--dry-run", 230 | action="store_true", 231 | default=False, 232 | help="do not make any changes", 233 | ) 234 | 235 | parser_sync.add_argument( 236 | "-f", 237 | "--force", 238 | action="store_true", 239 | default=False, 240 | help="Push even when there has been remote changes, and force a full remote-to-local synchronization", 241 | ) 242 | 243 | parser_sync.add_argument( 244 | "-r", 245 | "--resume", 246 | action="store_true", 247 | default=False, 248 | help="Resume previous incomplete synchronization if possible (this might cause local changes made in the interim to be ignored when pushing)", 249 | ) 250 | 251 | parser_sync.set_defaults(func=self.sync) 252 | 253 | # auth 254 | parser_auth = subparsers.add_parser( 255 | "auth", 256 | parents=[common], 257 | description="authorize", 258 | help="authorize lieer with your GMail account", 259 | ) 260 | 261 | parser_auth.add_argument( 262 | "-f", "--force", action="store_true", default=False, help="Re-authorize" 263 | ) 264 | 265 | # These are taken from oauth2lib/tools.py for compatibility with its 266 | # run_flow() method used during oauth 267 | parser_auth.add_argument( 268 | "--auth-host-name", 269 | default="localhost", 270 | help="Hostname when running a local web server", 271 | ) 272 | parser_auth.add_argument( 273 | "--auth-host-port", 274 | default=[8080, 8090], 275 | type=int, 276 | nargs="*", 277 | help="Port web server should listen on", 278 | ) 279 | parser_auth.add_argument( 280 | "--noauth_local_webserver", 281 | action="store_true", 282 | default=False, 283 | help="Do not run a local web server (no longer supported by Google)", 284 | ) 285 | 286 | parser_auth.set_defaults(func=self.authorize) 287 | 288 | # init 289 | parser_init = subparsers.add_parser( 290 | "init", 291 | parents=[common], 292 | description="initialize", 293 | help="initialize local e-mail repository and authorize", 294 | ) 295 | 296 | parser_init.add_argument( 297 | "--replace-slash-with-dot", 298 | action="store_true", 299 | default=False, 300 | help="This will replace '/' with '.' in gmail labels (make sure you realize the implications)", 301 | ) 302 | 303 | parser_init.add_argument( 304 | "--no-auth", 305 | action="store_true", 306 | default=False, 307 | help="Do not immediately authorize as well (you will need to run 'auth' afterwards)", 308 | ) 309 | 310 | parser_init.add_argument("account", type=str, help="GMail account to use") 311 | 312 | parser_init.set_defaults(func=self.initialize) 313 | 314 | # set option 315 | parser_set = subparsers.add_parser( 316 | "set", 317 | description="set option", 318 | parents=[common], 319 | help="set options for repository", 320 | ) 321 | 322 | parser_set.add_argument( 323 | "-t", 324 | "--timeout", 325 | type=float, 326 | default=None, 327 | help="Set HTTP timeout in seconds (0 means forever or system timeout)", 328 | ) 329 | 330 | parser_set.add_argument( 331 | "--replace-slash-with-dot", 332 | action="store_true", 333 | default=False, 334 | help="This will replace '/' with '.' in gmail labels (Important: see the manual and make sure you realize the implications)", 335 | ) 336 | 337 | parser_set.add_argument( 338 | "--no-replace-slash-with-dot", action="store_true", default=False 339 | ) 340 | 341 | parser_set.add_argument( 342 | "--drop-non-existing-labels", 343 | action="store_true", 344 | default=False, 345 | help="Allow missing labels on the GMail side to be dropped (see https://github.com/gauteh/lieer/issues/48)", 346 | ) 347 | 348 | parser_set.add_argument( 349 | "--no-drop-non-existing-labels", action="store_true", default=False 350 | ) 351 | 352 | parser_set.add_argument( 353 | "--ignore-empty-history", 354 | action="store_true", 355 | default=False, 356 | help="Sometimes GMail indicates more changes, but an empty set is returned (see https://github.com/gauteh/lieer/issues/120)", 357 | ) 358 | 359 | parser_set.add_argument( 360 | "--no-ignore-empty-history", action="store_true", default=False 361 | ) 362 | 363 | parser_set.add_argument( 364 | "--ignore-tags-local", 365 | type=str, 366 | default=None, 367 | help="Set custom tags to ignore when syncing from local to remote (comma-separated, after translations). Important: see the manual.", 368 | ) 369 | 370 | parser_set.add_argument( 371 | "--ignore-tags-remote", 372 | type=str, 373 | default=None, 374 | help="Set custom tags to ignore when syncing from remote to local (comma-separated, before translations). Important: see the manual.", 375 | ) 376 | 377 | parser_set.add_argument( 378 | "--file-extension", 379 | type=str, 380 | default=None, 381 | help='Add a file extension before the maildir status flags (e.g.: "mbox"). Important: see the manual about changing this setting after initial sync.', 382 | ) 383 | 384 | parser_set.add_argument( 385 | "--remove-local-messages", 386 | action="store_true", 387 | default=False, 388 | help="Remove messages that have been deleted on the remote (default is on)", 389 | ) 390 | parser_set.add_argument( 391 | "--no-remove-local-messages", 392 | action="store_true", 393 | default=False, 394 | help="Do not remove messages that have been deleted on the remote", 395 | ) 396 | 397 | parser_set.add_argument( 398 | "--local-trash-tag", 399 | type=str, 400 | default=None, 401 | help="The local tag to use for the remote label TRASH.", 402 | ) 403 | 404 | parser_set.add_argument( 405 | "--translation-list-overlay", 406 | type=str, 407 | default=None, 408 | help="A list with an even number of items representing a list of pairs of (remote, local), where each pair is added to the tag translation.", 409 | ) 410 | 411 | parser_set.set_defaults(func=self.set) 412 | 413 | args = parser.parse_args(sys.argv[1:]) 414 | self.args = args 415 | 416 | if args.quiet: 417 | args.no_progress = True 418 | 419 | args.func(args) 420 | 421 | def initialize(self, args): 422 | self.setup(args, False) 423 | self.local.initialize_repository(args.replace_slash_with_dot, args.account) 424 | 425 | if not args.no_auth: 426 | self.local.load_repository() 427 | self.remote = Remote(self) 428 | 429 | try: 430 | self.remote.authorize() 431 | except: 432 | print("") 433 | print("") 434 | print( 435 | "init: repository is set up, but authorization failed. re-run 'gmi auth' with proper parameters to complete authorization" 436 | ) 437 | print("") 438 | print("") 439 | print("") 440 | print("") 441 | raise 442 | 443 | def authorize(self, args): 444 | print("authorizing..") 445 | self.setup(args, False, True) 446 | self.remote.authorize(args.force) 447 | 448 | def setup(self, args, dry_run=False, load=False, block=False): 449 | global tqdm 450 | 451 | # common options 452 | if args.path is not None: 453 | self.vprint("path: %s" % args.path) 454 | if args.action == "init" and not os.path.exists(args.path): 455 | os.makedirs(args.path) 456 | 457 | args.path = os.path.expanduser(args.path) 458 | if os.path.isdir(args.path): 459 | self.cwd = os.getcwd() 460 | os.chdir(args.path) 461 | else: 462 | print("error: %s is not a valid path!" % args.path) 463 | raise NotADirectoryError("error: %s is not a valid path!" % args.path) 464 | 465 | self.dry_run = dry_run 466 | self.verbose = args.verbose 467 | self.HAS_TQDM = not args.no_progress 468 | self.credentials_file = args.credentials 469 | 470 | if self.HAS_TQDM: 471 | if not (sys.stderr.isatty() and sys.stdout.isatty()): 472 | self.HAS_TQDM = False 473 | else: 474 | try: 475 | from tqdm import tqdm 476 | 477 | self.HAS_TQDM = True 478 | except ImportError: 479 | self.HAS_TQDM = False 480 | 481 | if not self.HAS_TQDM: 482 | from .nobar import tqdm 483 | 484 | if self.dry_run: 485 | print("dry-run: ", self.dry_run) 486 | 487 | self.local = Local(self) 488 | if load: 489 | self.local.load_repository(block) 490 | self.remote = Remote(self) 491 | 492 | def sync(self, args): 493 | self.setup(args, args.dry_run, True) 494 | self.force = args.force 495 | self.limit = args.limit 496 | self.list_labels = False 497 | self.resume = args.resume 498 | 499 | self.remote.get_labels() 500 | 501 | # will try to push local changes, this operation should not make 502 | # any changes to the local store or any of the file names. 503 | self.push(args, True) 504 | 505 | # will pull in remote changes, overwriting local changes and effectively 506 | # resolving any conflicts. 507 | self.pull(args, True) 508 | 509 | def push(self, args, setup=False): 510 | if not setup: 511 | self.setup(args, args.dry_run, True) 512 | 513 | self.force = args.force 514 | self.limit = args.limit 515 | 516 | self.remote.get_labels() 517 | 518 | # loading local changes 519 | 520 | with notmuch2.Database() as db: 521 | rev = db.revision().rev 522 | if rev == self.local.state.lastmod: 523 | self.vprint("push: everything is up-to-date.") 524 | return 525 | 526 | qry = "path:%s/** and lastmod:%d..%d" % ( 527 | self.local.nm_relative, 528 | self.local.state.lastmod, 529 | rev, 530 | ) 531 | 532 | messages = [db.get(m.path) for m in db.messages(qry)] 533 | 534 | if self.limit is not None and len(messages) > self.limit: 535 | messages = messages[: self.limit] 536 | 537 | # get gids and filter out messages outside this repository 538 | messages, gids = self.local.messages_to_gids(messages) 539 | 540 | # get meta-data on changed messages from remote 541 | remote_messages = [] 542 | self.bar_create(leave=True, total=len(gids), desc="receiving metadata") 543 | 544 | def _got_msgs(ms): 545 | for m in ms: 546 | self.bar_update(1) 547 | remote_messages.append(m) 548 | 549 | self.remote.get_messages(gids, _got_msgs, "minimal") 550 | self.bar_close() 551 | 552 | # resolve changes 553 | self.bar_create(leave=True, total=len(gids), desc="resolving changes") 554 | actions = [] 555 | for rm, nm in zip(remote_messages, messages): 556 | actions.append( 557 | self.remote.update( 558 | rm, nm, self.local.state.last_historyId, self.force 559 | ) 560 | ) 561 | self.bar_update(1) 562 | 563 | self.bar_close() 564 | 565 | # remove no-ops 566 | actions = [a for a in actions if a] 567 | 568 | # limit 569 | if self.limit is not None and len(actions) >= self.limit: 570 | actions = actions[: self.limit] 571 | 572 | # push changes 573 | if len(actions) > 0: 574 | self.bar_create( 575 | leave=True, total=len(actions), desc="pushing, 0 changed" 576 | ) 577 | changed = 0 578 | 579 | def cb(_): 580 | nonlocal changed 581 | self.bar_update(1) 582 | changed += 1 583 | if not self.args.quiet and self.bar: 584 | self.bar.set_description("pushing, %d changed" % changed) 585 | 586 | self.remote.push_changes(actions, cb) 587 | 588 | self.bar_close() 589 | else: 590 | self.vprint("push: nothing to push") 591 | 592 | if not self.remote.all_updated: 593 | # will not set last_mod, this forces messages to be pushed again at next push 594 | print("push: not all changes could be pushed, will re-try at next push.") 595 | else: 596 | # TODO: Once we get more confident we might set the last history Id here to 597 | # avoid pulling back in the changes we just pushed. Currently there's a race 598 | # if something is modified remotely (new email, changed tags), so this might 599 | # not really be possible. 600 | pass 601 | 602 | if not self.dry_run and self.remote.all_updated: 603 | self.local.state.set_lastmod(rev) 604 | 605 | self.vprint( 606 | "remote historyId: %d" 607 | % self.remote.get_current_history_id(self.local.state.last_historyId) 608 | ) 609 | 610 | def pull(self, args, setup=False): 611 | if not setup: 612 | self.setup(args, args.dry_run, True) 613 | 614 | self.list_labels = args.list_labels 615 | self.force = args.force 616 | self.limit = args.limit 617 | self.resume = args.resume 618 | 619 | self.remote.get_labels() # to make sure label map is initialized 620 | 621 | if self.list_labels: 622 | for k, l in self.remote.labels.items(): 623 | print(f"{l: <30} {k}") 624 | return 625 | 626 | if self.force: 627 | self.vprint("pull: full synchronization (forced)") 628 | self.full_pull() 629 | 630 | elif self.local.state.last_historyId == 0: 631 | self.vprint( 632 | "pull: full synchronization (no previous synchronization state)" 633 | ) 634 | self.full_pull() 635 | 636 | else: 637 | self.vprint( 638 | "pull: partial synchronization.. (hid: %d)" 639 | % self.local.state.last_historyId 640 | ) 641 | self.partial_pull() 642 | 643 | def partial_pull(self): 644 | # get history 645 | bar = None 646 | history = [] 647 | last_id = self.remote.get_current_history_id(self.local.state.last_historyId) 648 | 649 | try: 650 | for hist in self.remote.get_history_since(self.local.state.last_historyId): 651 | history.extend(hist) 652 | 653 | if bar is None: 654 | self.bar_create(leave=True, desc="fetching changes") 655 | 656 | self.bar_update(len(hist)) 657 | 658 | if self.limit is not None and len(history) >= self.limit: 659 | break 660 | 661 | except googleapiclient.errors.HttpError as excep: 662 | if excep.resp.status == 404: 663 | print("pull: historyId is too old, full sync required.") 664 | self.full_pull() 665 | return 666 | else: 667 | raise 668 | 669 | except Remote.NoHistoryException: 670 | print("pull: failed, re-try in a bit.") 671 | raise 672 | 673 | finally: 674 | if bar is not None: 675 | self.bar_close() 676 | 677 | # figure out which changes need to be applied 678 | added_messages = [] # added messages, if they are later deleted they will be 679 | # removed from this list 680 | 681 | deleted_messages = [] # deleted messages, if they are later added they will be 682 | # removed from this list 683 | 684 | labels_changed = [] # list of messages which have had their label changed 685 | # the entry will be the last and most recent one in case 686 | # of multiple changes. if the message is either deleted 687 | # or added after the label change it will be removed from 688 | # this list. 689 | 690 | def remove_from_all(m): 691 | nonlocal added_messages, deleted_messages, labels_changed 692 | remove_from_list(deleted_messages, m) 693 | remove_from_list(labels_changed, m) 694 | remove_from_list(added_messages, m) 695 | 696 | def remove_from_list(lst, m): 697 | e = next((e for e in lst if e["id"] == m["id"]), None) 698 | if e is not None: 699 | lst.remove(e) 700 | return True 701 | return False 702 | 703 | if len(history) > 0: 704 | self.bar_create(total=len(history), leave=True, desc="resolving changes") 705 | else: 706 | bar = None 707 | 708 | for h in history: 709 | if "messagesAdded" in h: 710 | for m in h["messagesAdded"]: 711 | mm = m["message"] 712 | if not (set(mm.get("labelIds", [])) & self.remote.not_sync): 713 | remove_from_all(mm) 714 | added_messages.append(mm) 715 | 716 | if "messagesDeleted" in h: 717 | for m in h["messagesDeleted"]: 718 | mm = m["message"] 719 | # might silently fail to delete this 720 | remove_from_all(mm) 721 | if self.local.has(mm["id"]): 722 | deleted_messages.append(mm) 723 | 724 | # messages that are subsequently deleted by a later action will be removed 725 | # from either labels_changed or added_messages. 726 | if "labelsAdded" in h: 727 | for m in h["labelsAdded"]: 728 | mm = m["message"] 729 | if not (set(mm.get("labelIds", [])) & self.remote.not_sync): 730 | new = remove_from_list( 731 | added_messages, mm 732 | ) or not self.local.has(mm["id"]) 733 | remove_from_list(labels_changed, mm) 734 | if new: 735 | added_messages.append(mm) # needs to fetched 736 | else: 737 | labels_changed.append(mm) 738 | else: 739 | # in case a not_sync tag has been added to a scheduled message 740 | remove_from_list(added_messages, mm) 741 | remove_from_list(labels_changed, mm) 742 | 743 | if self.local.has(mm["id"]): 744 | remove_from_list(deleted_messages, mm) 745 | deleted_messages.append(mm) 746 | 747 | if "labelsRemoved" in h: 748 | for m in h["labelsRemoved"]: 749 | mm = m["message"] 750 | if not (set(mm.get("labelIds", [])) & self.remote.not_sync): 751 | new = remove_from_list( 752 | added_messages, mm 753 | ) or not self.local.has(mm["id"]) 754 | remove_from_list(labels_changed, mm) 755 | if new: 756 | added_messages.append(mm) # needs to fetched 757 | else: 758 | labels_changed.append(mm) 759 | else: 760 | # in case a not_sync tag has been added 761 | remove_from_list(added_messages, mm) 762 | remove_from_list(labels_changed, mm) 763 | 764 | if self.local.has(mm["id"]): 765 | remove_from_list(deleted_messages, mm) 766 | deleted_messages.append(mm) 767 | 768 | self.bar_update(1) 769 | 770 | if bar: 771 | self.bar_close() 772 | 773 | changed = False 774 | # fetching new messages 775 | if len(added_messages) > 0: 776 | message_gids = [m["id"] for m in added_messages] 777 | updated = self.get_content(message_gids) 778 | 779 | # updated labels for the messages that already existed 780 | needs_update_gid = list(set(message_gids) - set(updated)) 781 | needs_update = [m for m in added_messages if m["id"] in needs_update_gid] 782 | labels_changed.extend(needs_update) 783 | 784 | changed = True 785 | 786 | if self.local.config.remove_local_messages and len(deleted_messages) > 0: 787 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_WRITE) as db: 788 | for m in tqdm(deleted_messages, leave=True, desc="removing messages"): 789 | self.local.remove(m["id"], db) 790 | 791 | changed = True 792 | 793 | if len(labels_changed) > 0: 794 | lchanged = 0 795 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_WRITE) as db: 796 | self.bar_create( 797 | total=len(labels_changed), leave=True, desc="updating tags (0)" 798 | ) 799 | for m in labels_changed: 800 | r = self.local.update_tags(m, None, db) 801 | if r: 802 | lchanged += 1 803 | if not self.args.quiet and self.bar: 804 | self.bar.set_description("updating tags (%d)" % lchanged) 805 | 806 | self.bar_update(1) 807 | self.bar_close() 808 | 809 | changed = True 810 | 811 | if not changed: 812 | self.vprint("pull: everything is up-to-date.") 813 | 814 | if not self.dry_run: 815 | self.local.state.set_last_history_id(last_id) 816 | 817 | if last_id > 0: 818 | self.vprint("current historyId: %d" % last_id) 819 | 820 | def full_pull(self): 821 | total = 1 822 | 823 | self.bar_create(leave=True, total=total, desc="fetching messages") 824 | 825 | # NOTE: 826 | # this list might grow gigantic for large quantities of e-mail, not really sure 827 | # about how much memory this will take. this is just a list of some 828 | # simple metadata like message ids. 829 | message_gids = [] 830 | last_id = self.remote.get_current_history_id(self.local.state.last_historyId) 831 | 832 | resume_file = os.path.join(self.local.wd, ".resume-pull.gmailieer.json") 833 | 834 | if not self.resume: 835 | if os.path.exists(resume_file): 836 | self.vprint("pull: previous pull can be resumed using --resume") 837 | 838 | # continue filling up or create new resume-file 839 | previous = self.load_resume(resume_file, last_id) 840 | 841 | elif self.resume and not os.path.exists(resume_file): 842 | self.vprint( 843 | "pull: no previous resume file exists, continuing with full pull" 844 | ) 845 | previous = self.load_resume(resume_file, last_id) 846 | 847 | else: 848 | self.vprint("pull: attempting to resume previous pull..") 849 | assert self.resume 850 | previous = self.load_resume(resume_file, last_id) 851 | 852 | # check if lastid is still valid 853 | if not self.remote.is_history_id_valid(previous.lastId): 854 | self.vprint("pull: resume file too old, starting from scratch.") 855 | 856 | previous.delete() 857 | previous = self.load_resume(resume_file, last_id) 858 | 859 | for total, gids in self.remote.all_messages(): 860 | if not self.args.quiet and self.bar: 861 | self.bar.total = total 862 | self.bar_update(len(gids)) 863 | 864 | message_gids.extend(m["id"] for m in gids) 865 | 866 | if self.limit is not None and len(message_gids) >= self.limit: 867 | break 868 | 869 | self.bar_close() 870 | 871 | if self.local.config.remove_local_messages: 872 | if self.limit and not self.dry_run: 873 | raise ValueError( 874 | '--limit with "remove_local_messages" will cause lots of messages to be deleted' 875 | ) 876 | 877 | # removing files that have been deleted remotely 878 | all_remote = set(message_gids) 879 | all_local = set(self.local.gids.keys()) 880 | remove = list(all_local - all_remote) 881 | self.bar_create(leave=True, total=len(remove), desc="removing deleted") 882 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_WRITE) as db: 883 | for m in remove: 884 | self.local.remove(m, db) 885 | self.bar_update(1) 886 | 887 | self.bar_close() 888 | 889 | if len(message_gids) > 0: 890 | # get content for new messages 891 | updated = self.get_content(message_gids) 892 | 893 | # get updated labels for the rest 894 | needs_update = list(set(message_gids) - set(updated)) 895 | 896 | if self.resume: 897 | self.vprint( 898 | "pull: resume: skipping metadata for %d messages" 899 | % len(previous.meta_fetched) 900 | ) 901 | needs_update = list(set(needs_update) - set(previous.meta_fetched)) 902 | 903 | self.get_meta(needs_update, previous, self.resume) 904 | else: 905 | self.vprint("pull: no messages.") 906 | 907 | # set notmuch lastmod time, since we have now synced everything from remote 908 | # to local 909 | with notmuch2.Database() as db: 910 | rev = db.revision().rev 911 | 912 | if not self.dry_run: 913 | self.local.state.set_lastmod(rev) 914 | 915 | if self.resume: 916 | self.local.state.set_last_history_id(previous.lastId) 917 | else: 918 | self.local.state.set_last_history_id(last_id) 919 | 920 | self.vprint("pull: complete, removing resume file") 921 | previous.delete() 922 | 923 | self.vprint("current historyId: %d, current revision: %d" % (last_id, rev)) 924 | if self.resume: 925 | self.vprint("pull: resume: performing partial pull to complete") 926 | self.partial_pull() 927 | 928 | self.vprint( 929 | "pull: note that local changes made in the interim might be ignored in the next push" 930 | ) 931 | 932 | def get_meta(self, msgids, previous=None, resume=False): 933 | """ 934 | Only gets the minimal message objects in order to check if labels are up-to-date. 935 | 936 | `previous` and `resume` is passed by `full_pull` to track progress and resume previous metadata pull. 937 | """ 938 | 939 | if len(msgids) > 0: 940 | total = len(msgids) + len(previous.meta_fetched) if resume else len(msgids) 941 | 942 | self.bar_create(leave=True, total=total, desc="receiving metadata") 943 | 944 | if resume and previous is not None: 945 | self.bar_update(len(previous.meta_fetched)) 946 | 947 | # opening db for whole metadata sync 948 | def _got_msgs(ms): 949 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_WRITE) as db: 950 | for m in ms: 951 | self.bar_update(1) 952 | self.local.update_tags(m, None, db) 953 | 954 | if previous is not None: 955 | gids = [m["id"] for m in ms] 956 | previous.update(gids) 957 | 958 | self.remote.get_messages(msgids, _got_msgs, "minimal") 959 | 960 | self.bar_close() 961 | 962 | else: 963 | self.vprint("receiving metadata: everything up-to-date.") 964 | 965 | def get_content(self, msgids): 966 | """ 967 | Get the full email source of the messages that we do not already have 968 | 969 | Returns: 970 | list of messages which were updated, these have also been updated in Notmuch and 971 | does not need to be partially updated. 972 | 973 | """ 974 | 975 | need_content = [m for m in msgids if not self.local.has(m)] 976 | 977 | if len(need_content) > 0: 978 | self.bar_create( 979 | leave=True, total=len(need_content), desc="receiving content" 980 | ) 981 | 982 | def _got_msgs(ms): 983 | # opening db per message batch since it takes some time to download each one 984 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_WRITE) as db: 985 | for m in ms: 986 | self.bar_update(1) 987 | self.local.store(m, db) 988 | 989 | self.remote.get_messages(need_content, _got_msgs, "raw") 990 | 991 | self.bar_close() 992 | 993 | else: 994 | self.vprint("receiving content: everything up-to-date.") 995 | 996 | return need_content 997 | 998 | def load_resume(self, f, lastid): 999 | """ 1000 | Load a previous incomplete pull from resume file or create new resume file. 1001 | """ 1002 | from .resume import ResumePull 1003 | 1004 | if os.path.exists(f): 1005 | try: 1006 | return ResumePull.load(f) 1007 | except Exception as ex: 1008 | self.vprint("failed to load resume file, creating new: %s" % ex) 1009 | return ResumePull.new(f, lastid) 1010 | else: 1011 | return ResumePull.new(f, lastid) 1012 | 1013 | def send(self, args): 1014 | self.setup(args, args.dry_run, True, True) 1015 | self.remote.get_labels() 1016 | 1017 | msg = sys.stdin.buffer.read() 1018 | 1019 | # check if in-reply-to is set and find threadId 1020 | threadId = None 1021 | 1022 | import email 1023 | 1024 | eml = email.message_from_bytes(msg) 1025 | 1026 | # If there are recipients passed on the CLI, we need to compare them with 1027 | # what's in the message headers, as they need to match the message body 1028 | # (we can't express other recipients via the GMail API) 1029 | 1030 | cli_recipients = set(args.recipients) 1031 | 1032 | # construct existing recipient address list from To, Cc, Bcc headers 1033 | header_recipients = set() 1034 | for field_name in ("To", "Cc", "Bcc"): 1035 | # get all field values for the given field 1036 | field_values = eml.get_all(field_name, []) 1037 | 1038 | # parse these into a list of realnames and addresses 1039 | for _, address in email.utils.getaddresses(field_values): 1040 | header_recipients.add(address) 1041 | 1042 | if args.read_recipients: 1043 | if not header_recipients.issuperset(cli_recipients): 1044 | raise ValueError( 1045 | "Recipients passed via sendmail(1) arguments, but not part of message headers: {}".format( 1046 | ", ".join(cli_recipients.difference(header_recipients)) 1047 | ) 1048 | ) 1049 | elif header_recipients != cli_recipients: 1050 | raise ValueError( 1051 | "Recipients passed via sendmail(1) arguments ({}) differ from those in message headers ({}), perhaps you are missing the '-t' option?".format( 1052 | ", ".join(cli_recipients), ", ".join(header_recipients) 1053 | ) 1054 | ) 1055 | 1056 | self.vprint("sending message, from: %s.." % (eml.get("From"))) 1057 | 1058 | if "In-Reply-To" in eml: 1059 | repl = eml["In-Reply-To"].strip().strip("<>") 1060 | self.vprint("looking for original message: %s" % repl) 1061 | with notmuch2.Database(mode=notmuch2.Database.MODE.READ_ONLY) as db: 1062 | try: 1063 | nmsg = db.find(repl) 1064 | except LookupError: 1065 | nmsg = None 1066 | if nmsg is not None: 1067 | (_, gids) = self.local.messages_to_gids([nmsg]) 1068 | if nmsg.header("Subject") != eml["Subject"]: 1069 | self.vprint( 1070 | "warning: subject does not match, might not be able to associate with existing thread." 1071 | ) 1072 | 1073 | if len(gids) > 0: 1074 | gmsg = self.remote.get_message(gids[0]) 1075 | threadId = gmsg["threadId"] 1076 | self.vprint( 1077 | "found existing thread for new message: %s" % threadId 1078 | ) 1079 | else: 1080 | self.vprint( 1081 | "warning: could not find gid of parent message, sent message will not be associated in the same thread" 1082 | ) 1083 | else: 1084 | self.vprint( 1085 | "warning: could not find parent message, sent message will not be associated in the same thread" 1086 | ) 1087 | 1088 | if not args.dry_run: 1089 | msg = self.remote.send(msg, threadId) 1090 | self.get_content([msg["id"]]) 1091 | self.get_meta([msg["id"]]) 1092 | self.vprint("message sent successfully: %s" % msg["id"]) 1093 | else: 1094 | self.vprint("message sent successfully: dry-run") 1095 | 1096 | def set(self, args): 1097 | args.credentials = "" # for setup() 1098 | self.setup(args, False, True) 1099 | 1100 | if args.timeout is not None: 1101 | self.local.config.set_timeout(args.timeout) 1102 | 1103 | if args.replace_slash_with_dot: 1104 | self.local.config.set_replace_slash_with_dot(args.replace_slash_with_dot) 1105 | 1106 | if args.no_replace_slash_with_dot: 1107 | self.local.config.set_replace_slash_with_dot( 1108 | not args.no_replace_slash_with_dot 1109 | ) 1110 | 1111 | if args.drop_non_existing_labels: 1112 | self.local.config.set_drop_non_existing_label(args.drop_non_existing_labels) 1113 | 1114 | if args.no_drop_non_existing_labels: 1115 | self.local.config.set_drop_non_existing_label( 1116 | not args.no_drop_non_existing_labels 1117 | ) 1118 | 1119 | if args.ignore_empty_history: 1120 | self.local.config.set_ignore_empty_history(True) 1121 | 1122 | if args.no_ignore_empty_history: 1123 | self.local.config.set_ignore_empty_history(False) 1124 | 1125 | if args.remove_local_messages: 1126 | self.local.config.set_remove_local_messages(True) 1127 | 1128 | if args.no_remove_local_messages: 1129 | self.local.config.set_remove_local_messages(False) 1130 | 1131 | if args.ignore_tags_local is not None: 1132 | self.local.config.set_ignore_tags(args.ignore_tags_local) 1133 | 1134 | if args.ignore_tags_remote is not None: 1135 | self.local.config.set_ignore_remote_labels(args.ignore_tags_remote) 1136 | 1137 | if args.file_extension is not None: 1138 | self.local.config.set_file_extension(args.file_extension) 1139 | 1140 | if args.local_trash_tag is not None: 1141 | self.local.config.set_local_trash_tag(args.local_trash_tag) 1142 | 1143 | if args.translation_list_overlay is not None: 1144 | self.local.config.set_translation_list_overlay( 1145 | args.translation_list_overlay 1146 | ) 1147 | 1148 | print("Repository information and settings:") 1149 | print("Account ...........: %s" % self.local.config.account) 1150 | print("historyId .........: %d" % self.local.state.last_historyId) 1151 | print("lastmod ...........: %d" % self.local.state.lastmod) 1152 | print("Timeout ...........: %f" % self.local.config.timeout) 1153 | print("File extension ....: %s" % self.local.config.file_extension) 1154 | print("Remove local messages .....:", self.local.config.remove_local_messages) 1155 | print("Drop non existing labels...:", self.local.config.drop_non_existing_label) 1156 | print("Ignore empty history ......:", self.local.config.ignore_empty_history) 1157 | print("Replace . with / ..........:", self.local.config.replace_slash_with_dot) 1158 | print("Ignore tags (local) .......:", self.local.config.ignore_tags) 1159 | print("Ignore labels (remote) ....:", self.local.config.ignore_remote_labels) 1160 | print("Trash tag (local) .........:", self.local.config.local_trash_tag) 1161 | print( 1162 | "Translation list overlay ..:", self.local.config.translation_list_overlay 1163 | ) 1164 | 1165 | def vprint(self, *args, **kwargs): 1166 | """ 1167 | Print unless --quiet. 1168 | """ 1169 | if not self.args.quiet: 1170 | print(*args, **kwargs) 1171 | 1172 | def bar_create(self, leave=True, total=None, desc=""): 1173 | """ 1174 | Create progress bar. 1175 | """ 1176 | if not self.args.quiet: 1177 | self.bar = tqdm(leave=True, total=total, desc=desc) 1178 | 1179 | def bar_update(self, n): 1180 | """ 1181 | Update progress bar. 1182 | """ 1183 | if not self.args.quiet: 1184 | self.bar.update(n) 1185 | 1186 | def bar_close(self): 1187 | """ 1188 | Close progress bar. 1189 | """ 1190 | if not self.args.quiet: 1191 | self.bar.close() 1192 | -------------------------------------------------------------------------------- /lieer/local.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2020 Gaute Hope 2 | # 3 | # This file is part of Lieer. 4 | # 5 | # Lieer is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | import base64 19 | import fcntl 20 | import json 21 | import os 22 | import shutil 23 | import tempfile 24 | from pathlib import Path 25 | 26 | import notmuch2 27 | 28 | from .remote import Remote 29 | 30 | 31 | class Local: 32 | wd = None 33 | loaded = False 34 | 35 | # NOTE: Update README when changing this map. 36 | translate_labels_default = { 37 | "INBOX": "inbox", 38 | "SPAM": "spam", 39 | "TRASH": "trash", 40 | "UNREAD": "unread", 41 | "STARRED": "flagged", 42 | "IMPORTANT": "important", 43 | "SENT": "sent", 44 | "DRAFT": "draft", 45 | "CHAT": "chat", 46 | "CATEGORY_PERSONAL": "personal", 47 | "CATEGORY_SOCIAL": "social", 48 | "CATEGORY_PROMOTIONS": "promotions", 49 | "CATEGORY_UPDATES": "updates", 50 | "CATEGORY_FORUMS": "forums", 51 | } 52 | 53 | labels_translate_default = {v: k for k, v in translate_labels_default.items()} 54 | 55 | ignore_labels = { 56 | "archive", 57 | "arxiv", 58 | "attachment", 59 | "encrypted", 60 | "signed", 61 | "passed", 62 | "replied", 63 | "muted", 64 | "mute", 65 | "todo", 66 | "Trash", 67 | "voicemail", 68 | } 69 | 70 | def update_translation(self, remote, local): 71 | """ 72 | Convenience function to ensure both maps (remote -> local and local -> remote) 73 | get updated when you update a translation. 74 | """ 75 | # Did you reverse the parameters? 76 | assert remote in self.translate_labels 77 | self.translate_labels[remote] = local 78 | self.labels_translate = {v: k for k, v in self.translate_labels.items()} 79 | 80 | def update_translation_list_with_overlay(self, translation_list_overlay): 81 | """ 82 | Takes a list with an even number of items. The list is interpreted as a list of pairs 83 | of (remote, local), where each member of each pair is a string. Each pair is added to the 84 | translation, overwriting the translation if one already exists (in either direction). 85 | If either the remote or the local labels are non-unique, the later items in the list will 86 | overwrite the earlier ones in the direction in which the source is non-unique (for example, 87 | ["a", "1", "b", 2", "a", "3"] will yield {'a': 3, 'b': 2} in one direction and {1: 'a', 2: 'b', 3: 'a'} 88 | in the other). 89 | """ 90 | 91 | if len(translation_list_overlay) % 2 != 0: 92 | raise Exception( 93 | f"Translation list overlay must have an even number of items: {translation_list_overlay}" 94 | ) 95 | 96 | for i in range(0, len(translation_list_overlay), 2): 97 | (remote, local) = ( 98 | translation_list_overlay[i], 99 | translation_list_overlay[i + 1], 100 | ) 101 | self.translate_labels[remote] = local 102 | self.labels_translate[local] = remote 103 | 104 | class RepositoryException(Exception): 105 | pass 106 | 107 | class Config: 108 | replace_slash_with_dot = False 109 | account = None 110 | timeout = 10 * 60 111 | drop_non_existing_label = False 112 | ignore_empty_history = False 113 | ignore_tags = None 114 | ignore_remote_labels = None 115 | remove_local_messages = True 116 | file_extension = None 117 | local_trash_tag = "trash" 118 | translation_list_overlay = None 119 | 120 | def __init__(self, config_f): 121 | self.config_f = config_f 122 | 123 | if os.path.exists(self.config_f): 124 | try: 125 | with open(self.config_f) as fd: 126 | self.json = json.load(fd) 127 | except json.decoder.JSONDecodeError: 128 | print(f"Failed to decode config file `{self.config_f}`.") 129 | raise 130 | else: 131 | self.json = {} 132 | 133 | self.replace_slash_with_dot = self.json.get("replace_slash_with_dot", False) 134 | self.account = self.json.get("account", "me") 135 | self.timeout = self.json.get("timeout", 10 * 60) 136 | self.drop_non_existing_label = self.json.get( 137 | "drop_non_existing_label", False 138 | ) 139 | self.ignore_empty_history = self.json.get("ignore_empty_history", False) 140 | self.remove_local_messages = self.json.get("remove_local_messages", True) 141 | self.ignore_tags = set(self.json.get("ignore_tags", [])) 142 | self.ignore_remote_labels = set( 143 | self.json.get("ignore_remote_labels", Remote.DEFAULT_IGNORE_LABELS) 144 | ) 145 | self.file_extension = self.json.get("file_extension", "") 146 | self.local_trash_tag = self.json.get("local_trash_tag", "trash") 147 | self.translation_list_overlay = self.json.get( 148 | "translation_list_overlay", [] 149 | ) 150 | 151 | def write(self): 152 | self.json = {} 153 | 154 | self.json["replace_slash_with_dot"] = self.replace_slash_with_dot 155 | self.json["account"] = self.account 156 | self.json["timeout"] = self.timeout 157 | self.json["drop_non_existing_label"] = self.drop_non_existing_label 158 | self.json["ignore_empty_history"] = self.ignore_empty_history 159 | self.json["ignore_tags"] = list(self.ignore_tags) 160 | self.json["ignore_remote_labels"] = list(self.ignore_remote_labels) 161 | self.json["remove_local_messages"] = self.remove_local_messages 162 | self.json["file_extension"] = self.file_extension 163 | self.json["local_trash_tag"] = self.local_trash_tag 164 | self.json["translation_list_overlay"] = self.translation_list_overlay 165 | 166 | if os.path.exists(self.config_f): 167 | shutil.copyfile(self.config_f, self.config_f + ".bak") 168 | 169 | with tempfile.NamedTemporaryFile( 170 | mode="w+", dir=os.path.dirname(self.config_f), delete=False 171 | ) as fd: 172 | json.dump(self.json, fd) 173 | os.rename(fd.name, self.config_f) 174 | 175 | def set_account(self, a): 176 | self.account = a 177 | self.write() 178 | 179 | def set_timeout(self, t): 180 | self.timeout = t 181 | self.write() 182 | 183 | def set_replace_slash_with_dot(self, r): 184 | self.replace_slash_with_dot = r 185 | self.write() 186 | 187 | def set_drop_non_existing_label(self, r): 188 | self.drop_non_existing_label = r 189 | self.write() 190 | 191 | def set_ignore_empty_history(self, r): 192 | self.ignore_empty_history = r 193 | self.write() 194 | 195 | def set_remove_local_messages(self, r): 196 | self.remove_local_messages = r 197 | self.write() 198 | 199 | def set_ignore_tags(self, t): 200 | if len(t.strip()) == 0: 201 | self.ignore_tags = set() 202 | else: 203 | self.ignore_tags = {tt.strip() for tt in t.split(",")} 204 | 205 | self.write() 206 | 207 | def set_ignore_remote_labels(self, t): 208 | if len(t.strip()) == 0: 209 | self.ignore_remote_labels = set() 210 | else: 211 | self.ignore_remote_labels = {tt.strip() for tt in t.split(",")} 212 | 213 | self.write() 214 | 215 | def set_file_extension(self, t): 216 | try: 217 | with tempfile.NamedTemporaryFile( 218 | dir=os.path.dirname(self.config_f), suffix=t 219 | ) as _: 220 | pass 221 | 222 | self.file_extension = t.strip() 223 | self.write() 224 | except OSError: 225 | print( 226 | "Failed creating test file with file extension: " + t + ", not set." 227 | ) 228 | raise 229 | 230 | def set_local_trash_tag(self, t): 231 | if "," in t: 232 | print( 233 | "The local_trash_tag must be a single tag, not a list. Commas are not allowed." 234 | ) 235 | raise ValueError() 236 | self.local_trash_tag = t.strip() or "trash" 237 | self.write() 238 | 239 | def set_translation_list_overlay(self, t): 240 | if len(t.strip()) == 0: 241 | self.translation_list_overlay = [] 242 | else: 243 | self.translation_list_overlay = [tt.strip() for tt in t.split(",")] 244 | if len(self.translation_list_overlay) % 2 != 0: 245 | raise Exception( 246 | f"Translation list overlay must have an even number of items: {self.translation_list_overlay}" 247 | ) 248 | self.write() 249 | 250 | class State: 251 | # last historyid of last synchronized message, anything that has happened 252 | # remotely after this needs to be synchronized. gmail may return a 404 error 253 | # if the history records have been deleted, in which case we have to do a full 254 | # sync. 255 | last_historyId = 0 256 | 257 | # this is the last modification id of the notmuch db when the previous push was completed. 258 | lastmod = 0 259 | 260 | def __init__(self, state_f, config): 261 | self.state_f = state_f 262 | 263 | # True if config file contains state keys and should be migrated. 264 | # We will write both state and config after load if true. 265 | migrate_from_config = False 266 | 267 | if os.path.exists(self.state_f): 268 | try: 269 | with open(self.state_f) as fd: 270 | self.json = json.load(fd) 271 | except json.decoder.JSONDecodeError: 272 | print(f"Failed to decode state file `{self.state_f}`.") 273 | raise 274 | 275 | elif os.path.exists(config.config_f): 276 | try: 277 | with open(config.config_f) as fd: 278 | self.json = json.load(fd) 279 | except json.decoder.JSONDecodeError: 280 | print(f"Failed to decode config file `{config.config_f}`.") 281 | raise 282 | if any(k in self.json for k in ["last_historyId", "lastmod"]): 283 | migrate_from_config = True 284 | else: 285 | self.json = {} 286 | 287 | self.last_historyId = self.json.get("last_historyId", 0) 288 | self.lastmod = self.json.get("lastmod", 0) 289 | 290 | if migrate_from_config: 291 | self.write() 292 | config.write() 293 | 294 | def write(self): 295 | self.json = {} 296 | 297 | self.json["last_historyId"] = self.last_historyId 298 | self.json["lastmod"] = self.lastmod 299 | 300 | if os.path.exists(self.state_f): 301 | shutil.copyfile(self.state_f, self.state_f + ".bak") 302 | 303 | with tempfile.NamedTemporaryFile( 304 | mode="w+", dir=os.path.dirname(self.state_f), delete=False 305 | ) as fd: 306 | json.dump(self.json, fd) 307 | os.rename(fd.name, self.state_f) 308 | 309 | def set_last_history_id(self, hid): 310 | self.last_historyId = hid 311 | self.write() 312 | 313 | def set_lastmod(self, m): 314 | self.lastmod = m 315 | self.write() 316 | 317 | # we are in the class "Local"; this is the Local instance constructor 318 | def __init__(self, g): 319 | self.gmailieer = g 320 | self.wd = os.getcwd() 321 | self.dry_run = g.dry_run 322 | self.verbose = g.verbose 323 | 324 | # config and state files for local repository 325 | self.config_f = os.path.join(self.wd, ".gmailieer.json") 326 | self.state_f = os.path.join(self.wd, ".state.gmailieer.json") 327 | self.credentials_f = os.path.join(self.wd, ".credentials.gmailieer.json") 328 | 329 | # mail store 330 | self.md = os.path.join(self.wd, "mail") 331 | 332 | # initialize label translation instance variables 333 | self.translate_labels = Local.translate_labels_default.copy() 334 | self.labels_translate = Local.labels_translate_default.copy() 335 | 336 | def load_repository(self, block=False): 337 | """ 338 | Loads the current local repository 339 | 340 | block (boolean): if repository is in use, wait for lock to be freed (default: False) 341 | """ 342 | 343 | if not os.path.exists(self.config_f): 344 | raise Local.RepositoryException( 345 | "local repository not initialized: could not find config file" 346 | ) 347 | 348 | if any( 349 | not os.path.exists(os.path.join(self.md, mail_dir)) 350 | for mail_dir in ("cur", "new", "tmp") 351 | ): 352 | raise Local.RepositoryException( 353 | "local repository not initialized: could not find mail dir structure" 354 | ) 355 | 356 | ## Check if we are in the notmuch db 357 | with notmuch2.Database() as db: 358 | try: 359 | self.nm_relative = str(Path(self.md).relative_to(db.path)) 360 | except ValueError: 361 | raise Local.RepositoryException( 362 | "local mail repository not in notmuch db" 363 | ) 364 | self.nm_dir = str(Path(self.md).resolve()) 365 | 366 | ## Lock repository 367 | try: 368 | self.lckf = open(".lock", "w") # noqa: SIM115 369 | if block: 370 | fcntl.lockf(self.lckf, fcntl.LOCK_EX) 371 | else: 372 | fcntl.lockf(self.lckf, fcntl.LOCK_EX | fcntl.LOCK_NB) 373 | except OSError: 374 | raise Local.RepositoryException( 375 | "failed to lock repository (probably in use by another gmi instance)" 376 | ) 377 | 378 | self.config = Local.Config(self.config_f) 379 | self.state = Local.State(self.state_f, self.config) 380 | 381 | self.ignore_labels = self.ignore_labels | self.config.ignore_tags 382 | self.update_translation("TRASH", self.config.local_trash_tag) 383 | self.update_translation_list_with_overlay(self.config.translation_list_overlay) 384 | 385 | self.__load_cache__() 386 | 387 | # load notmuch config 388 | with notmuch2.Database() as db: 389 | self.new_tags = db.config.get("new.tags", "").split(";") 390 | self.new_tags = [t.strip() for t in self.new_tags if len(t.strip()) > 0] 391 | 392 | self.loaded = True 393 | 394 | def __load_cache__(self): 395 | ## The Cache: 396 | ## 397 | ## this cache is used to know which messages we have a physical copy of. 398 | ## hopefully this won't grow too gigantic with lots of messages. 399 | self.files = [] 400 | for _, _, fnames in os.walk(os.path.join(self.md, "cur")): 401 | _fnames = ("cur/" + f for f in fnames) 402 | self.files.extend(_fnames) 403 | break 404 | 405 | for _, _, fnames in os.walk(os.path.join(self.md, "new")): 406 | _fnames = ("new/" + f for f in fnames) 407 | self.files.extend(_fnames) 408 | break 409 | 410 | # exclude files that are unlikely to be real message files 411 | self.files = [f for f in self.files if os.path.basename(f)[0] != "."] 412 | 413 | self.gids = {} 414 | for f in self.files: 415 | m = self.__filename_to_gid__(os.path.basename(f)) 416 | self.gids[m] = f 417 | 418 | def initialize_repository(self, replace_slash_with_dot, account): 419 | """ 420 | Sets up a local repository 421 | """ 422 | print("initializing repository in: %s.." % self.wd) 423 | 424 | # check if there is a repository here already or if there is anything that will conflict with setting up one 425 | if os.path.exists(self.config_f): 426 | raise Local.RepositoryException( 427 | "'.gmailieer.json' exists: this repository seems to already be set up!" 428 | ) 429 | 430 | if os.path.exists(self.md): 431 | raise Local.RepositoryException( 432 | "'mail' exists: this repository seems to already be set up!" 433 | ) 434 | 435 | self.config = Local.Config(self.config_f) 436 | self.config.replace_slash_with_dot = replace_slash_with_dot 437 | self.config.account = account 438 | self.config.write() 439 | os.makedirs(os.path.join(self.md, "cur")) 440 | os.makedirs(os.path.join(self.md, "new")) 441 | os.makedirs(os.path.join(self.md, "tmp")) 442 | 443 | def has(self, m): 444 | """Check whether we have message id""" 445 | return m in self.gids 446 | 447 | def contains(self, fname): 448 | """Check whether message file exists is in repository""" 449 | return Path(self.md) in Path(fname).parents 450 | 451 | def __update_cache__(self, nmsg, old=None): 452 | """ 453 | Update cache with filenames from nmsg, removing the old: 454 | 455 | nmsg - notmuch2.Message 456 | old - tuple of old gid and old fname 457 | """ 458 | 459 | # remove old file from cache 460 | if old is not None: 461 | (old_gid, old_f) = old 462 | 463 | old_f = Path(old_f) 464 | self.files.remove(os.path.join(old_f.parent.name, old_f.name)) 465 | self.gids.pop(old_gid) 466 | 467 | # add message to cache 468 | fname_iter = nmsg.filenames() 469 | for _f in fname_iter: 470 | if self.contains(_f): 471 | new_f = Path(_f) 472 | 473 | # there might be more GIDs (and files) for each NotmuchMessage, if so, 474 | # the last matching file will be used in the gids map. 475 | 476 | _m = self.__filename_to_gid__(new_f.name) 477 | self.gids[_m] = os.path.join(new_f.parent.name, new_f.name) 478 | self.files.append(os.path.join(new_f.parent.name, new_f.name)) 479 | 480 | def messages_to_gids(self, msgs): 481 | """ 482 | Gets GIDs from a list of NotmuchMessages, the returned list of tuples may contain 483 | the same NotmuchMessage several times for each matching file. Files outside the 484 | repository are filtered out. 485 | """ 486 | gids = [] 487 | messages = [] 488 | 489 | for m in msgs: 490 | for fname in m.filenames(): 491 | if self.contains(fname): 492 | # get gmail id 493 | gid = self.__filename_to_gid__(os.path.basename(fname)) 494 | if gid: 495 | gids.append(gid) 496 | messages.append(m) 497 | 498 | return (messages, gids) 499 | 500 | def __filename_to_gid__(self, fname): 501 | ext = "" 502 | if self.config.file_extension: 503 | ext = "." + self.config.file_extension 504 | ext += ":2," 505 | 506 | f = fname.rfind(ext) 507 | if f > 5: 508 | return fname[:f] 509 | else: 510 | print( 511 | "'%s' does not contain valid maildir delimiter, correct file name extension, or does not seem to have a valid GID, ignoring." 512 | % fname 513 | ) 514 | return None 515 | 516 | def __make_maildir_name__(self, m, labels): 517 | # https://cr.yp.to/proto/maildir.html 518 | ext = "" 519 | if self.config.file_extension: 520 | ext = "." + self.config.file_extension 521 | 522 | p = m + ext + ":" 523 | info = "2," 524 | 525 | # must be ascii sorted 526 | if "DRAFT" in labels: 527 | info += "D" 528 | 529 | if "STARRED" in labels: 530 | info += "F" 531 | 532 | ## notmuch does not add 'T', so it will only be removed at the next 533 | ## maildir sync flags anyway. 534 | 535 | # if 'TRASH' in labels: 536 | # info += 'T' 537 | 538 | if "UNREAD" not in labels: 539 | info += "S" 540 | 541 | return p + info 542 | 543 | def remove(self, gid, db): 544 | """ 545 | Remove message from local store 546 | """ 547 | assert ( 548 | self.config.remove_local_messages 549 | ), "tried to remove message when 'remove_local_messages' was set to False" 550 | 551 | fname = self.gids.get(gid, None) 552 | ffname = fname 553 | 554 | if fname is None: 555 | print("remove: message does not exist in store: %s" % gid) 556 | return 557 | 558 | fname = os.path.join(self.md, fname) 559 | try: 560 | nmsg = db.get(fname) 561 | except LookupError: 562 | nmsg = None 563 | 564 | self.print_changes(f"deleting {gid}: {fname}.") 565 | 566 | if not self.dry_run: 567 | if nmsg is not None: 568 | db.remove(fname) 569 | os.unlink(fname) 570 | 571 | self.files.remove(ffname) 572 | self.gids.pop(gid) 573 | 574 | def store(self, m, db): 575 | """ 576 | Store message in local store 577 | """ 578 | 579 | gid = m["id"] 580 | msg_str = base64.urlsafe_b64decode(m["raw"].encode("ASCII")) 581 | 582 | # messages from GMail have windows line endings 583 | if os.linesep == "\n": 584 | msg_str = msg_str.replace(b"\r\n", b"\n") 585 | 586 | labels = m.get("labelIds", []) 587 | 588 | bname = self.__make_maildir_name__(gid, labels) 589 | 590 | # add to cache 591 | self.files.append(os.path.join("cur", bname)) 592 | self.gids[gid] = os.path.join("cur", bname) 593 | 594 | p = os.path.join(self.md, "cur", bname) 595 | tmp_p = os.path.join(self.md, "tmp", bname) 596 | 597 | if os.path.exists(p): 598 | raise Local.RepositoryException("local file already exists: %s" % p) 599 | 600 | if os.path.exists(tmp_p): 601 | raise Local.RepositoryException( 602 | "local temporary file already exists: %s" % tmp_p 603 | ) 604 | 605 | if not self.dry_run: 606 | with open(tmp_p, "wb") as fd: 607 | fd.write(msg_str) 608 | 609 | # Set atime and mtime of the message file to Gmail receive date 610 | internalDate = int(m["internalDate"]) / 1000 # ms to s 611 | os.utime(tmp_p, (internalDate, internalDate)) 612 | 613 | os.rename(tmp_p, p) 614 | 615 | # add to notmuch 616 | self.update_tags(m, p, db) 617 | 618 | def update_tags(self, m, fname, db): 619 | # make sure notmuch tags reflect gmail labels 620 | gid = m["id"] 621 | glabels = m.get("labelIds", []) 622 | 623 | # translate labels. Remote.get_labels () must have been called first 624 | labels = [] 625 | for l in glabels: 626 | ll = self.gmailieer.remote.labels.get(l, None) 627 | 628 | if ll is None and not self.config.drop_non_existing_label: 629 | err = "error: GMail supplied a label that there exists no record for! You can `gmi set --drop-non-existing-labels` to work around the issue (https://github.com/gauteh/lieer/issues/48)" 630 | print(err) 631 | raise Local.RepositoryException(err) 632 | elif ll is None: 633 | pass # drop 634 | else: 635 | labels.append(ll) 636 | 637 | # remove ignored labels 638 | labels = set(labels) 639 | labels = list(labels - self.gmailieer.remote.ignore_labels) 640 | 641 | # translate to notmuch tags 642 | labels = [self.translate_labels.get(l, l) for l in labels] 643 | 644 | # this is my weirdness 645 | if self.config.replace_slash_with_dot: 646 | labels = [l.replace("/", ".") for l in labels] 647 | 648 | if fname is None: 649 | # this file hopefully already exists and just needs it tags updated, 650 | # let's try to find its name in the gid to fname table. 651 | fname = os.path.join(self.md, self.gids[gid]) 652 | 653 | else: 654 | # new file 655 | fname = os.path.join(self.md, "cur", fname) 656 | 657 | if not os.path.exists(fname): 658 | if not self.dry_run: 659 | print( 660 | "missing file: reloading cache to check for changes..", 661 | end="", 662 | flush=True, 663 | ) 664 | self.__load_cache__() 665 | fname = os.path.join(self.md, self.gids[gid]) 666 | print("done.") 667 | 668 | if not os.path.exists(fname): 669 | raise Local.RepositoryException( 670 | "tried to update tags on non-existent file: %s" % fname 671 | ) 672 | 673 | self.print_changes("tried to update tags on non-existent file: %s" % fname) 674 | 675 | try: 676 | nmsg = db.get(fname) 677 | except LookupError: 678 | nmsg = None 679 | 680 | if nmsg is None: 681 | self.print_changes(f"adding message: {gid}: {fname}, with tags: {labels}") 682 | if not self.dry_run: 683 | try: 684 | (nmsg, _) = db.add(fname, sync_flags=True) 685 | except notmuch2.FileNotEmailError: 686 | print("%s is not an email" % fname) 687 | return True 688 | 689 | # adding initial tags 690 | with nmsg.frozen(): 691 | for t in labels: 692 | nmsg.tags.add(t) 693 | 694 | for t in self.new_tags: 695 | nmsg.tags.add(t) 696 | 697 | nmsg.tags.to_maildir_flags() 698 | self.__update_cache__(nmsg) 699 | 700 | return True 701 | 702 | else: 703 | # message is already in db, set local tags to match remote tags 704 | otags = nmsg.tags 705 | igntags = otags & self.ignore_labels 706 | otags = otags - self.ignore_labels # remove ignored tags while checking 707 | if otags != set(labels): 708 | labels.extend(igntags) # add back local ignored tags before adding 709 | if not self.dry_run: 710 | with nmsg.frozen(): 711 | nmsg.tags.clear() 712 | for t in labels: 713 | nmsg.tags.add(t) 714 | nmsg.tags.to_maildir_flags() 715 | self.__update_cache__(nmsg, (gid, fname)) 716 | 717 | self.print_changes( 718 | f"changing tags on message: {gid} from: {str(otags)} to: {str(labels)}" 719 | ) 720 | 721 | return True 722 | else: 723 | return False 724 | 725 | def print_changes(self, changes): 726 | if self.dry_run: 727 | print("(dry-run) " + changes) 728 | elif self.verbose: 729 | print(changes) 730 | -------------------------------------------------------------------------------- /lieer/nobar.py: -------------------------------------------------------------------------------- 1 | # Regular non-TTY drop-in replacement for tqdm 2 | # 3 | # Copyright © 2020 Gaute Hope 4 | # 5 | # This file is part of Lieer. 6 | # 7 | # Lieer is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | 20 | import time 21 | from math import floor 22 | 23 | 24 | class tqdm: 25 | def __init__(self, iterable=None, leave=True, total=None, desc="", *args, **kwargs): 26 | self.desc = desc 27 | self.args = args 28 | self.kwargs = kwargs 29 | 30 | if total is not None: 31 | print(desc, "(%d)" % total, "...", end="", flush=True) 32 | else: 33 | print(desc, "...", end="", flush=True) 34 | 35 | self.start = time.perf_counter() 36 | self.it = 0 37 | 38 | if iterable is not None: 39 | self.iterable = (i for i in iterable) 40 | 41 | def __next__(self): 42 | if self.iterable is not None: 43 | self.update(1) 44 | 45 | try: 46 | return next(self.iterable) 47 | except StopIteration: 48 | self.close() 49 | raise 50 | else: 51 | raise StopIteration 52 | 53 | def __iter__(self): 54 | return self 55 | 56 | def update(self, n, *args): 57 | self.it += n 58 | 59 | INTERVAL = 10 60 | 61 | if self.it % INTERVAL == 0: 62 | print(".", end="", flush=True) 63 | 64 | def set_description(self, *args, **kwargs): 65 | pass 66 | 67 | def close(self): 68 | self.end = time.perf_counter() 69 | print("done:", self.it, "its in", self.pp_duration(self.end - self.start)) 70 | 71 | def pp_duration(self, d=None): 72 | dys = floor(d / (24 * 60 * 60)) 73 | d = d - (dys * 24 * 60 * 60) 74 | 75 | h = floor(d / (60 * 60)) 76 | d = d - (h * 60 * 60) 77 | 78 | m = floor(d / 60) 79 | d = d - (m * 60) 80 | 81 | s = d 82 | 83 | o = "" 84 | above = False 85 | if dys > 0: 86 | o = "%dd-" % dys 87 | above = True 88 | 89 | if above or h > 0: 90 | o = o + "%02dh:" % h 91 | above = True 92 | 93 | if above or m > 0: 94 | o = o + "%02dm:" % m 95 | above = True 96 | 97 | o = o + "%06.3fs" % s 98 | 99 | return o 100 | -------------------------------------------------------------------------------- /lieer/remote.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2020 Gaute Hope 2 | # 3 | # This file is part of Lieer. 4 | # 5 | # Lieer is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | import json 19 | import os 20 | import time 21 | 22 | import googleapiclient 23 | from apiclient import discovery 24 | from google.auth.transport.requests import Request 25 | from google.oauth2.credentials import Credentials 26 | from google_auth_oauthlib.flow import InstalledAppFlow 27 | 28 | 29 | class Remote: 30 | SCOPES = [ 31 | "https://www.googleapis.com/auth/gmail.readonly", 32 | "https://www.googleapis.com/auth/gmail.labels", 33 | "https://www.googleapis.com/auth/gmail.modify", 34 | ] 35 | APPLICATION_NAME = "Lieer" 36 | CLIENT_SECRET_FILE = None 37 | authorized = False 38 | 39 | # nothing to see here, move along.. 40 | # 41 | # no seriously: this is not dangerous to keep here, in order to gain 42 | # access to an users account the access_token and/or refresh_token must be 43 | # compromised. these are stored locally. 44 | # 45 | # * https://github.com/gauteh/lieer/pull/9 46 | # * https://stackoverflow.com/questions/25957027/oauth-2-installed-application-client-secret-considerationsgoogle-api/43061998#43061998 47 | # * https://stackoverflow.com/questions/19615372/client-secret-in-oauth-2-0?rq=1 48 | # 49 | OAUTH2_CLIENT_SECRET = { 50 | "client_id": "753933720722-ju82fu305lii0v9rdo6mf9hj40l5juv0.apps.googleusercontent.com", 51 | "project_id": "capable-pixel-160614", 52 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 53 | "token_uri": "https://accounts.google.com/o/oauth2/token", 54 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 55 | "client_secret": "8oudEG0Tvb7YI2V0ykp2Pzz9", 56 | "redirect_uris": ["urn:ietf:wg:oauth:2.0:oob", "http://localhost"], 57 | } 58 | 59 | # Not used, here for documentation purposes 60 | special_labels = [ 61 | "INBOX", 62 | "SPAM", 63 | "TRASH", 64 | "UNREAD", 65 | "STARRED", 66 | "IMPORTANT", 67 | "SENT", 68 | "DRAFT", 69 | "CHAT", 70 | "CATEGORY_PERSONAL", 71 | "CATEGORY_SOCIAL", 72 | "CATEGORY_PROMOTIONS", 73 | "CATEGORY_UPDATES", 74 | "CATEGORY_FORUMS", 75 | ] 76 | 77 | # these cannot be changed manually 78 | read_only_labels = {"SENT", "DRAFT"} 79 | read_only_tags = {"sent", "draft"} 80 | 81 | DEFAULT_IGNORE_LABELS = [ 82 | "CATEGORY_PERSONAL", 83 | "CATEGORY_SOCIAL", 84 | "CATEGORY_PROMOTIONS", 85 | "CATEGORY_UPDATES", 86 | "CATEGORY_FORUMS", 87 | ] 88 | 89 | ignore_labels = set() 90 | 91 | # query to use 92 | query = "-in:chats" 93 | 94 | not_sync = {"CHAT"} 95 | 96 | # used to indicate whether all messages that should be updated where updated 97 | all_updated = True 98 | 99 | # Handle exponential back-offs in non-batch requests. 100 | _delay = 0 101 | _delay_ok = 0 102 | MAX_DELAY = 100 103 | MAX_CONNECTION_ERRORS = 20 104 | 105 | ## Batch requests should generally be of size 50, and at most 100. Best overall 106 | ## performance is likely to be at 50 since we will not be throttled. 107 | ## 108 | ## * https://developers.google.com/gmail/api/guides/batch 109 | ## * https://developers.google.com/gmail/api/v1/reference/quota 110 | BATCH_REQUEST_SIZE = 50 111 | MIN_BATCH_REQUEST_SIZE = 1 112 | 113 | class BatchException(Exception): 114 | pass 115 | 116 | class UserRateException(Exception): 117 | pass 118 | 119 | class GenericException(Exception): 120 | pass 121 | 122 | class NoHistoryException(Exception): 123 | pass 124 | 125 | def __init__(self, g): 126 | self.gmailieer = g 127 | 128 | assert g.local.loaded, "local repository must be loaded!" 129 | 130 | self.CLIENT_SECRET_FILE = g.credentials_file 131 | self.account = g.local.config.account 132 | self.dry_run = g.dry_run 133 | self.verbose = g.verbose 134 | 135 | self.ignore_labels = self.gmailieer.local.config.ignore_remote_labels 136 | 137 | def __require_auth__(func): 138 | def func_wrap(self, *args, **kwargs): 139 | if not self.authorized: 140 | self.authorize() 141 | return func(self, *args, **kwargs) 142 | 143 | return func_wrap 144 | 145 | def __wait_delay__(self): 146 | if self._delay: 147 | time.sleep(self._delay) 148 | 149 | def __request_done__(self, success): 150 | if success: 151 | if self._delay: 152 | if self._delay_ok > 10: 153 | # after 10 good requests, reduce request delay 154 | self._delay = self._delay // 2 155 | self._delay_ok = 0 156 | else: 157 | self._delay_ok += 1 158 | else: 159 | self._delay = self._delay * 2 + 1 160 | self._delay_ok = 0 161 | if self._delay <= self.MAX_DELAY: 162 | print( 163 | "remote: request failed, increasing delay between requests to: %d s" 164 | % self._delay 165 | ) 166 | else: 167 | print( 168 | "remote: increased delay to more than maximum of %d s." 169 | % self.MAX_DELAY 170 | ) 171 | raise Remote.GenericException( 172 | "cannot increase delay more to more than maximum %d s" 173 | % self.MAX_DELAY 174 | ) 175 | 176 | @__require_auth__ 177 | def get_labels(self): 178 | results = self.service.users().labels().list(userId=self.account).execute() 179 | labels = results.get("labels", []) 180 | 181 | self.labels = {} 182 | self.invlabels = {} 183 | for l in labels: 184 | self.labels[l["id"]] = l["name"] 185 | self.invlabels[l["name"]] = l["id"] 186 | 187 | return self.labels 188 | 189 | @__require_auth__ 190 | def get_current_history_id(self, start): 191 | """ 192 | Get the current history id of the mailbox 193 | """ 194 | try: 195 | results = ( 196 | self.service.users() 197 | .history() 198 | .list(userId=self.account, startHistoryId=start) 199 | .execute() 200 | ) 201 | if "historyId" in results: 202 | return int(results["historyId"]) 203 | else: 204 | raise Remote.GenericException("no historyId field returned") 205 | 206 | except googleapiclient.errors.HttpError: 207 | # this happens if the original historyId is too old, 208 | # try to get last message and the historyId from it. 209 | for mset in self.all_messages(1): 210 | (total, mset) = mset 211 | m = mset[0] 212 | msg = self.get_message(m["id"]) 213 | return int(msg["historyId"]) 214 | 215 | @__require_auth__ 216 | def is_history_id_valid(self, historyId): 217 | """ 218 | Check if the historyId is valid or too old. 219 | """ 220 | try: 221 | results = ( 222 | self.service.users() 223 | .history() 224 | .list(userId=self.account, startHistoryId=historyId) 225 | .execute() 226 | ) 227 | if "historyId" in results: 228 | return True 229 | else: 230 | raise Remote.GenericException("no historyId field returned") 231 | 232 | except googleapiclient.errors.HttpError: 233 | return False 234 | 235 | @__require_auth__ 236 | def get_history_since(self, start): 237 | """ 238 | Get all changes since start historyId 239 | """ 240 | self.__wait_delay__() 241 | results = ( 242 | self.service.users() 243 | .history() 244 | .list(userId=self.account, startHistoryId=start) 245 | .execute() 246 | ) 247 | if "history" in results: 248 | self.__request_done__(True) 249 | yield results["history"] 250 | 251 | # no history field means that there is no history 252 | 253 | while "nextPageToken" in results: 254 | pt = results["nextPageToken"] 255 | 256 | self.__wait_delay__() 257 | _results = ( 258 | self.service.users() 259 | .history() 260 | .list(userId=self.account, startHistoryId=start, pageToken=pt) 261 | .execute() 262 | ) 263 | 264 | if "history" in _results: 265 | self.__request_done__(True) 266 | results = _results 267 | yield results["history"] 268 | else: 269 | print("remote: no 'history' when more pages were indicated.") 270 | if not self.gmailieer.local.config.ignore_empty_history: 271 | self.__request_done__(False) 272 | print( 273 | "You can ignore this error with: gmi set --ignore-empty-history (https://github.com/gauteh/lieer/issues/120)" 274 | ) 275 | raise Remote.NoHistoryException() 276 | else: 277 | self.__request_done__(True) 278 | 279 | @__require_auth__ 280 | def all_messages(self, limit=None): 281 | """ 282 | Get a list of all messages 283 | """ 284 | 285 | self.__wait_delay__() 286 | results = ( 287 | self.service.users() 288 | .messages() 289 | .list( 290 | userId=self.account, 291 | q=self.query, 292 | maxResults=limit, 293 | includeSpamTrash=True, 294 | ) 295 | .execute() 296 | ) 297 | 298 | if "messages" in results: 299 | self.__request_done__(True) 300 | yield (results["resultSizeEstimate"], results["messages"]) 301 | 302 | # no messages field presumably means no messages 303 | 304 | while "nextPageToken" in results: 305 | pt = results["nextPageToken"] 306 | _results = ( 307 | self.service.users() 308 | .messages() 309 | .list( 310 | userId=self.account, 311 | pageToken=pt, 312 | q=self.query, 313 | maxResults=limit, 314 | includeSpamTrash=True, 315 | ) 316 | .execute() 317 | ) 318 | 319 | if "messages" in _results: 320 | self.__request_done__(True) 321 | results = _results 322 | yield (results["resultSizeEstimate"], results["messages"]) 323 | else: 324 | self.__request_done__(True) 325 | print("remote: warning: no messages when several pages were indicated.") 326 | break 327 | 328 | @__require_auth__ 329 | def get_messages(self, gids, cb, format): 330 | """ 331 | Get the messages 332 | """ 333 | 334 | max_req = self.BATCH_REQUEST_SIZE 335 | req_ok = 0 336 | N = len(gids) 337 | i = 0 338 | j = 0 339 | 340 | # How much to wait before contacting the remote. 341 | user_rate_delay = 0 342 | # How many requests with the current delay returned ok. 343 | user_rate_ok = 0 344 | 345 | conn_errors = 0 346 | 347 | msg_batch = ( 348 | [] 349 | ) # queue up received batch and send in one go to content / db routine 350 | 351 | def _cb(rid, resp, excep): 352 | nonlocal j, msg_batch 353 | if excep is not None: 354 | if ( 355 | type(excep) is googleapiclient.errors.HttpError 356 | and excep.resp.status == 404 357 | ): 358 | # message could not be found this is probably a deleted message, spam or draft 359 | # message since these are not included in the messages.get() query by default. 360 | print("remote: could not find remote message: %s!" % gids[j]) 361 | j += 1 362 | return 363 | 364 | elif ( 365 | type(excep) is googleapiclient.errors.HttpError 366 | and excep.resp.status == 400 367 | ): 368 | # message id invalid, probably caused by stray files in the mail repo 369 | print( 370 | "remote: message id: %s is invalid! are there any non-lieer files created in the lieer repository?" 371 | % gids[j] 372 | ) 373 | j += 1 374 | return 375 | 376 | elif ( 377 | type(excep) is googleapiclient.errors.HttpError 378 | and excep.resp.status == 403 379 | ): 380 | raise Remote.UserRateException(excep) 381 | 382 | else: 383 | raise Remote.BatchException(excep) 384 | else: 385 | j += 1 386 | 387 | msg_batch.append(resp) 388 | 389 | while i < N: 390 | n = 0 391 | j = i 392 | batch = self.service.new_batch_http_request(callback=_cb) 393 | 394 | while n < max_req and i < N: 395 | gid = gids[i] 396 | batch.add( 397 | self.service.users() 398 | .messages() 399 | .get(userId=self.account, id=gid, format=format) 400 | ) 401 | n += 1 402 | i += 1 403 | 404 | # we wait if there is a user_rate_delay 405 | if user_rate_delay: 406 | print("remote: waiting %.1f seconds.." % user_rate_delay) 407 | time.sleep(user_rate_delay) 408 | 409 | try: 410 | batch.execute() 411 | 412 | # gradually reduce user delay upon every ok batch 413 | user_rate_ok += 1 414 | if user_rate_delay > 0 and user_rate_ok > 0: 415 | user_rate_delay = user_rate_delay // 2 416 | print("remote: decreasing delay to %s" % user_rate_delay) 417 | user_rate_ok = 0 418 | 419 | # gradually increase batch request size upon every ok request 420 | req_ok += 1 421 | if max_req < self.BATCH_REQUEST_SIZE and req_ok > 0: 422 | max_req = min(max_req * 2, self.BATCH_REQUEST_SIZE) 423 | print("remote: increasing batch request size to: %d" % max_req) 424 | req_ok = 0 425 | 426 | conn_errors = 0 427 | 428 | except Remote.UserRateException: 429 | user_rate_delay = user_rate_delay * 2 + 1 430 | print( 431 | "remote: user rate error, increasing delay to %s" % user_rate_delay 432 | ) 433 | user_rate_ok = 0 434 | 435 | i = j # reset 436 | 437 | except Remote.BatchException: 438 | max_req = max_req // 2 439 | req_ok = 0 440 | 441 | if max_req >= self.MIN_BATCH_REQUEST_SIZE: 442 | i = j # reset 443 | print("remote: reducing batch request size to: %d" % max_req) 444 | else: 445 | max_req = self.MIN_BATCH_REQUEST_SIZE 446 | raise Remote.BatchException("cannot reduce request any further") 447 | 448 | except ConnectionError as ex: 449 | print("connection failed, re-trying:", ex) 450 | i = j # reset 451 | conn_errors += 1 452 | 453 | if conn_errors > self.MAX_CONNECTION_ERRORS: 454 | print("too many connection errors") 455 | raise 456 | 457 | time.sleep(1) 458 | 459 | finally: 460 | # handle batch 461 | if len(msg_batch) > 0: 462 | cb(msg_batch) 463 | msg_batch.clear() 464 | 465 | @__require_auth__ 466 | def get_message(self, gid, format="minimal"): 467 | """ 468 | Get a single message 469 | """ 470 | self.__wait_delay__() 471 | try: 472 | result = ( 473 | self.service.users() 474 | .messages() 475 | .get(userId=self.account, id=gid, format=format) 476 | .execute() 477 | ) 478 | 479 | except googleapiclient.errors.HttpError as excep: 480 | if excep.resp.status == 403 or excep.resp.status == 500: 481 | self.__request_done__(False) 482 | return self.get_message(gid, format) 483 | else: 484 | raise 485 | 486 | self.__request_done__(True) 487 | 488 | return result 489 | 490 | def authorize(self, reauth=False): 491 | if reauth: 492 | credential_path = self.gmailieer.local.credentials_f 493 | if os.path.exists(credential_path): 494 | print("reauthorizing..") 495 | os.unlink(credential_path) 496 | 497 | self.credentials = self.__get_credentials__() 498 | 499 | timeout = self.gmailieer.local.config.timeout 500 | if timeout == 0: 501 | timeout = None 502 | 503 | self.service = discovery.build("gmail", "v1", credentials=self.credentials) 504 | self.authorized = True 505 | 506 | def __store_credentials__(self, path, credentials): 507 | """ 508 | Store valid credentials in json format 509 | """ 510 | with open(path + ".new", "w") as storage: 511 | try: 512 | storage.write(credentials.to_json()) 513 | except AttributeError: 514 | storage.write( 515 | json.dumps( 516 | { 517 | "token": credentials.token, 518 | "refresh_token": credentials.refresh_token, 519 | "token_uri": credentials.token_uri, 520 | "client_id": credentials.client_id, 521 | "client_secret": credentials.client_secret, 522 | "scopes": credentials.scopes, 523 | } 524 | ) 525 | ) 526 | os.rename(path + ".new", path) 527 | 528 | def __get_credentials__(self): 529 | """ 530 | Gets valid user credentials from storage. 531 | 532 | If nothing has been stored, or if the stored credentials are invalid, 533 | the OAuth2 flow is completed to obtain the new credentials. 534 | 535 | Returns: 536 | Credentials, the obtained credential. 537 | """ 538 | credentials = None 539 | credential_path = self.gmailieer.local.credentials_f 540 | 541 | if os.path.exists(credential_path): 542 | credentials = Credentials.from_authorized_user_file( 543 | credential_path, self.SCOPES 544 | ) 545 | 546 | if not credentials or not credentials.valid: 547 | if ( 548 | credentials 549 | and (credentials.expired or not credentials.token) 550 | and credentials.refresh_token 551 | ): 552 | credentials.refresh(Request()) 553 | 554 | elif self.CLIENT_SECRET_FILE is not None: 555 | # use user-provided client_secret 556 | print("auth: using user-provided api id and secret") 557 | if not os.path.exists(self.CLIENT_SECRET_FILE): 558 | raise Remote.GenericException( 559 | "error: no secret client API key file found for authentication at: %s" 560 | % self.CLIENT_SECRET_FILE 561 | ) 562 | 563 | flow = InstalledAppFlow.from_client_secrets_file( 564 | self.CLIENT_SECRET_FILE, self.SCOPES 565 | ) 566 | credentials = flow.run_local_server() 567 | self.__store_credentials__(credential_path, credentials) 568 | 569 | else: 570 | # use default id and secret 571 | client_config = { 572 | "installed": { 573 | "auth_uri": self.OAUTH2_CLIENT_SECRET["auth_uri"], 574 | "token_uri": self.OAUTH2_CLIENT_SECRET["token_uri"], 575 | "client_id": self.OAUTH2_CLIENT_SECRET["client_id"], 576 | "client_secret": self.OAUTH2_CLIENT_SECRET["client_secret"], 577 | } 578 | } 579 | flow = InstalledAppFlow.from_client_config(client_config, self.SCOPES) 580 | credentials = flow.run_local_server() 581 | self.__store_credentials__(credential_path, credentials) 582 | 583 | return credentials 584 | 585 | @__require_auth__ 586 | def update(self, gmsg, nmsg, last_hist, force): 587 | """ 588 | Gets a message and checks which labels it should add and which to delete, returns a 589 | operation which can be submitted in a batch. 590 | """ 591 | 592 | # DUPLICATES: 593 | # 594 | # there might be duplicate messages across gmail accounts with the same 595 | # message id, messages outside the repository are skipped. if there are 596 | # duplicate messages in the same account they are all updated. if one of 597 | # them is changed remotely it will not be updated, any changes on it will 598 | # then be pulled back on next pull overwriting the changes that might have 599 | # been pushed on another duplicate. this will again trigger a change on the 600 | # next push for the other duplicates. after the 2nd pull things should 601 | # settle unless there's been any local changes. 602 | # 603 | 604 | gid = gmsg["id"] 605 | 606 | found = False 607 | for f in nmsg.filenames(): 608 | if gid in str(f): 609 | found = True 610 | 611 | # this can happen if a draft is edited remotely and is synced before it is sent. we'll 612 | # just skip it and it should be resolved on the next pull. 613 | if not found: 614 | print( 615 | "update: gid does not match any file name of message, probably a draft, skipping: %s" 616 | % gid 617 | ) 618 | return None 619 | 620 | glabels = gmsg.get("labelIds", []) 621 | 622 | # translate labels. Remote.get_labels () must have been called first 623 | labels = [] 624 | for l in glabels: 625 | ll = self.labels.get(l, None) 626 | 627 | if ll is None and not self.gmailieer.local.config.drop_non_existing_label: 628 | err = "error: GMail supplied a label that there exists no record for! You can `gmi set --drop-non-existing-labels` to work around the issue (https://github.com/gauteh/lieer/issues/48)" 629 | print(err) 630 | raise Remote.GenericException(err) 631 | elif ll is None: 632 | pass # drop 633 | else: 634 | labels.append(ll) 635 | 636 | # remove ignored labels 637 | labels = set(labels) 638 | labels = labels - self.ignore_labels 639 | 640 | # translate to notmuch tags 641 | labels = [self.gmailieer.local.translate_labels.get(l, l) for l in labels] 642 | 643 | # this is my weirdness 644 | if self.gmailieer.local.config.replace_slash_with_dot: 645 | labels = [l.replace("/", ".") for l in labels] 646 | 647 | labels = set(labels) 648 | 649 | # current tags 650 | tags = nmsg.tags 651 | 652 | # remove special notmuch tags 653 | tags = tags - self.gmailieer.local.ignore_labels 654 | 655 | add = list((tags - labels) - self.read_only_tags) 656 | rem = list((labels - tags) - self.read_only_tags) 657 | 658 | # translate back to gmail labels 659 | add = [self.gmailieer.local.labels_translate.get(k, k) for k in add] 660 | rem = [self.gmailieer.local.labels_translate.get(k, k) for k in rem] 661 | 662 | if self.gmailieer.local.config.replace_slash_with_dot: 663 | add = [a.replace(".", "/") for a in add] 664 | rem = [r.replace(".", "/") for r in rem] 665 | 666 | if len(add) > 0 or len(rem) > 0: 667 | # check if this message has been changed remotely since last pull 668 | hist_id = int(gmsg["historyId"]) 669 | if hist_id > last_hist and not force: 670 | print( 671 | "update: remote has changed, will not update: %s (add: %s, rem: %s) (%d > %d)" 672 | % (gid, add, rem, hist_id, last_hist) 673 | ) 674 | self.all_updated = False 675 | return None 676 | 677 | if "TRASH" in add: 678 | if "SPAM" in add: 679 | print( 680 | f"update: {gid}: Trying to add both TRASH and SPAM, dropping SPAM (add: {add}, rem: {rem})" 681 | ) 682 | add.remove("SPAM") 683 | if "INBOX" in add: 684 | print( 685 | f"update: {gid}: Trying to add both TRASH and INBOX, dropping INBOX (add: {add}, rem: {rem})" 686 | ) 687 | add.remove("INBOX") 688 | elif "SPAM" in add: 689 | if "INBOX" in add: 690 | print( 691 | "update: {gid}: Trying to add both SPAM and INBOX, dropping INBOX (add: {add}, rem: {rem})" 692 | ) 693 | add.remove("INBOX") 694 | 695 | self.print_changes(f"gid: {gid}: add: {str(add)}, remove: {str(rem)}") 696 | if self.dry_run: 697 | return None 698 | else: 699 | return self.__push_tags__(gid, add, rem) 700 | 701 | else: 702 | return None 703 | 704 | @__require_auth__ 705 | def __push_tags__(self, gid, add, rem): 706 | """ 707 | Push message changes 708 | """ 709 | 710 | _add = [] 711 | for a in add: 712 | _a = self.invlabels.get(a, None) 713 | if _a is None: 714 | # label does not exist 715 | (lid, ll) = self.__create_label__(a) 716 | self.labels[lid] = ll 717 | self.invlabels[ll] = lid 718 | _add.append(lid) 719 | else: 720 | _add.append(_a) 721 | 722 | _rem = [self.invlabels[r] for r in rem] 723 | 724 | body = {"addLabelIds": _add, "removeLabelIds": _rem} 725 | 726 | return ( 727 | self.service.users() 728 | .messages() 729 | .modify(userId=self.account, id=gid, body=body) 730 | ) 731 | 732 | @__require_auth__ 733 | def push_changes(self, actions, cb): 734 | """ 735 | Push label changes 736 | """ 737 | max_req = self.BATCH_REQUEST_SIZE 738 | N = len(actions) 739 | i = 0 740 | j = 0 741 | 742 | # How much to wait before contacting the remote. 743 | user_rate_delay = 0 744 | # How many requests with the current delay returned ok. 745 | user_rate_ok = 0 746 | 747 | def _cb(rid, resp, excep): 748 | nonlocal j 749 | if excep is not None: 750 | if ( 751 | type(excep) is googleapiclient.errors.HttpError 752 | and excep.resp.status == 404 753 | ): 754 | # message could not be found this is probably a deleted message, spam or draft 755 | # message since these are not included in the messages.get() query by default. 756 | print("remote: could not find remote message: %s!" % resp) 757 | j += 1 758 | return 759 | 760 | elif ( 761 | type(excep) is googleapiclient.errors.HttpError 762 | and excep.resp.status == 400 763 | ): 764 | # message id invalid, probably caused by stray files in the mail repo 765 | print( 766 | "remote: message id is invalid! are there any non-lieer files created in the lieer repository? %s" 767 | % resp 768 | ) 769 | j += 1 770 | return 771 | 772 | elif ( 773 | type(excep) is googleapiclient.errors.HttpError 774 | and excep.resp.status == 403 775 | ): 776 | raise Remote.UserRateException(excep) 777 | 778 | else: 779 | raise Remote.BatchException(excep) 780 | else: 781 | j += 1 782 | 783 | cb(resp) 784 | 785 | while i < N: 786 | n = 0 787 | j = i 788 | batch = self.service.new_batch_http_request(callback=_cb) 789 | 790 | while n < max_req and i < N: 791 | a = actions[i] 792 | batch.add(a) 793 | n += 1 794 | i += 1 795 | 796 | # we wait if there is a user_rate_delay 797 | if user_rate_delay: 798 | print("remote: waiting %.1f seconds.." % user_rate_delay) 799 | time.sleep(user_rate_delay) 800 | 801 | try: 802 | batch.execute() 803 | 804 | # gradually reduce if we had 10 ok batches 805 | user_rate_ok += 1 806 | if user_rate_ok > 10: 807 | user_rate_delay = user_rate_delay // 2 808 | user_rate_ok = 0 809 | 810 | except Remote.UserRateException: 811 | user_rate_delay = user_rate_delay * 2 + 1 812 | print( 813 | "remote: user rate error, increasing delay to %s" % user_rate_delay 814 | ) 815 | user_rate_ok = 0 816 | 817 | i = j # reset 818 | 819 | except Remote.BatchException: 820 | if max_req > self.MIN_BATCH_REQUEST_SIZE: 821 | max_req = max_req / 2 822 | i = j # reset 823 | print("reducing batch request size to: %d" % max_req) 824 | else: 825 | raise Remote.BatchException("cannot reduce request any further") 826 | 827 | @__require_auth__ 828 | def __create_label__(self, l): 829 | """ 830 | Creates a new label 831 | 832 | Returns: 833 | 834 | (labelId, label) 835 | 836 | """ 837 | 838 | print("push: creating label: %s.." % l) 839 | 840 | label = { 841 | "messageListVisibility": "show", 842 | "name": l, 843 | "labelListVisibility": "labelShow", 844 | } 845 | 846 | if not self.dry_run: 847 | self.__wait_delay__() 848 | try: 849 | lr = ( 850 | self.service.users() 851 | .labels() 852 | .create(userId=self.account, body=label) 853 | .execute() 854 | ) 855 | 856 | return (lr["id"], l) 857 | 858 | except googleapiclient.errors.HttpError as excep: 859 | if excep.resp.status == 403 or excep.resp.status == 500: 860 | self.__request_done__(False) 861 | return self.__create_label__(l) 862 | else: 863 | raise 864 | 865 | self.__request_done__(True) 866 | 867 | else: 868 | return (None, None) 869 | 870 | @__require_auth__ 871 | def send(self, message, threadId=None): 872 | """ 873 | Send message 874 | 875 | message: MIME message as bytes 876 | 877 | Returns: 878 | 879 | Message 880 | """ 881 | import base64 882 | 883 | message = {"raw": base64.urlsafe_b64encode(message).decode()} 884 | 885 | if threadId is not None: 886 | message["threadId"] = threadId 887 | 888 | return ( 889 | self.service.users() 890 | .messages() 891 | .send(userId=self.account, body=message) 892 | .execute() 893 | ) 894 | 895 | def print_changes(self, changes): 896 | if self.dry_run: 897 | print("(dry-run) " + changes) 898 | elif self.verbose: 899 | print(changes) 900 | -------------------------------------------------------------------------------- /lieer/resume.py: -------------------------------------------------------------------------------- 1 | # Copyright © 2020 Gaute Hope 2 | # 3 | # This file is part of Lieer. 4 | # 5 | # Lieer is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | import json 19 | import os 20 | import tempfile 21 | 22 | 23 | class ResumePull: 24 | lastId = None 25 | version = None 26 | VERSION = 1 27 | 28 | meta_fetched = None 29 | 30 | @staticmethod 31 | def load(resume_file): 32 | """ 33 | Construct from existing resume 34 | """ 35 | with open(resume_file) as fd: 36 | j = json.load(fd) 37 | 38 | version = j["version"] 39 | if version != ResumePull.VERSION: 40 | print( 41 | "error: mismatching version in resume file: %d != %d" 42 | % (version, ResumePull.VERSION) 43 | ) 44 | raise ValueError() 45 | 46 | lastId = j["lastId"] 47 | meta_fetched = j["meta_fetched"] 48 | 49 | r = ResumePull(resume_file, lastId) 50 | r.meta_fetched = meta_fetched 51 | 52 | return r 53 | 54 | @staticmethod 55 | def new(resume_file, lastId): 56 | r = ResumePull(resume_file, lastId) 57 | r.meta_fetched = [] 58 | r.save() 59 | 60 | return r 61 | 62 | def __init__(self, resume_file, lastId): 63 | self.resume_file = resume_file 64 | self.lastId = lastId 65 | self.meta_fetched = [] 66 | 67 | def update(self, fetched): 68 | """ 69 | fetched: new messages with metadata fetched 70 | """ 71 | self.meta_fetched.extend(fetched) 72 | self.meta_fetched = list(set(self.meta_fetched)) 73 | self.save() 74 | 75 | def save(self): 76 | j = { 77 | "version": self.VERSION, 78 | "lastId": self.lastId, 79 | "meta_fetched": self.meta_fetched, 80 | } 81 | 82 | with tempfile.NamedTemporaryFile( 83 | mode="w+", dir=os.path.dirname(self.resume_file), delete=False 84 | ) as fd: 85 | json.dump(j, fd) 86 | 87 | if os.path.exists(self.resume_file): 88 | os.rename(self.resume_file, self.resume_file + ".bak") 89 | 90 | os.rename(fd.name, self.resume_file) 91 | 92 | def delete(self): 93 | os.unlink(self.resume_file) 94 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client 2 | google_auth_oauthlib 3 | tqdm 4 | notmuch2 5 | setuptools 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """A setuptools based setup module. 2 | 3 | See: 4 | https://packaging.python.org/en/latest/distributing.html 5 | https://github.com/pypa/sampleproject 6 | """ 7 | 8 | # To use a consistent encoding 9 | from codecs import open 10 | from os import path 11 | 12 | # Always prefer setuptools over distutils 13 | from setuptools import find_packages, setup 14 | 15 | here = path.abspath(path.dirname(__file__)) 16 | 17 | # Get the long description from the README file 18 | with open(path.join(here, "README.md"), encoding="utf-8") as f: 19 | long_description = f.read() 20 | 21 | setup( 22 | name="lieer", 23 | # Versions should comply with PEP440. For a discussion on single-sourcing 24 | # the version across setup.py and the project code, see 25 | # https://packaging.python.org/en/latest/single_source_version.html 26 | version="1.6", 27 | description="Fast fetch and two-way tag synchronization between notmuch and GMail", 28 | long_description=long_description, 29 | long_description_content_type="text/markdown", 30 | # The project's main homepage. 31 | url="https://github.com/gauteh/lieer", 32 | # Author details 33 | author="Gaute Hope", 34 | author_email="eg@gaute.vetsj.com", 35 | # Choose your license 36 | license="GPLv3+", 37 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 38 | classifiers=[ 39 | # How mature is this project? Common values are 40 | # 3 - Alpha 41 | # 4 - Beta 42 | # 5 - Production/Stable 43 | "Development Status :: 4 - Beta", 44 | # Indicate who your project is intended for 45 | # Specify the Python versions you support here. In particular, ensure 46 | # that you indicate whether you support Python 2, Python 3 or both. 47 | "Programming Language :: Python :: 3", 48 | ], 49 | # What does your project relate to? 50 | keywords="gmail notmuch synchronization tags", 51 | # You can just specify the packages manually here if your project is 52 | # simple. Or you can use find_packages(). 53 | packages=find_packages(exclude=["tests"]), 54 | # Alternatively, if you want to distribute just a my_module.py, uncomment 55 | # this: 56 | # py_modules=["my_module"], 57 | # List run-time dependencies here. These will be installed by pip when 58 | # your project is installed. For an analysis of "install_requires" vs pip's 59 | # requirements files see: 60 | # https://packaging.python.org/en/latest/requirements.html 61 | install_requires=[ 62 | "google_auth_oauthlib", 63 | "google-api-python-client", 64 | "tqdm", 65 | "notmuch2", 66 | ], 67 | # List additional groups of dependencies here (e.g. development 68 | # dependencies). You can install these using the following syntax, 69 | # for example: 70 | # $ pip install -e .[dev,test] 71 | extras_require={ 72 | # 'dev': ['check-manifest'], 73 | # 'test': ['coverage'], 74 | }, 75 | # If there are data files included in your packages that need to be 76 | # installed, specify them here. If using Python 2.6 or less, then these 77 | # have to be included in MANIFEST.in as well. 78 | package_data={ 79 | # 'sample': ['package_data.dat'], 80 | }, 81 | # Although 'package_data' is the preferred approach, in some case you may 82 | # need to place data files outside of your packages. See: 83 | # https://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa 84 | # In this case, 'data_file' will be installed into '/my_data' 85 | # data_files=[('my_data', ['data/data_file'])], 86 | # To provide executable scripts, use entry points in preference to the 87 | # "scripts" keyword. Entry points provide cross-platform support and allow 88 | # pip to create the appropriate form of executable for the target platform. 89 | # entry_points={ 90 | # 'console_scripts': [ 91 | # 'sample=sample:main', 92 | # ], 93 | # }, 94 | scripts=[ 95 | "gmi", 96 | ], 97 | ) 98 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gauteh/lieer/5c9ca43d3bc040bdb012dbfc1ca4cf15b1451dca/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | class MockGmi: 5 | dry_run = False 6 | verbose = False 7 | 8 | def __init__(self): 9 | pass 10 | 11 | 12 | @pytest.fixture 13 | def gmi(): 14 | """ 15 | Test gmi 16 | """ 17 | 18 | return MockGmi() 19 | -------------------------------------------------------------------------------- /tests/test_local.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import lieer 4 | 5 | 6 | def test_update_translation_list(gmi): 7 | l = lieer.Local(gmi) 8 | l.update_translation_list_with_overlay(["a", "1", "b", "2"]) 9 | assert l.translate_labels["a"] == "1" 10 | assert l.translate_labels["b"] == "2" 11 | assert l.labels_translate["1"] == "a" 12 | assert l.labels_translate["2"] == "b" 13 | 14 | with pytest.raises(Exception): 15 | l.update_translation_list_with_overlay(["a", "1", "b", "2", "c"]) 16 | --------------------------------------------------------------------------------