├── img
    ├── logo.png
    ├── settings.png
    └── logo-small.png
├── app.py
├── semiphemeral
    ├── static
    │   ├── img
    │   │   └── loading.gif
    │   ├── js
    │   │   ├── base.js
    │   │   ├── settings.js
    │   │   └── tweets.js
    │   └── style.css
    ├── templates
    │   ├── tweets.html
    │   ├── base.html
    │   └── settings.html
    ├── import_export.py
    ├── settings.py
    ├── __init__.py
    ├── db.py
    ├── common.py
    ├── web.py
    └── twitter.py
├── Pipfile
├── scripts
    └── pypi_release.sh
├── LICENSE
├── setup.py
├── CHANGELOG.md
├── .gitignore
├── README.md
└── Pipfile.lock


/img/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/micahflee/semiphemeral/HEAD/img/logo.png


--------------------------------------------------------------------------------
/app.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import semiphemeral
3 | 
4 | semiphemeral.main()
5 | 


--------------------------------------------------------------------------------
/img/settings.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/micahflee/semiphemeral/HEAD/img/settings.png


--------------------------------------------------------------------------------
/img/logo-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/micahflee/semiphemeral/HEAD/img/logo-small.png


--------------------------------------------------------------------------------
/semiphemeral/static/img/loading.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/micahflee/semiphemeral/HEAD/semiphemeral/static/img/loading.gif


--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
 1 | [[source]]
 2 | name = "pypi"
 3 | url = "https://pypi.org/simple"
 4 | verify_ssl = true
 5 | 
 6 | [dev-packages]
 7 | twine = "*"
 8 | black = "*"
 9 | 
10 | [packages]
11 | click = "*"
12 | colorama = "*"
13 | flask = "*"
14 | sqlalchemy = "*"
15 | tweepy = "~=3.10"
16 | 
17 | [requires]
18 | python_version = "3"
19 | 
20 | [pipenv]
21 | allow_prereleases = true
22 | 


--------------------------------------------------------------------------------
/scripts/pypi_release.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && cd .. && pwd )"
 4 | cd $DIR
 5 | 
 6 | # Delete old build
 7 | rm -rf build dist > /dev/null
 8 | 
 9 | # Make sure pipenv is good
10 | pipenv install --dev
11 | 
12 | # Create new source and binary build
13 | pipenv run python setup.py sdist bdist_wheel
14 | 
15 | # Upload to PyPI
16 | pipenv run python -m twine upload dist/*
17 | 


--------------------------------------------------------------------------------
/semiphemeral/templates/tweets.html:
--------------------------------------------------------------------------------
 1 | {% extends "base.html" %}
 2 | {% block title %}Semiphemeral Tweets{% endblock %}
 3 | {% block header_text %}Semiphemeral Tweets{% endblock %}
 4 | {% block content %}
 5 | 
 6 | <div class="tweets">
 7 |   <div class="controls">
 8 |     <div class="filter">
 9 |       <input placeholder="Filter" type="text" />
10 |     </div>
11 |     <div class="options"><label><input type="checkbox" /> Show replies</label></div>
12 |     <div class="info"></div>
13 |     <div class="pagination"></div>
14 |   </div>
15 | 
16 |   <div class="tweets-to-delete"></div>
17 | </div>
18 | 
19 | <script sync src="https://platform.twitter.com/widgets.js"></script>
20 | <script src="/static/js/tweets.js"></script>
21 | {% endblock %}
22 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License
 2 | 
 3 | Copyright (c) 2010-2019 Google, Inc. http://angularjs.org
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/semiphemeral/static/js/base.js:
--------------------------------------------------------------------------------
 1 | function comma_formatted(x) {
 2 |   return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
 3 | }
 4 | 
 5 | $(function () {
 6 |   $.get('/api/statistics', function (data) {
 7 |     is_configured = (data.is_configured ? 'yes' : 'no');
 8 |     $('.statistics .is_configured').text(is_configured);
 9 |     $('.statistics .last_fetch').text(data.last_fetch);
10 |     $('.statistics .my_tweets').text(comma_formatted(data.my_tweets));
11 |     $('.statistics .tweets_to_delete').text(comma_formatted(data.tweets_to_delete));
12 |     $('.statistics .my_retweets').text(comma_formatted(data.my_retweets));
13 |     $('.statistics .my_likes').text(comma_formatted(data.my_likes));
14 |     $('.statistics .deleted_tweets').text(comma_formatted(data.deleted_tweets));
15 |     $('.statistics .deleted_retweets').text(comma_formatted(data.deleted_retweets));
16 |     $('.statistics .unliked_tweets').text(comma_formatted(data.unliked_tweets));
17 |     $('.statistics .excluded_tweets').text(comma_formatted(data.excluded_tweets));
18 |     $('.statistics .other_tweets').text(comma_formatted(data.other_tweets));
19 |     $('.statistics .threads').text(comma_formatted(data.threads));
20 |   })
21 | })
22 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | import setuptools
 2 | import semiphemeral
 3 | 
 4 | with open("README.md", "r") as fh:
 5 |     long_description = fh.read()
 6 | 
 7 | setuptools.setup(
 8 |     name="semiphemeral",
 9 |     version=semiphemeral.version,
10 |     author="Micah Lee",
11 |     author_email="micah@micahflee.com",
12 |     long_description=long_description,
13 |     description="Automatically delete your old tweets, except for the ones you want to keep",
14 |     long_description_content_type="text/markdown",
15 |     license="MIT",
16 |     url="https://github.com/micahflee/semiphemeral",
17 |     packages=["semiphemeral"],
18 |     package_data={
19 |         "semiphemeral": [
20 |             "templates/*",
21 |             "static/*",
22 |             "static/img/*",
23 |             "static/js/*",
24 |             "static/js/lib/*",
25 |         ]
26 |     },
27 |     classifiers=(
28 |         "Development Status :: 2 - Pre-Alpha",
29 |         "Environment :: Console",
30 |         "Intended Audience :: End Users/Desktop",
31 |         "License :: OSI Approved :: MIT License",
32 |     ),
33 |     entry_points={"console_scripts": ["semiphemeral = semiphemeral:main",],},
34 |     install_requires=["click", "colorama", "tweepy", "flask", "sqlalchemy"],
35 | )
36 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Semiphemeral Changelog
 2 | 
 3 | ## 0.7
 4 | 
 5 | * Feature: Support for proxies, including Tor
 6 | * Feature: Add `--host` option to complement `--port` option
 7 | * Bugfix: Fix `direct-messages.js` filename check in Twitter archive
 8 | 
 9 | ## 0.6
10 | 
11 | * Feature: Support deleting DMs
12 | 
13 | ## 0.5
14 | 
15 | * Bugfix: Fixed issue fetching tweets from the Twitter API in unlike mode
16 | 
17 | ## 0.4
18 | 
19 | * Feature: Support for unliking old tweets that the Twitter API doesn't make easy, by reliking/unliking them all
20 | * Feature: Add support for logging events to file
21 | * Feature: Add `configure --debug` option
22 | * Feature: Add `configure --port` option
23 | * Bugfix: Fetches 240 character tweets, instead of just the first 140 characters
24 | 
25 | ## 0.3
26 | 
27 | * Bugfix: jQuery wasn't included in the PyPi package, so tweets page of configure web app was broken. jQuery is now included
28 | * Bugfix: When fetching, fetch likes if the delete likes setting is enabled rather than the delete tweets setting
29 | * Bugfix: Exit gracefully when running stats if semiphemeral isn't configured yet
30 | 
31 | ## 0.2
32 | 
33 | * Bugfix: PyPi package now include HTML templates and static resources
34 | 
35 | ## 0.1
36 | 
37 | * First release
38 | 


--------------------------------------------------------------------------------
/semiphemeral/static/js/settings.js:
--------------------------------------------------------------------------------
 1 | $(function () {
 2 |   function update_ui() {
 3 |     if ($('.log-to-file-checkbox').prop('checked')) {
 4 |       $('.log-to-file-fieldset').show();
 5 |     } else {
 6 |       $('.log-to-file-fieldset').hide();
 7 |     }
 8 | 
 9 |     if ($('.delete-tweets-checkbox').prop('checked')) {
10 |       $('.delete-tweets-fieldset').show();
11 |     } else {
12 |       $('.delete-tweets-fieldset').hide();
13 |     }
14 | 
15 |     if ($('.retweets-likes-checkbox').prop('checked')) {
16 |       $('.retweets-likes-fieldset').show();
17 |     } else {
18 |       $('.retweets-likes-fieldset').hide();
19 |     }
20 | 
21 |     if ($('.dms-checkbox').prop('checked')) {
22 |       $('.dms-fieldset').show();
23 |     } else {
24 |       $('.dms-fieldset').hide();
25 |     }
26 | 
27 |     if ($('.use-tor-checkbox').prop('checked')) {
28 |       $('.proxy-text').val('socks5://localhost:9050').attr('readonly', 'readonly');
29 |     } else{
30 |       $('.proxy-text').removeAttr('readonly', 'false');
31 |     }
32 |   }
33 | 
34 |   $('.log-to-file-checkbox').change(update_ui);
35 |   $('.delete-tweets-checkbox').change(update_ui);
36 |   $('.retweets-likes-checkbox').change(update_ui);
37 |   $('.use-tor-checkbox').change(update_ui);
38 |   $('.dms-checkbox').change(update_ui);
39 |   update_ui();
40 | })
41 | 


--------------------------------------------------------------------------------
/semiphemeral/templates/base.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html>
 3 | <head>
 4 |   <title>{% block title %}{% endblock %}</title>
 5 |   <link rel="stylesheet" href="/static/style.css">
 6 |   <script src="/static/js/lib/jquery-3.4.1.min.js"></script>
 7 |   <script src="/static/js/base.js"></script>
 8 | </head>
 9 | 
10 | <body>
11 |   <div class="sidebar">
12 |     <ul>
13 |       <li><a href="{{ url_for('edit_settings') }}">settings</a></li>
14 |       <li><a href="{{ url_for('tweets') }}">tweets</a></li>
15 |     </ul>
16 | 
17 |     <div class="statistics">
18 |       <ul>
19 |         <li>configured: <span class="value is_configured"></span></li>
20 |         <li>last fetch: <span class="value last_fetch"></span></li>
21 |         <li><span class="value my_tweets"></span> tweets</li>
22 |         <li><span class="value my_retweets"></span> retweets</li>
23 |         <li><span class="value my_likes"></span> likes</li>
24 |         <li><span class="value deleted_tweets"></span> deleted tweets</li>
25 |         <li><span class="value deleted_retweets"></span> deleted retweets</li>
26 |         <li><span class="value unliked_tweets"></span> unliked tweets</li>
27 |         <li><span class="value excluded_tweets"></span> excluded tweets</li>
28 |         <li><span class="value other_tweets"></span> tweets from other users</li>
29 |         <li><span class="value threads"></span> threads</li>
30 |         <li><span class="value tweets_to_delete"></span> tweets to delete</li>
31 |       </ul>
32 |     </div>
33 | 
34 |   </div>
35 |   <div class="content">
36 |     <h1>{% block header_text %}{% endblock %}</h1>
37 |     {% block content %}{% endblock %}
38 |   </div>
39 | </body>
40 | </html>
41 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 
106 | # other
107 | .DS_Store
108 | .vscode/
109 | 


--------------------------------------------------------------------------------
/semiphemeral/import_export.py:
--------------------------------------------------------------------------------
 1 | import click
 2 | import json
 3 | 
 4 | from .db import Tweet
 5 | 
 6 | 
 7 | class ImportExport:
 8 |     def __init__(self, common, twitter=None):
 9 |         self.common = common
10 |         self.twitter = twitter
11 | 
12 |     def excluded_export(self, filename):
13 |         tweets = (
14 |             self.common.session.query(Tweet)
15 |             .filter(Tweet.user_id == int(self.common.settings.get("user_id")))
16 |             .filter(Tweet.exclude_from_delete == 1)
17 |             .order_by(Tweet.created_at)
18 |             .all()
19 |         )
20 | 
21 |         tweets_to_exclude = []
22 |         for tweet in tweets:
23 |             tweets_to_exclude.append(tweet.status_id)
24 | 
25 |         with open(filename, "w") as f:
26 |             f.write(json.dumps(tweets_to_exclude))
27 | 
28 |         click.echo("Exported {} tweet status_ids".format(len(tweets_to_exclude)))
29 | 
30 |     def excluded_import(self, filename):
31 |         with open(filename, "r") as f:
32 |             try:
33 |                 status_ids = json.loads(f.read())
34 |             except:
35 |                 click.echo("Error JSON decoding input file")
36 |                 return
37 | 
38 |         # Validate
39 |         if type(status_ids) != list:
40 |             click.echo("Input file should be a list")
41 |             return
42 |         for status_id in status_ids:
43 |             if type(status_id) != int:
44 |                 click.echo("All items in the input file list should be ints")
45 |                 return
46 | 
47 |         # Import
48 |         for status_id in status_ids:
49 |             tweet = (
50 |                 self.common.session.query(Tweet).filter_by(status_id=status_id).first()
51 |             )
52 |             if tweet:
53 |                 tweet.exclude_from_delete = True
54 |                 self.common.session.add(tweet)
55 |                 tweet.excluded_summarize()
56 |             else:
57 |                 try:
58 |                     status = self.twitter.api.get_status(status_id)
59 |                     tweet = Tweet(status)
60 |                     tweet.exclude_from_delete = True
61 |                     self.common.session.add(tweet)
62 |                     tweet.excluded_fetch_summarize()
63 |                 except tweepy.error.TweepError as e:
64 |                     click.echo("Error for tweet {}: {}".format(tweet.status_id, e))
65 | 
66 |         self.common.session.commit()
67 | 


--------------------------------------------------------------------------------
/semiphemeral/settings.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import json
 3 | 
 4 | 
 5 | class Settings(object):
 6 |     def __init__(self, filename):
 7 |         self.filename = filename
 8 |         self.default_settings = {
 9 |             "api_key": "",
10 |             "api_secret": "",
11 |             "access_token_key": "",
12 |             "access_token_secret": "",
13 |             "username": "",
14 |             "user_id": None,
15 |             "delete_tweets": True,
16 |             "tweets_days_threshold": 30,
17 |             "tweets_retweet_threshold": 100,
18 |             "tweets_like_threshold": 100,
19 |             "tweets_threads_threshold": True,
20 |             "retweets_likes": True,
21 |             "retweets_likes_delete_retweets": True,
22 |             "retweets_likes_retweets_threshold": 30,
23 |             "retweets_likes_delete_likes": True,
24 |             "retweets_likes_likes_threshold": 60,
25 |             "delete_dms": True,
26 |             "dms_days_threshold": 14,
27 |             "since_id": None,
28 |             "last_fetch": None,
29 |             "unlike_ignore_list": [],
30 |             "logging": False,
31 |             "log_filename": os.path.expanduser("~/.semiphemeral/log"),
32 |             "log_format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
33 |             'proxy': "",
34 |             'use_tor': False,
35 |         }
36 |         self.load()
37 | 
38 |     def get(self, key):
39 |         return self.settings[key]
40 | 
41 |     def set(self, key, val):
42 |         self.settings[key] = val
43 | 
44 |     def load(self):
45 |         if os.path.exists(self.filename):
46 |             with open(self.filename, "r") as f:
47 |                 self.settings = json.load(f)
48 | 
49 |             for key in self.default_settings:
50 |                 if key not in self.settings:
51 |                     self.set(key, self.default_settings[key])
52 |         else:
53 |             self.settings = self.default_settings.copy()
54 | 
55 |     def save(self):
56 |         with open(self.filename, "w") as f:
57 |             os.chmod(self.filename, 0o0600)
58 |             json.dump(self.settings, f)
59 | 
60 |     def is_configured(self):
61 |         if (
62 |             self.get("api_key") == ""
63 |             or self.get("api_secret") == ""
64 |             or self.get("access_token_key") == ""
65 |             or self.get("access_token_secret") == ""
66 |             or self.get("username") == ""
67 |         ):
68 |             return False
69 |         return True
70 | 
71 |     def unlike_should_ignore(self, status_id):
72 |         return status_id in self.get("unlike_ignore_list")
73 | 
74 |     def unlike_ignore(self, status_id):
75 |         if status_id not in self.settings["unlike_ignore_list"]:
76 |             self.settings["unlike_ignore_list"].append(status_id)
77 |         self.save()
78 | 


--------------------------------------------------------------------------------
/semiphemeral/static/style.css:
--------------------------------------------------------------------------------
  1 | body {
  2 |   font-family: sans-serif;
  3 | }
  4 | 
  5 | .sidebar {
  6 |   height: 100%;
  7 |   width: 250px;
  8 |   position: fixed;
  9 |   z-index: 1;
 10 |   top: 0;
 11 |   left: 0;
 12 |   background-color: #111111;
 13 |   color: #ffffff;
 14 |   overflow-x: hidden;
 15 |   padding-top: 20px;
 16 |   padding: 10px;
 17 | }
 18 | 
 19 | .sidebar ul {
 20 |   list-style: none;
 21 |   margin: 0;
 22 |   padding: 0;
 23 | }
 24 | 
 25 | .sidebar ul li {
 26 |   text-align: center;
 27 | }
 28 | 
 29 | .sidebar ul li a {
 30 |   display: inline-block;
 31 |   padding: 5px;
 32 |   margin: 0 10px 0 0;
 33 |   text-decoration: none;
 34 |   color: #a1acfc;
 35 |   font-weight: bold;
 36 | }
 37 | 
 38 | .sidebar .statistics {
 39 |   margin-top: 20px;
 40 | }
 41 | 
 42 | .sidebar .statistics ul li {
 43 |   text-align: center;
 44 |   color: #666666;
 45 |   margin-bottom: 10px;
 46 | }
 47 | 
 48 | .sidebar .statistics .value {
 49 |   color: #e5d204;
 50 | }
 51 | 
 52 | .content {
 53 |   margin-left: 270px;
 54 |   padding: 0px 10px;
 55 | }
 56 | 
 57 | .content h1 {
 58 |   font-style: italic;
 59 | }
 60 | 
 61 | .content a {
 62 |   color: #444d93;
 63 | }
 64 | 
 65 | .settings form ul {
 66 |   list-style: none;
 67 |   margin: 0;
 68 |   padding: 0;
 69 | }
 70 | 
 71 | .settings form ul li {
 72 |   margin-bottom: 1em;
 73 | }
 74 | 
 75 | .settings form label {
 76 |   display: inline-block;
 77 |   width: 12em;
 78 |   text-align: right;
 79 | }
 80 | 
 81 | .settings form input[type="text"] {
 82 |   width: 500px;
 83 | }
 84 | 
 85 | .settings form label.checkbox, .settings fieldset label {
 86 |   display: inline;
 87 |   width: auto;
 88 |   text-align: left;
 89 | }
 90 | 
 91 | .settings form input.small {
 92 |   width: 50px;
 93 | }
 94 | 
 95 | .tweets .controls {
 96 |   display: none;
 97 |   position: fixed;
 98 |   bottom: 10px;
 99 |   z-index: 999;
100 |   height: 100px;
101 |   background-color: #ffffff;
102 |   border: 1px solid #000000;
103 |   padding: 20px;
104 | }
105 | 
106 | .tweets .controls .filter {
107 |   margin: 0 0 10px 0;
108 | }
109 | 
110 | .tweets .controls .filter input {
111 |   width: 600px;
112 |   padding: 5px;
113 |   font-size: 1.2em;
114 | }
115 | 
116 | .tweets .controls .options {
117 |   margin: 0 20px 10px 0;
118 |   color: #666666;
119 |   font-size: 0.8em;
120 |   display: inline-block;
121 | }
122 | 
123 | .tweets .controls .info {
124 |   margin: 0 0 10px 0;
125 |   color: #666666;
126 |   font-size: 0.8em;
127 |   display: inline-block;
128 | }
129 | 
130 | .tweets .controls .pagination {
131 |   margin: 15px 0 0 0;
132 | }
133 | 
134 | .tweets .controls .pagination .pagination-item {
135 |   padding: 5px 10px;
136 |   margin: 0 5px 0 0;
137 |   font-size: 1em;
138 |   border: 1px solid #999999;
139 | }
140 | 
141 | .tweets .pagination .pagination-item a {
142 |   text-decoration: none;
143 |   color: #034b9e;
144 | }
145 | 
146 | .tweets .controls .pagination .pagination-item-current {
147 |   color: #000000;
148 |   font-weight: bold;
149 |   border: 0;
150 | }
151 | 
152 | .tweets .tweets-to-delete {
153 |   padding-bottom: 130px;
154 | }
155 | 
156 | .tweets .tweets-to-delete .tweet {
157 |   display: inline-block;
158 |   margin: 0 20px 20px 0;
159 |   margin-top: 5px;
160 | }
161 | 
162 | .tweets .tweets-to-delete .tweet .info .toggle-exclude-label {
163 |   margin-left: 5px;
164 | }
165 | 
166 | .tweets .tweets-to-delete .tweet .info .toggle-exclude-label.excluded {
167 |   font-weight: bold;
168 | }
169 | 
170 | .tweets .tweets-to-delete .tweet .info .stats {
171 |   font-size: 0.8em;
172 |   color: #666666;
173 | }
174 | 


--------------------------------------------------------------------------------
/semiphemeral/__init__.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import click
  3 | import json
  4 | 
  5 | from .common import Common
  6 | from .settings import Settings
  7 | from .db import create_db
  8 | from .web import create_app
  9 | from .twitter import Twitter
 10 | from .import_export import ImportExport
 11 | 
 12 | version = "0.7"
 13 | 
 14 | 
 15 | def init():
 16 |     click.echo(click.style("semiphemeral {}".format(version), fg="yellow"))
 17 | 
 18 |     # Initialize stuff
 19 |     base = os.path.expanduser("~/.semiphemeral")
 20 |     os.makedirs(base, mode=0o700, exist_ok=True)
 21 |     # Fix insecure prior installation permissions
 22 |     os.chmod(base, 0o700)
 23 |     settings = Settings(os.path.join(base, "settings.json"))
 24 |     session = create_db(os.path.join(base, "tweets.db"))
 25 | 
 26 |     common = Common(settings, session)
 27 |     return common
 28 | 
 29 | 
 30 | @click.group()
 31 | def main():
 32 |     """Automatically delete your old tweets, except for the ones you want to keep"""
 33 | 
 34 | 
 35 | @main.command("configure", short_help="Start the web server to configure semiphemeral")
 36 | @click.option("--debug", is_flag=True, help="Start web server in debug mode")
 37 | @click.option("--host", default="127.0.0.1", help="Host to expose the web server on")
 38 | @click.option("--port", default=8080, help="Port to expose the web server on")
 39 | def configure(debug, host, port):
 40 |     common = init()
 41 |     click.echo(
 42 |         "Load this website in a browser to configure semiphemeral, and press Ctrl-C when done"
 43 |     )
 44 |     click.echo("http://{host}:{port}".format(host=host, port=port))
 45 |     click.echo("")
 46 |     app = create_app(common)
 47 |     app.run(host=host, port=port, threaded=False, debug=debug)
 48 | 
 49 | 
 50 | @main.command("stats", short_help="Show stats about tweets in the database")
 51 | def stats():
 52 |     common = init()
 53 |     t = Twitter(common)
 54 |     if common.settings.is_configured():
 55 |         t.stats()
 56 | 
 57 | 
 58 | @main.command("fetch", short_help="Download all tweets/DMs")
 59 | def fetch():
 60 |     common = init()
 61 |     t = Twitter(common)
 62 |     if common.settings.is_configured():
 63 |         t.fetch()
 64 | 
 65 | 
 66 | @main.command(
 67 |     "delete",
 68 |     short_help="Delete tweets that aren't automatically or manually excluded, likes, and DMs",
 69 | )
 70 | def delete():
 71 |     common = init()
 72 |     t = Twitter(common)
 73 |     if common.settings.is_configured():
 74 |         t.delete()
 75 | 
 76 | 
 77 | @main.command('import', short_help='Import tweets from a Twitter data export')
 78 | @click.argument('path', type=click.Path(exists=True))
 79 | def archive_import(path):
 80 |     common = init()
 81 |     t = Twitter(common)
 82 |     if common.settings.is_configured():
 83 |         t.import_dump(path)
 84 | 
 85 | 
 86 | @main.command(
 87 |     "unlike",
 88 |     short_help="Delete old likes that aren't available through the Twitter API",
 89 | )
 90 | @click.option(
 91 |     "--filename",
 92 |     required=True,
 93 |     help="Path to like.js from Twitter data downloaded from https://twitter.com/settings/your_twitter_data",
 94 | )
 95 | def unlike(filename):
 96 |     common = init()
 97 | 
 98 |     t = Twitter(common)
 99 |     if common.settings.is_configured():
100 |         t.unlike(filename)
101 | 
102 | 
103 | @main.command(
104 |     "delete_dms", short_help="Delete DMs that aren't available through the Twitter API"
105 | )
106 | @click.option(
107 |     "--filename",
108 |     required=True,
109 |     help="Path to direct-message.js from Twitter data downloaded from https://twitter.com/settings/your_twitter_data",
110 | )
111 | def delete_dms(filename):
112 |     common = init()
113 | 
114 |     t = Twitter(common)
115 |     if common.settings.is_configured():
116 |         t.delete_dms(filename)
117 | 
118 | 
119 | @main.command(
120 |     "excluded_export",
121 |     short_help="Export tweets excluded that are excluded from deletion",
122 | )
123 | @click.option(
124 |     "--filename",
125 |     required=True,
126 |     help="Output JSON file to save a list of tweet status_ids",
127 | )
128 | def excluded_export(filename):
129 |     common = init()
130 |     ie = ImportExport(common)
131 |     ie.excluded_export(filename)
132 | 
133 | 
134 | @main.command(
135 |     "excluded_import",
136 |     short_help="Import tweets excluded that are excluded from deletion",
137 | )
138 | @click.option(
139 |     "--filename",
140 |     required=True,
141 |     help="Input JSON file that contains a list of tweet status_ids",
142 | )
143 | def excluded_import(filename):
144 |     common = init()
145 |     t = Twitter(common)
146 |     if common.settings.is_configured():
147 |         ie = ImportExport(common, t)
148 |         ie.excluded_import(filename)
149 | 


--------------------------------------------------------------------------------
/semiphemeral/db.py:
--------------------------------------------------------------------------------
  1 | import click
  2 | 
  3 | from datetime import datetime
  4 | from sqlalchemy import Column, String, Integer, Boolean, DateTime, ForeignKey
  5 | from sqlalchemy.ext.declarative import declarative_base
  6 | from sqlalchemy import create_engine
  7 | from sqlalchemy.orm import sessionmaker, relationship
  8 | 
  9 | Base = declarative_base()
 10 | 
 11 | 
 12 | class Thread(Base):
 13 |     __tablename__ = "threads"
 14 |     id = Column(Integer, primary_key=True)
 15 |     root_status_id = Column(Integer)
 16 |     should_exclude = Column(Boolean)
 17 | 
 18 |     tweets = relationship("Tweet", back_populates="thread")
 19 | 
 20 |     def __init__(self, root_status_id):
 21 |         self.root_status_id = root_status_id
 22 |         self.should_exclude = False
 23 | 
 24 | 
 25 | class Tweet(Base):
 26 |     __tablename__ = "tweets"
 27 |     id = Column(Integer, primary_key=True)
 28 |     created_at = Column(DateTime)
 29 |     user_id = Column(Integer)  # we download all threads too, including with other users
 30 |     user_screen_name = Column(String)
 31 |     status_id = Column(Integer)
 32 |     lang = Column(String)
 33 |     source = Column(String)
 34 |     source_url = Column(String)
 35 |     text = Column(String)
 36 |     in_reply_to_screen_name = Column(String)
 37 |     in_reply_to_status_id = Column(Integer)
 38 |     in_reply_to_user_id = Column(Integer)
 39 |     retweet_count = Column(Integer)
 40 |     favorite_count = Column(Integer)
 41 |     retweeted = Column(Boolean)
 42 |     favorited = Column(Boolean)
 43 |     is_retweet = Column(Boolean)
 44 |     is_deleted = Column(Boolean)
 45 |     is_unliked = Column(Boolean)
 46 |     exclude_from_delete = Column(Boolean)
 47 | 
 48 |     thread_id = Column(Integer, ForeignKey("threads.id"))
 49 |     thread = relationship("Thread", back_populates="tweets")
 50 | 
 51 |     def __init__(self, status):
 52 |         self.created_at = status.created_at
 53 |         self.user_id = status.author.id
 54 |         self.user_screen_name = status.author.screen_name
 55 |         self.status_id = status.id
 56 |         self.lang = status.lang
 57 |         self.source = status.source
 58 |         self.source_url = status.source_url
 59 |         self.text = status.full_text
 60 | 
 61 |         # These fields don't exist in imported non-reply tweets
 62 |         for field in ('in_reply_to_screen_name', 'in_reply_to_status_id', 'in_reply_to_user_id'):
 63 |             setattr(self, field, getattr(status, field, None))
 64 | 
 65 |         self.retweet_count = status.retweet_count
 66 |         self.favorite_count = status.favorite_count
 67 |         self.retweeted = status.retweeted
 68 |         self.favorited = status.favorited
 69 |         self.is_retweet = hasattr(status, "retweeted_status")
 70 |         self.is_deleted = False
 71 |         self.is_unliked = False
 72 |         self.exclude_from_delete = False
 73 | 
 74 |     def already_saved(self, session):
 75 |         """
 76 |         Returns true if a tweet with this status_id is already in the db
 77 |         """
 78 |         tweet = session.query(Tweet).filter_by(status_id=self.status_id).first()
 79 |         if tweet:
 80 |             click.secho(
 81 |                 "Skipped {} @{}, id={}".format(
 82 |                     self.created_at.strftime("%Y-%m-%d"),
 83 |                     self.user_screen_name,
 84 |                     self.status_id,
 85 |                 ),
 86 |                 dim=True,
 87 |             )
 88 |             return True
 89 | 
 90 |     def fetch_summarize(self):
 91 |         click.echo("Fetched {}".format(self.summarize_string()))
 92 | 
 93 |     def unretweet_summarize(self):
 94 |         click.echo("Unretweeted {}".format(self.summarize_string(True)))
 95 | 
 96 |     def unlike_summarize(self):
 97 |         click.echo("Unliked {}".format(self.summarize_string()))
 98 | 
 99 |     def relike_unlike_summarize(self):
100 |         click.echo("Reliked and unliked {}".format(self.summarize_string()))
101 | 
102 |     def delete_summarize(self):
103 |         click.echo("Deleted {}".format(self.summarize_string()))
104 | 
105 |     def excluded_summarize(self):
106 |         click.echo("Excluded from deletion {}".format(self.summarize_string()))
107 | 
108 |     def excluded_fetch_summarize(self):
109 |         click.echo(
110 |             "Fetched and excluded from deletion {}".format(self.summarize_string())
111 |         )
112 | 
113 |     def summarize_string(self, include_rt_user=False):
114 |         if include_rt_user:
115 |             return "{} @{} {}, id={}".format(
116 |                 self.created_at.strftime("%Y-%m-%d"),
117 |                 self.user_screen_name,
118 |                 self.text.split(":")[0],
119 |                 self.status_id,
120 |             )
121 |         else:
122 |             return "{} @{}, id={}".format(
123 |                 self.created_at.strftime("%Y-%m-%d"),
124 |                 self.user_screen_name,
125 |                 self.status_id,
126 |             )
127 | 
128 | 
129 | def create_db(database_path):
130 |     engine = create_engine("sqlite:///{}".format(database_path))
131 | 
132 |     session = sessionmaker()
133 |     session.configure(bind=engine)
134 |     Base.metadata.create_all(engine)
135 | 
136 |     return session()
137 | 


--------------------------------------------------------------------------------
/semiphemeral/common.py:
--------------------------------------------------------------------------------
  1 | import datetime
  2 | 
  3 | from .db import Tweet, Thread
  4 | 
  5 | import logging
  6 | 
  7 | 
  8 | class Common:
  9 |     def __init__(self, settings, session):
 10 |         self.settings = settings
 11 |         self.session = session
 12 | 
 13 |         self.logger = logging.getLogger("semiphemeral-log")
 14 |         self.logger.setLevel(logging.INFO)
 15 |         handler = logging.FileHandler(self.settings.get("log_filename"))
 16 |         handler.setLevel(logging.INFO)
 17 |         formatter = logging.Formatter(self.settings.get("log_format"))
 18 |         handler.setFormatter(formatter)
 19 |         self.logger.addHandler(handler)
 20 | 
 21 |     def get_stats(self):
 22 |         self.settings.load()
 23 | 
 24 |         is_configured = self.settings.is_configured()
 25 |         if not is_configured:
 26 |             return {
 27 |                 "is_configured": is_configured,
 28 |             }
 29 | 
 30 |         last_fetch = self.settings.get("last_fetch")
 31 |         my_tweets = self.session.execute(
 32 |             "SELECT COUNT(*) FROM tweets WHERE user_id={} AND is_deleted=0 AND is_retweet=0".format(
 33 |                 int(self.settings.get("user_id"))
 34 |             )
 35 |         ).first()[0]
 36 |         my_retweets = self.session.execute(
 37 |             "SELECT COUNT(*) FROM tweets WHERE user_id={} AND is_deleted=0 AND is_retweet=1".format(
 38 |                 int(self.settings.get("user_id"))
 39 |             )
 40 |         ).first()[0]
 41 |         my_likes = self.session.execute(
 42 |             "SELECT COUNT(*) FROM tweets WHERE favorited=1"
 43 |         ).first()[0]
 44 |         deleted_tweets = self.session.execute(
 45 |             "SELECT COUNT(*) FROM tweets WHERE user_id={} AND is_deleted=1 AND is_retweet=0".format(
 46 |                 int(self.settings.get("user_id"))
 47 |             )
 48 |         ).first()[0]
 49 |         deleted_retweets = self.session.execute(
 50 |             "SELECT COUNT(*) FROM tweets WHERE user_id={} AND is_deleted=1 AND is_retweet=1".format(
 51 |                 int(self.settings.get("user_id"))
 52 |             )
 53 |         ).first()[0]
 54 |         unliked_tweets = self.session.execute(
 55 |             "SELECT COUNT(*) FROM tweets WHERE favorited=1 AND is_unliked=1"
 56 |         ).first()[0]
 57 |         excluded_tweets = self.session.execute(
 58 |             "SELECT COUNT(*) FROM tweets WHERE user_id={} AND exclude_from_delete=1".format(
 59 |                 int(self.settings.get("user_id"))
 60 |             )
 61 |         ).first()[0]
 62 |         other_tweets = self.session.execute(
 63 |             "SELECT COUNT(*) FROM tweets WHERE user_id!={}".format(
 64 |                 int(self.settings.get("user_id"))
 65 |             )
 66 |         ).first()[0]
 67 |         threads = self.session.execute("SELECT COUNT(*) FROM threads").first()[0]
 68 |         tweets_to_delete = self.get_tweets_to_delete()
 69 | 
 70 |         return {
 71 |             "is_configured": is_configured,
 72 |             "last_fetch": last_fetch,
 73 |             "my_tweets": my_tweets,
 74 |             "my_retweets": my_retweets,
 75 |             "my_likes": my_likes,
 76 |             "deleted_tweets": deleted_tweets,
 77 |             "deleted_retweets": deleted_retweets,
 78 |             "unliked_tweets": unliked_tweets,
 79 |             "excluded_tweets": excluded_tweets,
 80 |             "other_tweets": other_tweets,
 81 |             "threads": threads,
 82 |             "tweets_to_delete": len(tweets_to_delete),
 83 |         }
 84 | 
 85 |     def get_tweets_to_delete(self, include_excluded=False):
 86 |         """
 87 |         Returns a list of Tweet objects for tweets that should be deleted based
 88 |         on criteria in settings. This list includes tweets where exclude_from_delete=True,
 89 |         so it's important to manually exclude those before deleting
 90 |         """
 91 |         self.settings.load()
 92 |         datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
 93 |             days=self.settings.get("tweets_days_threshold")
 94 |         )
 95 | 
 96 |         # Select tweets from threads to exclude
 97 |         tweets_to_exclude = []
 98 |         threads = self.session.query(Thread).filter(Thread.should_exclude == True).all()
 99 |         for thread in threads:
100 |             for tweet in thread.tweets:
101 |                 if tweet.user_id == self.settings.get("user_id"):
102 |                     tweets_to_exclude.append(tweet.status_id)
103 | 
104 |         # Select tweets that we will delete
105 |         tweets_to_delete = []
106 |         q = (
107 |             self.session.query(Tweet)
108 |             .filter(Tweet.user_id == int(self.settings.get("user_id")))
109 |             .filter(Tweet.is_deleted == 0)
110 |             .filter(Tweet.is_retweet == 0)
111 |             .filter(Tweet.created_at < datetime_threshold)
112 |             .filter(Tweet.retweet_count < self.settings.get("tweets_retweet_threshold"))
113 |             .filter(Tweet.favorite_count < self.settings.get("tweets_like_threshold"))
114 |         )
115 | 
116 |         # Should we also filter out exclude_from_delete?
117 |         if not include_excluded:
118 |             q = q.filter(Tweet.exclude_from_delete != True)
119 | 
120 |         q = q.order_by(Tweet.created_at)
121 | 
122 |         tweets = q.all()
123 |         for tweet in tweets:
124 |             if tweet.status_id not in tweets_to_exclude:
125 |                 tweets_to_delete.append(tweet)
126 | 
127 |         return tweets_to_delete
128 | 
129 |     def log(self, message):
130 |         if self.settings.get("logging"):
131 |             self.logger.info(message)
132 | 


--------------------------------------------------------------------------------
/semiphemeral/templates/settings.html:
--------------------------------------------------------------------------------
  1 | {% extends "base.html" %}
  2 | {% block title %}Semiphemeral Settings{% endblock %}
  3 | {% block header_text %}Semiphemeral Settings{% endblock %}
  4 | {% block content %}
  5 | 
  6 | <div class="settings">
  7 |   <form method="post" action="{{ url_for('edit_settings') }}">
  8 | 
  9 |     <h3>Twitter settings</h3>
 10 | 
 11 |     <p>See <a href="https://python-twitter.readthedocs.io/en/latest/getting_started.html">here</a> for documentation on how to get these Twitter API credentials.</p>
 12 | 
 13 |     <ul>
 14 |       <li>
 15 |         <label for="api_key">API key</label>
 16 |         <input type="text" name="api_key" value="{{ api_key }}" \>
 17 |       </li>
 18 |       <li>
 19 |         <label for="api_secret">API secret</label>
 20 |         <input type="text" name="api_secret" value="{{ api_secret }}" \>
 21 |       </li>
 22 |       <li>
 23 |         <label for="access_token_key">Access token key</label>
 24 |         <input type="text" name="access_token_key" value="{{ access_token_key }}" \>
 25 |       </li>
 26 |       <li>
 27 |         <label for="access_token_secret">Access token secret</label>
 28 |         <input type="text" name="access_token_secret" value="{{ access_token_secret }}" \>
 29 |       </li>
 30 |       <li>
 31 |         <label for="username">Twitter username</label>
 32 |         <input type="text" name="username" value="{{ username }}" \>
 33 |       </li>
 34 |     </ul>
 35 |     
 36 |     <p>
 37 |       <label class="checkbox">
 38 |         <input type="checkbox" class="log-to-file-checkbox" name="log_to_file" {%if log_to_file %}checked="checked" {% endif %} />
 39 |         Log events (last_fetch, deleted tweets/RTs/favorites) to a file
 40 |       </label>
 41 |     </p>
 42 |     <fieldset class="log-to-file-fieldset">
 43 |       <legend>Logging</legend>
 44 |       <p>
 45 |         <label for="log_filename">Log file name: </label>
 46 |         <input type="text" name="log_filename" value="{{ log_filename }}" \ />
 47 |       </p>
 48 |       <p>
 49 |         <label for="log_format">Log format: </label>
 50 |         <input type="text" name="log_format" value="{{ log_format }}" \ />
 51 |       </p>
 52 |     </fieldset>
 53 | 
 54 |     <p>
 55 |       <label class="checkbox">
 56 |         <input type="checkbox" class="delete-tweets-checkbox" name="delete_tweets" {% if delete_tweets %}checked="checked"{% endif %} />
 57 |         Delete old tweets
 58 |       </label>
 59 |     </p>
 60 |     <fieldset class="delete-tweets-fieldset">
 61 |       <legend>Tweets</legend>
 62 |       <p>
 63 |         Delete tweets older than
 64 |         <input class="small" type="number" min="0"  name="tweets_days_threshold" value="{{ tweets_days_threshold }}" \>
 65 |         days
 66 |       </p>
 67 |       <p>
 68 |         Unless they have at least
 69 |         <input class="small" type="number" min="0"  name="tweets_retweet_threshold" value="{{ tweets_retweet_threshold }}" \>
 70 |         retweets
 71 |       </p>
 72 |       <p>
 73 |         Or at least
 74 |         <input class="small" type="number" min="0"  name="tweets_like_threshold" value="{{ tweets_like_threshold }}" \>
 75 |         likes
 76 |       </p>
 77 |       <p>
 78 |         <label class="checkbox">
 79 |           <input type="checkbox" name="tweets_threads_threshold" {% if tweets_threads_threshold %}checked="checked"{% endif %} />
 80 |           Don't delete tweets that are part of a thread that contains at least one tweet that meets these thresholds
 81 |         </label>
 82 |       </p>
 83 |     </fieldset>
 84 | 
 85 |     <p>
 86 |       <label class="checkbox">
 87 |         <input type="checkbox" class="retweets-likes-checkbox" name="retweets_likes" {% if retweets_likes %}checked="checked"{% endif %} />
 88 |         Unretweet and unlike old tweets
 89 |       </label>
 90 |     </p>
 91 | 
 92 |     <fieldset class="retweets-likes-fieldset">
 93 |       <legend>Retweets and likes</legend>
 94 | 
 95 |       <p>
 96 |         <label class="checkbox">
 97 |           <input type="checkbox" name="retweets_likes_delete_retweets" {% if retweets_likes_delete_retweets %}checked="checked"{% endif %} />
 98 |           Unretweet tweets
 99 |         </label>
100 |         older than
101 |         <input class="small" type="number" min="0"  name="retweets_likes_retweets_threshold" value="{{ retweets_likes_retweets_threshold }}" \>
102 |         days
103 |       </p>
104 | 
105 |       <p>
106 |         <label class="checkbox">
107 |           <input type="checkbox" name="retweets_likes_delete_likes" {% if retweets_likes_delete_likes %}checked="checked"{% endif %} />
108 |           Unlike tweets
109 |         </label>
110 |         older than
111 |         <input class="small" type="number" min="0"  name="retweets_likes_likes_threshold" value="{{ retweets_likes_likes_threshold }}" \>
112 |         days
113 |       </p>
114 | 
115 |     </fieldset>
116 | 
117 |     <p>
118 |       <label class="checkbox">
119 |         <input type="checkbox" class="dms-checkbox" name="delete_dms" {% if delete_dms %}checked="checked"{% endif %} />
120 |         Delete direct messages
121 |       </label>
122 |     </p>
123 | 
124 |     <fieldset class="dms-fieldset">
125 |       <legend>Direct messages</legend>
126 | 
127 |       <p>
128 |         Delete direct messages older than
129 |         <input class="small" type="number" min="0"  name="dms_days_threshold" value="{{ dms_days_threshold }}" \>
130 |         days
131 |       </p>
132 |     </fieldset>
133 | 
134 |     <fieldset class="privacy-fieldset">
135 |         <legend>Privacy</legend>
136 | 
137 |         <p>
138 |             <label for="proxy">Proxy</label>
139 |             <input type="text" class="proxy-text" name="proxy" value="{{ proxy }}" \>
140 |             <label class="checkbox">
141 |               <input type="checkbox" class="use-tor-checkbox" name="use_tor" {% if use_tor %}checked="checked"{% endif %} />
142 |               Use Tor
143 |             </label>
144 |         </p>
145 |     </fieldset>
146 | 
147 |     <p><input type="submit" value="Save" /></p>
148 | 
149 |   </form>
150 | </div>
151 | 
152 | <script src="/static/js/settings.js"></script>
153 | {% endblock %}
154 | 


--------------------------------------------------------------------------------
/semiphemeral/static/js/tweets.js:
--------------------------------------------------------------------------------
  1 | $(function () {
  2 | 
  3 |   function change_state(q, page, count, replies) {
  4 |     var new_state = { q: q, page: page, count: count, replies: replies };
  5 |     window.location = '#' + JSON.stringify(new_state);
  6 |   }
  7 | 
  8 |   function toggle_exclude(id, excluded) {
  9 |     function update_text(excluded) {
 10 |       if (excluded) {
 11 |         $('#toggle-exclude-label-' + id).addClass('excluded').text('Excluded from deletion');
 12 |       } else {
 13 |         $('#toggle-exclude-label-' + id).removeClass('excluded').text('Staged for deletion');
 14 |       }
 15 |     }
 16 | 
 17 |     // If excluded is the same as we already know, just update the text
 18 |     if (window.semiphemeral.tweets[id].excluded == excluded) {
 19 |       update_text(excluded);
 20 |       return;
 21 |     }
 22 | 
 23 |     // Otherwise make ajax request
 24 |     $('#toggle-exclude-label-' + id).removeClass('excluded').text('Saving...');
 25 |     var exclude_from_delete = (excluded ? 1 : 0);
 26 |     $.ajax('/api/exclude/' + id + '/' + exclude_from_delete, {
 27 |       method: 'POST',
 28 |       success: function () {
 29 |         // Update the tweets, and the text
 30 |         window.semiphemeral.tweets[id].excluded = excluded;
 31 |         update_text(excluded);
 32 |       },
 33 |       error: function () {
 34 |         alert('API error');
 35 |       }
 36 |     });
 37 |   }
 38 | 
 39 |   function display_tweets() {
 40 |     console.log('display_tweets', window.semiphemeral);
 41 | 
 42 |     var q = window.semiphemeral.state.q;
 43 |     var page = window.semiphemeral.state.page;
 44 |     var count = window.semiphemeral.state.count;
 45 |     var replies = window.semiphemeral.state.replies;
 46 | 
 47 |     // Build list of ids to filter
 48 |     var ids = [];
 49 |     for (var i = 0; i < window.semiphemeral.ids.length; i++) {
 50 |       var id = window.semiphemeral.ids[i];
 51 |       if (window.semiphemeral.tweets[id].text.toLowerCase().includes(q.toLowerCase())) {
 52 |         if (replies || (!replies && !window.semiphemeral.tweets[id].is_reply)) {
 53 |           ids.push(id);
 54 |         }
 55 |       }
 56 |     }
 57 | 
 58 |     // Empty what previous page
 59 |     $('.info').empty();
 60 |     $('.tweets-to-delete').empty();
 61 |     $('.pagination').empty();
 62 | 
 63 |     // Display info
 64 |     var num_pages = Math.ceil(ids.length / count);
 65 |     var info_string = 'Page ' + comma_formatted(page) + ' of ' + comma_formatted(num_pages) + ' - ';
 66 |     if (ids.length != window.semiphemeral.ids.length) {
 67 |       info_string += 'filtering to ' + comma_formatted(ids.length) + ' tweets - '
 68 |     }
 69 |     info_string += comma_formatted(window.semiphemeral.ids.length) + ' tweets are staged for deletion';
 70 |     $('.info').append($('<div/>').html(info_string));
 71 | 
 72 |     // Pagination controls
 73 |     function add_pagination_item(text, new_page) {
 74 |       var $item = $('<span/>').addClass('pagination-item');
 75 |       if (new_page == page) {
 76 |         $item.addClass('pagination-item-current').text(text);
 77 |       } else {
 78 |         var new_state = { q: q, page: new_page, count: count, replies: true };
 79 |         var $link = $('<a/>').attr('href', '#' + JSON.stringify(new_state)).text(text);
 80 |         $item.append($link);
 81 |       }
 82 | 
 83 |       $('.pagination').append($item);
 84 |     }
 85 |     if (page > 0) {
 86 |       add_pagination_item('Previous', page - 1);
 87 |     }
 88 |     for (var i = page - 5; i < page + 5; i++) {
 89 |       if (i >= 0 && i <= num_pages - 1) {
 90 |         add_pagination_item(i, i);
 91 |       }
 92 |     }
 93 |     if (page < num_pages - 1) {
 94 |       add_pagination_item('Next', page + 1);
 95 |     }
 96 | 
 97 |     // Display the page of tweets
 98 |     for (var i = page * count; i < (page + 1) * count; i++) {
 99 |       var $embed = $('<div/>').prop('id', 'tweet-' + ids[i]);
100 |       var $info = $('<div/>').addClass('info')
101 |         .append(
102 |           $('<label/>')
103 |             .append($('<input type="checkbox">')
104 |               .prop('id', 'toggle-exclude-checkbox-' + ids[i])
105 |               .data('tweet-id', ids[i])
106 |               .change(function () {
107 |                 toggle_exclude($(this).data('tweet-id'), this.checked);
108 |               }))
109 |             .append($('<span/>').addClass('toggle-exclude-label').prop('id', 'toggle-exclude-label-' + ids[i]))
110 |         )
111 |         .append($('<div class="stats">' + window.semiphemeral.tweets[ids[i]].retweets + ' retweets, ' + window.semiphemeral.tweets[ids[i]].likes + ' likes</div>'));
112 | 
113 |       var $tweet = $('<div/>').addClass('tweet').append($info).append($embed);
114 | 
115 |       $('.tweets-to-delete').append($tweet);
116 | 
117 |       // Set the text and checkbox initially
118 |       toggle_exclude(ids[i], window.semiphemeral.tweets[ids[i]].excluded);
119 |       if (window.semiphemeral.tweets[ids[i]].excluded) {
120 |         $('#toggle-exclude-checkbox-' + ids[i]).prop('checked', true);
121 |       }
122 | 
123 |       twttr.widgets.createTweet(ids[i], $('#tweet-' + ids[i])[0], {
124 |         'dnt': true
125 |       });
126 |     }
127 |   }
128 | 
129 |   // Ajax loader
130 |   $('.tweets-to-delete').html('<img src="/static/img/loading.gif" alt="">');
131 | 
132 |   // Load all tweets to delete
133 |   $.get('/api/tweets-to-delete', function (tweets) {
134 |     $('.controls').show();
135 | 
136 |     var ids = [];
137 |     for (var id in tweets) {
138 |       ids.push(id);
139 |     }
140 | 
141 |     window.semiphemeral = {
142 |       tweets: tweets,
143 |       ids: ids
144 |     }
145 | 
146 |     // When the hash contains a JSON object, pass it into display_tweets
147 |     function parse_hash() {
148 |       if (window.location.hash == "") return false;
149 |       try {
150 |         var hash = decodeURIComponent(window.location.hash).substr(1);
151 |         window.semiphemeral.state = JSON.parse(hash);
152 |         $('.filter input').val(window.semiphemeral.state.q);
153 |         $('.options input').prop('checked', window.semiphemeral.state.replies);
154 |       } catch {
155 |         console.log('parsing hash failed', hash);
156 |         return false;
157 |       }
158 | 
159 |       display_tweets();
160 |       return true;
161 |     }
162 | 
163 |     // Watch for changes in the URL hash
164 |     $(window).on('hashchange', function (e) {
165 |       parse_hash();
166 |     });
167 | 
168 |     // Filter search results
169 |     $('.filter input').change(function () {
170 |       var q = $(this).val();
171 |       console.log('filtering on', q);
172 |       if (q != window.semiphemeral.state.count) {
173 |         change_state(q, 0, window.semiphemeral.state.count, window.semiphemeral.state.replies);
174 |       }
175 |     });
176 | 
177 |     // Toggling show replies
178 |     $('.options input').change(function () {
179 |       var replies = $('.options input').prop('checked') ? true : false;
180 |       console.log('show replies', replies);
181 |       if (replies != window.semiphemeral.state.replies) {
182 |         change_state(window.semiphemeral.state.q, 0, window.semiphemeral.state.count, replies);
183 |       }
184 |     });
185 | 
186 |     // Display tweets
187 |     if (!parse_hash()) {
188 |       change_state("", 0, 50, true);
189 |     }
190 |   })
191 | })
192 | 


--------------------------------------------------------------------------------
/semiphemeral/web.py:
--------------------------------------------------------------------------------
  1 | import datetime
  2 | import json
  3 | from flask import Flask, request, render_template, jsonify, abort, redirect
  4 | 
  5 | from .db import Tweet, Thread
  6 | from .twitter import Twitter
  7 | 
  8 | 
  9 | def create_app(common):
 10 |     app = Flask(__name__)
 11 | 
 12 |     @app.route("/")
 13 |     def index():
 14 |         return redirect("/settings")
 15 | 
 16 |     @app.route("/settings", methods=["GET", "POST"])
 17 |     def edit_settings():
 18 |         common.settings.load()
 19 | 
 20 |         if request.method == "POST":
 21 |             common.settings.set("api_key", request.form["api_key"].strip())
 22 |             common.settings.set("api_secret", request.form["api_secret"].strip())
 23 |             common.settings.set(
 24 |                 "access_token_key", request.form["access_token_key"].strip()
 25 |             )
 26 |             common.settings.set(
 27 |                 "access_token_secret", request.form["access_token_secret"].strip()
 28 |             )
 29 |             common.settings.set("username", request.form["username"])
 30 | 
 31 |             if "log_to_file" in request.form:
 32 |                 common.settings.set("logging", request.form["log_to_file"] == "on")
 33 |             else:
 34 |                 common.settings.set("logging", False)
 35 |             common.settings.set("log_filename", request.form["log_filename"])
 36 |             common.settings.set("log_format", request.form["log_format"])
 37 | 
 38 |             if "delete_tweets" in request.form:
 39 |                 common.settings.set(
 40 |                     "delete_tweets", request.form["delete_tweets"] == "on"
 41 |                 )
 42 |             else:
 43 |                 common.settings.set("delete_tweets", False)
 44 |             common.settings.set(
 45 |                 "tweets_days_threshold", int(request.form["tweets_days_threshold"])
 46 |             )
 47 |             common.settings.set(
 48 |                 "tweets_retweet_threshold",
 49 |                 int(request.form["tweets_retweet_threshold"]),
 50 |             )
 51 |             common.settings.set(
 52 |                 "tweets_like_threshold", int(request.form["tweets_like_threshold"])
 53 |             )
 54 |             if "tweets_threads_threshold" in request.form:
 55 |                 common.settings.set(
 56 |                     "tweets_threads_threshold",
 57 |                     request.form["tweets_threads_threshold"] == "on",
 58 |                 )
 59 |             else:
 60 |                 common.settings.set("tweets_threads_threshold", False)
 61 | 
 62 |             if "retweets_likes" in request.form:
 63 |                 common.settings.set(
 64 |                     "retweets_likes", request.form["retweets_likes"] == "on"
 65 |                 )
 66 |             else:
 67 |                 common.settings.set("retweets_likes", False)
 68 |             if "retweets_likes_delete_retweets" in request.form:
 69 |                 common.settings.set(
 70 |                     "retweets_likes_delete_retweets",
 71 |                     request.form["retweets_likes_delete_retweets"] == "on",
 72 |                 )
 73 |             else:
 74 |                 common.settings.set("retweets_likes_delete_retweets", False)
 75 |             common.settings.set(
 76 |                 "retweets_likes_retweets_threshold",
 77 |                 int(request.form["retweets_likes_retweets_threshold"]),
 78 |             )
 79 |             if "retweets_likes_delete_likes" in request.form:
 80 |                 common.settings.set(
 81 |                     "retweets_likes_delete_likes",
 82 |                     request.form["retweets_likes_delete_likes"] == "on",
 83 |                 )
 84 |             else:
 85 |                 common.settings.set("retweets_likes_delete_likes", False)
 86 |             common.settings.set(
 87 |                 "retweets_likes_likes_threshold",
 88 |                 int(request.form["retweets_likes_likes_threshold"]),
 89 |             )
 90 | 
 91 |             if "delete_dms" in request.form:
 92 |                 common.settings.set("delete_dms", request.form["delete_dms"] == "on")
 93 |             else:
 94 |                 common.settings.set("delete_dms", False)
 95 |             common.settings.set(
 96 |                 "dms_days_threshold", int(request.form["dms_days_threshold"])
 97 |             )
 98 | 
 99 |             common.settings.set('proxy', request.form['proxy'].strip())
100 |             common.settings.set('use_tor', 'use_tor' in request.form)
101 | 
102 |             common.settings.save()
103 | 
104 |             if common.settings.is_configured():
105 |                 # Recalculate excluded threads with these new settings
106 |                 twitter = Twitter(common)
107 |                 twitter.calculate_excluded_threads()
108 | 
109 |         return render_template(
110 |             "settings.html",
111 |             api_key=common.settings.get("api_key"),
112 |             api_secret=common.settings.get("api_secret"),
113 |             access_token_key=common.settings.get("access_token_key"),
114 |             access_token_secret=common.settings.get("access_token_secret"),
115 |             username=common.settings.get("username"),
116 |             log_to_file=common.settings.get("logging"),
117 |             log_filename=common.settings.get("log_filename"),
118 |             log_format=common.settings.get("log_format"),
119 |             delete_tweets=common.settings.get("delete_tweets"),
120 |             tweets_days_threshold=common.settings.get("tweets_days_threshold"),
121 |             tweets_retweet_threshold=common.settings.get("tweets_retweet_threshold"),
122 |             tweets_like_threshold=common.settings.get("tweets_like_threshold"),
123 |             tweets_threads_threshold=common.settings.get("tweets_threads_threshold"),
124 |             retweets_likes=common.settings.get("retweets_likes"),
125 |             retweets_likes_delete_retweets=common.settings.get(
126 |                 "retweets_likes_delete_retweets"
127 |             ),
128 |             retweets_likes_retweets_threshold=common.settings.get(
129 |                 "retweets_likes_retweets_threshold"
130 |             ),
131 |             retweets_likes_delete_likes=common.settings.get(
132 |                 "retweets_likes_delete_likes"
133 |             ),
134 |             retweets_likes_likes_threshold=common.settings.get(
135 |                 "retweets_likes_likes_threshold"
136 |             ),
137 |             delete_dms=common.settings.get("delete_dms"),
138 |             dms_days_threshold=common.settings.get("dms_days_threshold"),
139 |             proxy=common.settings.get('proxy'),
140 |             use_tor=common.settings.get('use_tor'),
141 |         )
142 | 
143 |     @app.route("/tweets")
144 |     def tweets():
145 |         return render_template("tweets.html")
146 | 
147 |     @app.route("/api/statistics")
148 |     def api_statistics():
149 |         return jsonify(common.get_stats())
150 | 
151 |     @app.route("/api/tweets-to-delete")
152 |     def api_tweets_to_delete():
153 |         """
154 |         This returns a dictionary of status_ids mapped to the text of all tweets that should be deleted
155 |         """
156 |         tweets_to_delete = common.get_tweets_to_delete(include_excluded=True)
157 | 
158 |         ret = {}
159 |         for tweet in tweets_to_delete:
160 |             if tweet.in_reply_to_status_id:
161 |                 is_reply = True
162 |             else:
163 |                 is_reply = False
164 | 
165 |             ret[tweet.status_id] = {
166 |                 "text": tweet.text,
167 |                 "retweets": tweet.retweet_count,
168 |                 "likes": tweet.favorite_count,
169 |                 "is_reply": is_reply,
170 |                 "excluded": tweet.exclude_from_delete,
171 |             }
172 |         return jsonify(ret)
173 | 
174 |     @app.route(
175 |         "/api/exclude/<int:status_id>/<int:exclude_from_delete>", methods=["POST"]
176 |     )
177 |     def api_exclude(status_id, exclude_from_delete):
178 |         if exclude_from_delete == 1:
179 |             exclude_from_delete = True
180 |         else:
181 |             exclude_from_delete = False
182 | 
183 |         tweet = common.session.query(Tweet).filter_by(status_id=status_id).first()
184 |         if not tweet:
185 |             abort(400)
186 | 
187 |         tweet.exclude_from_delete = exclude_from_delete
188 |         common.session.add(tweet)
189 |         common.session.commit()
190 |         return jsonify(True)
191 | 
192 |     return app
193 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Unfortunately I have shut down the hosted Semiphemeral service, and I no longer plan to maintain this open source version. From the message posted on https://semiphemeral.com/:
  2 | 
  3 | > ### So long and thanks for all the tweets
  4 | > 
  5 | > Hello, this is Semiphemeral 🐦. I used to be a Twitter bot that deleted all of your tweets, likes, and DMs, except for the ones that you wanted to keep.
  6 | > 
  7 | > But sadly, a reactionary billionaire took over my habitat (the Twitter API) and for some inexplicable reason spread toxic waste (hate speech, science denialism, crypto scams, etc.) across it at an impressive speed. It's no longer a livable environment, so unfortunately I was forced to shut down my service.
  8 | > 
  9 | > It was a good run. While I was active, I deleted:
 10 | > 
 11 | > - 45.4 million tweets
 12 | > - 26.2 million retweets
 13 | > - 69.7 million likes
 14 | > - 24.6 million direct messages
 15 | > 
 16 | > I helped a total of 43,049 users. I blocked 248 of them though because they consistently liked tweets from fascists.
 17 | > 
 18 | > And finally, as my last act of privacy protection, I have deleted my database (which contained everything I used to know about my users) and all of its backups, and I have shut down my servers for good.
 19 | > 
 20 | > So long! 💕
 21 | 
 22 | You can find the now-public source code for the hosted service at https://github.com/micahflee/semiphemeral.com.
 23 | 
 24 | If you'd like to see this open source version continue to live on, please fork this project and breath new life into it! It's released under the [MIT license](./LICENSE).
 25 | 
 26 | ---
 27 | 
 28 | ![Logo](/img/logo-small.png)
 29 | 
 30 | # Semiphemeral
 31 | 
 32 | There are two different versions of Semiphemeral:
 33 | 
 34 | - **[Semiphemeral.com](https://semiphemeral.com/) is a hosted service that's easy for anyone to use**, but does not have all the same features as this open source project.
 35 | - **This is the _open source_ version.** It's for advanced users who want total control (non-nerds are better off using semiphemeral.com). It requires using the terminal, generating your own Twitter API key, and if you want to automate it, setting up a server with a cron job. However you don't need to give any access to your Twitter account to a 3rd party (me) and it has more features.
 36 | 
 37 | ## What is this?
 38 | 
 39 | There are plenty of tools that let you make your Twitter feed ephemeral, automatically deleting tweets older than some threshold, like one month.
 40 | 
 41 | Semiphemeral does this, but also lets you automatically exclude tweets based on criteria: how many RTs or likes they have, and if they're part of a thread where one of your tweets has that many RTs or likes. It also lets you manually select tweets you'd like to exclude from deleting.
 42 | 
 43 | It delete all of your old likes, even really old ones that Twitter makes difficult to delete. And it can automatically delete your old direct messages.
 44 | 
 45 | Read more:
 46 | 
 47 | * [Semiphemeral: Automatically delete your old tweets, except for the ones you want to keep](https://micahflee.com/2019/06/semiphemeral-automatically-delete-your-old-tweets-except-for-the-ones-you-want-to-keep/)
 48 | * [With Semiphemeral you can delete your old Twitter likes, but it's noisy](https://micahflee.com/2019/07/with-semiphemeral-you-can-delete-your-old-twitter-likes-but-its-noisy/)
 49 | 
 50 | ## Installation
 51 | 
 52 | ```
 53 | pip3 install semiphemeral
 54 | ```
 55 | 
 56 | ## How it works
 57 | 
 58 | Semiphemeral is a command line tool that you run locally on your computer, or on a server.
 59 | 
 60 | ```
 61 | $ semiphemeral
 62 | Usage: -c [OPTIONS] COMMAND [ARGS]...
 63 | 
 64 |   Automatically delete your old tweets, except for the ones you want to keep
 65 | 
 66 | Options:
 67 |   --help  Show this message and exit.
 68 | 
 69 | Commands:
 70 |   configure        Start the web server to configure semiphemeral
 71 |   delete           Delete tweets that aren't automatically or manually
 72 |                    excluded, likes, and DMs
 73 | 
 74 |   delete_dms       Delete DMs that aren't available through the Twitter API
 75 |   excluded_export  Export tweets excluded that are excluded from deletion
 76 |   excluded_import  Import tweets excluded that are excluded from deletion
 77 |   fetch            Download all tweets/DMs
 78 |   stats            Show stats about tweets in the database
 79 |   unlike           Delete old likes that aren't available through the Twitter
 80 |                    API
 81 | ```
 82 | 
 83 | Start by running `semiphemeral configure`, which starts a local web server at http://127.0.0.1:8080/. Load that website in a browser.
 84 | 
 85 | You must supply Twitter API credentials here, which you can get by following [this guide](https://python-twitter.readthedocs.io/en/latest/getting_started.html). Basically, you need to login to https://developer.twitter.com/ and create a new "Twitter app" that only you will be using (when creating an app, you're welcome to use https://github.com/micahflee/semiphemeral as the website URL for your app).
 86 | 
 87 | If you want to delete your DMs, you'll have to make sure to give your Twitter app "Read, write, and Direct Messages" permissions, instead of just "Read and write". (If you're modifying an existing app to add Direct Message permissions, you'll have to go to the app's "Keys and tokens" page, and under "Access token & access token secret" click "Regenerate". Then add the new access token key and secret into semiphemeral's settings.)
 88 | 
 89 | From the settings page you also tell semiphemeral which tweets to exclude from deletion:
 90 | 
 91 | ![Settings](/img/settings.png)
 92 | 
 93 | Once you have configured semiphemeral, fetch all of the tweets from your account by running `semiphemeral fetch`. (It may take a long time if you have a lot of tweets -- when semiphemeral hits a Twitter rate limit, it just waits the shortest amount of time allowed until it can continue fetching.)
 94 | 
 95 | Then go back to the configuration web app and look at the tweets page. From here, you can look at all of the tweets that are going to get deleted the next time you run `semiphemeral delete`, and choose to manually exclude some of them from deletion. This interface paginates all of the tweets that are staged for deletion, and allows you to filter them by searching for phrases in the text of your tweets.
 96 | 
 97 | Once you have chosen all tweets you want to exclude, you may want to [download your Twitter archive](https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive) for your records.
 98 | 
 99 | Then run `semiphemeral delete` (this also fetches latest tweets before deleting). The first time it might take a long time. Like with fetching, it will wait when it hits a Twitter rate limit. Let it run once first before automating it.
100 | 
101 | After you have manually deleted once, you can automatically delete your old tweets by running `semiphemeral delete` once a day in a cron job.
102 | 
103 | Settings are stored in `~/.semiphemeral/settings.json`. All tweets (including exceptions, and deleted tweets) are stored in a sqlite database `~/.semiphemeral/tweets.db`.
104 | 
105 | ## Deleting old direct messages
106 | 
107 | The Twitter API only lets you fetch your last 30 days worth of DMs. If you have years worth of old DMs, you can still delete them all.
108 | 
109 | This is how to use the `delete_dms` command:
110 | 
111 | ```sh
112 | $ semiphemeral delete_dms --help
113 | Usage: semiphemeral delete_dms [OPTIONS]
114 | 
115 | Options:
116 |   --filename TEXT  Path to direct-message.js from Twitter data downloaded from
117 |                    https://twitter.com/settings/your_twitter_data  [required]
118 | 
119 |   --help           Show this message and exit.
120 | ```
121 | 
122 | In order to get a list of all of your DMs (since the Twitter API won't give it to you), you must go to https://twitter.com/settings/your_twitter_data and download your Twitter data (note that this is different than your "Twitter archive", which doesn't include information about your DMs). Twitter will email you a link to a zip file. When you unzip it there will be many files, including a file called `direct-message.js`. Run this command, with the path to your `direct-message.js`, for example:
123 | 
124 | ```sh
125 | semiphemeral delete_dms --filename ~/Downloads/twitter-2020-07-22/direct-message.js
126 | ```
127 | 
128 | Your filename will be different than this one, so make sure you update the command to match it.
129 | 
130 | This will delete all of your old DMs. New DMs don't have this problem. So as long as you regularly run `semiphemeral delete`, your new DMs will automatically get deleted.
131 | 
132 | ## Deleting old tweets
133 | 
134 | Semiphemeral deleted about 20,000 of my tweets, starting from my earliest in 2009. But I've heard reports that for some people, Semiphemeral only deletes about 3,000 tweets and can't delete anything earlier.
135 | 
136 | I don't know why the Twitter API works the way it does, but unfortunately for some users when you say, "hey Twitter, give me a list of all my tweets" it only gives you a list of the most recent 3,000. And if you delete all of those, it gives you a list of 0, even if you have way more.
137 | 
138 | _At the moment, Semiphemeral can't help you solve this problem._
139 | 
140 | ## Deleting old likes
141 | 
142 | The Twitter API is only willing to tell you about your last 4000 likes. If you've already tried to fetch and delete your likes, but still have a lot of old likes, you can use semiphemeral to automate unliking them.
143 | 
144 | _**WARNING: One does not simply unlike old tweets.** Twitter works in mysterious ways. For some reason, even though these old tweets are listed in your like history, the API doesn't believe that you actually liked them. The only way to remove them from your like history is to LIKE THEM AGAIN, and then you can unlike them, and they actually get removed from your like history, and your like count goes down. But this is VERY NOISY. Every time you re-like a tweet, the user will get a notification. If you need to unlike many thousands of old likes, expect messages from friends thinking your Twitter account was hacked or something._
145 | 
146 | _**WARNING: Prepare to spend WEEKS running this command.** Twitter only allows users to like up to 1000 tweets per day. This means that if you need to unlike 15,000 old tweets, then this script will take 15 days to run! After the first 1000 like/unlikes, the script will pause for 24 hours. If possible, I recommend you run this on a server in a screen or tmux session, so that it doesn't get interrupted when you suspend your laptop. And if you're running semiphemeral on a cron job, I recommend that you disable it first, and then start it up again when it's finished._
147 | 
148 | This is how to use the `unlike` command:
149 | 
150 | ```sh
151 | $ semiphemeral unlike --help
152 | Usage: semiphemeral unlike [OPTIONS]
153 | 
154 | Options:
155 |   --filename TEXT  Path to like.js from Twitter data downloaded from
156 |                    https://twitter.com/settings/your_twitter_data  [required]
157 | 
158 |   --help           Show this message and exit.
159 | ```
160 | 
161 | In order to get a list of all of your old likes (since the Twitter API won't give it to you), you must go to https://twitter.com/settings/your_twitter_data and download your Twitter data (note that this is different than your "Twitter archive", which doesn't include information about your likes). Twitter will email you a link to a zip file. When you unzip it there will be many files, including a file called `like.js`. Run this command, with the path to your `like.js`, for example:
162 | 
163 | ```sh
164 | semiphemeral unlike --filename ~/Downloads/twitter-2020-07-22/like.js
165 | ```
166 | 
167 | Your filename will be different than this one, so make sure you update the command to match it.
168 | 
169 | This will fetch all of the old tweets you liked a long time ago, and then relike and unlike each one of them. Every relike will cause a notification, but at the end of the process your likes will have actually been deleted. If the command crashes or you cancel in the middle for any reason, it's safe to run it again to continue where you left off.
170 | 
171 | New likes don't have this problem, so as long as you regularly run `semiphemeral delete`, your new likes will automatically get deleted.
172 | 
173 | ## Development
174 | 
175 | Make sure you have [pipenv](https://pipenv.readthedocs.io/en/latest/). Then install dependencies:
176 | 
177 | ```sh
178 | pipenv install --dev --pre
179 | ```
180 | 
181 | And run the program like this:
182 | 
183 | ```sh
184 | pipenv run python ./app.py --help
185 | ```
186 | 


--------------------------------------------------------------------------------
/semiphemeral/twitter.py:
--------------------------------------------------------------------------------
  1 | import tweepy
  2 | import click
  3 | import json
  4 | import datetime
  5 | import os
  6 | import io
  7 | import time
  8 | from zipfile import ZipFile
  9 | 
 10 | from tweepy.models import Status
 11 | from .db import Tweet, Thread
 12 | 
 13 | 
 14 | class Twitter(object):
 15 |     def __init__(self, common):
 16 |         self.common = common
 17 | 
 18 |         self.authenticated = False
 19 | 
 20 |         if not self.common.settings.is_configured():
 21 |             click.echo(
 22 |                 "Twitter API is not configured yet, configure it with --configure"
 23 |             )
 24 |             return
 25 | 
 26 |         # Check for tweepy version at run-time to avoid later crashes.
 27 |         if not tweepy.__version__.startswith('3.10.'):
 28 |             click.echo(
 29 |                 "Python package 'tweepy' must be version 3.10.x"
 30 |             )
 31 |             return
 32 | 
 33 |         auth = tweepy.OAuthHandler(
 34 |             self.common.settings.get("api_key"), self.common.settings.get("api_secret")
 35 |         )
 36 |         auth.set_access_token(
 37 |             self.common.settings.get("access_token_key"),
 38 |             self.common.settings.get("access_token_secret"),
 39 |         )
 40 |         proxy=self.common.settings.get('proxy')
 41 |         if proxy == "" or proxy == "None":
 42 |             proxy = None
 43 |         self.api = tweepy.API(
 44 |             auth,
 45 |             wait_on_rate_limit=True,
 46 |             wait_on_rate_limit_notify=True,
 47 |             proxy=proxy,
 48 |         )
 49 | 
 50 |         self.authenticated = True
 51 | 
 52 |         # Make sure we've saved the user id
 53 |         user = self.api.get_user(self.common.settings.get("username"))
 54 |         if user:
 55 |             self.common.settings.set("user_id", user.id)
 56 |             self.common.settings.save()
 57 | 
 58 |         # Date format for saving last_fetch setting
 59 |         self.last_fetch_format = "%Y-%m-%d %I:%M%p"
 60 | 
 61 |     def stats(self):
 62 |         click.secho("Statistics", fg="cyan")
 63 |         stats = self.common.get_stats()
 64 |         click.echo(json.dumps(stats, indent=2))
 65 | 
 66 |         if self.common.settings.get("delete_tweets"):
 67 |             tweets_to_delete = self.common.get_tweets_to_delete()
 68 |             click.secho(
 69 |                 "Want to delete {} tweets".format(len(tweets_to_delete)),
 70 |                 fg="cyan",
 71 |             )
 72 | 
 73 | 
 74 |     def fetch(self):
 75 |         if not self.authenticated:
 76 |             return
 77 | 
 78 |         if self.common.settings.get("delete_tweets"):
 79 |             # We fetch tweets since the last fetch (or all tweets, if it's None)
 80 |             since_id = self.common.settings.get("since_id")
 81 |             if since_id:
 82 |                 click.secho("Fetching all recent tweets", fg="cyan")
 83 |             else:
 84 |                 click.secho(
 85 |                     "Fetching all tweets, this first run may take a long time",
 86 |                     fg="cyan",
 87 |                 )
 88 | 
 89 |             # Fetch tweets from timeline a page at a time
 90 |             for page in tweepy.Cursor(
 91 |                 self.api.user_timeline,
 92 |                 id=self.common.settings.get("username"),
 93 |                 since_id=since_id,
 94 |                 tweet_mode="extended",
 95 |             ).pages():
 96 |                 fetched_count = 0
 97 | 
 98 |                 # Import these tweets, and all their threads
 99 |                 for status in page:
100 |                     fetched_count += self.import_tweet_and_thread(Tweet(status))
101 | 
102 |                     # Only commit every 20 tweets
103 |                     if fetched_count % 20 == 0:
104 |                         self.common.session.commit()
105 | 
106 |                 # Commit the leftovers
107 |                 self.common.session.commit()
108 | 
109 |                 # Now hunt for threads. This is a dict that maps the root status_id
110 |                 # to a list of status_ids in the thread
111 |                 threads = {}
112 |                 for status in page:
113 |                     if status.in_reply_to_status_id:
114 |                         status_ids = self.calculate_thread(status.id)
115 |                         root_status_id = status_ids[0]
116 |                         if root_status_id in threads:
117 |                             for status_id in status_ids:
118 |                                 if status_id not in threads[root_status_id]:
119 |                                     threads[root_status_id].append(status_id)
120 |                         else:
121 |                             threads[root_status_id] = status_ids
122 | 
123 |                 # For each thread, does this thread already exist, or do we create a new one?
124 |                 for root_status_id in threads:
125 |                     status_ids = threads[root_status_id]
126 |                     thread = (
127 |                         self.common.session.query(Thread)
128 |                         .filter_by(root_status_id=root_status_id)
129 |                         .first()
130 |                     )
131 |                     if not thread:
132 |                         thread = Thread(root_status_id)
133 |                         count = 0
134 |                         for status_id in status_ids:
135 |                             tweet = (
136 |                                 self.common.session.query(Tweet)
137 |                                 .filter_by(status_id=status_id)
138 |                                 .first()
139 |                             )
140 |                             if tweet:
141 |                                 thread.tweets.append(tweet)
142 |                                 count += 1
143 |                         if count > 0:
144 |                             click.echo(
145 |                                 "Added new thread with {} tweets (root id={})".format(
146 |                                     count, root_status_id
147 |                                 )
148 |                             )
149 |                     else:
150 |                         count = 0
151 |                         for status_id in status_ids:
152 |                             tweet = (
153 |                                 self.common.session.query(Tweet)
154 |                                 .filter_by(status_id=status_id)
155 |                                 .first()
156 |                             )
157 |                             if tweet and tweet not in thread.tweets:
158 |                                 thread.tweets.append(tweet)
159 |                                 count += 1
160 |                         if count > 0:
161 |                             click.echo(
162 |                                 "Added {} tweets to existing thread (root id={})".format(
163 |                                     count, root_status_id
164 |                                 )
165 |                             )
166 |                     self.common.session.commit()
167 | 
168 |         if self.common.settings.get("retweets_likes") and self.common.settings.get(
169 |             "retweets_likes_delete_likes"
170 |         ):
171 |             like_since_id = self.common.settings.get("since_id")
172 | 
173 |             # Fetch tweets that are liked
174 |             click.secho("Fetching tweets that you liked", fg="cyan")
175 |             for page in tweepy.Cursor(
176 |                 self.api.favorites,
177 |                 id=self.common.settings.get("username"),
178 |                 since_id=like_since_id,
179 |                 tweet_mode="extended",
180 |             ).pages():
181 |                 # Import these tweets
182 |                 for status in page:
183 |                     tweet = Tweet(status)
184 |                     if not tweet.already_saved(self.common.session):
185 |                         tweet.fetch_summarize()
186 |                         self.common.session.add(tweet)
187 |                 # Commit a page of tweets at a time
188 |                 self.common.session.commit()
189 | 
190 |             # All done, update the since_id
191 |             tweet = (
192 |                 self.common.session.query(Tweet)
193 |                 .order_by(Tweet.status_id.desc())
194 |                 .first()
195 |             )
196 |             if tweet:
197 |                 self.common.settings.set("since_id", tweet.status_id)
198 |                 self.common.settings.save()
199 | 
200 |         # Calculate which threads should be excluded from deletion
201 |         self.calculate_excluded_threads()
202 | 
203 |         self.common.settings.set(
204 |             "last_fetch", datetime.datetime.today().strftime(self.last_fetch_format)
205 |         )
206 |         self.common.settings.save()
207 | 
208 |         self.common.log(
209 |             "last_fetch: %s"
210 |             % datetime.datetime.today().strftime(self.last_fetch_format)
211 |         )
212 | 
213 |     def calculate_thread(self, status_id):
214 |         """
215 |         Given a tweet, recursively add its parents to a thread. In this end, the first
216 |         element of the list should be the root of the thread
217 |         """
218 |         tweet = self.common.session.query(Tweet).filter_by(status_id=status_id).first()
219 |         if not tweet:
220 |             return []
221 |         if not tweet.in_reply_to_status_id:
222 |             return [status_id]
223 |         return self.calculate_thread(tweet.in_reply_to_status_id) + [status_id]
224 | 
225 |     def import_tweet_and_thread(self, tweet):
226 |         """
227 |         This imports a tweet, and recursively imports all tweets that it's in reply to,
228 |         and returns the number of tweets fetched
229 |         """
230 |         fetched_count = 0
231 | 
232 |         # Save the tweet, if it's not already saved
233 |         if not tweet.already_saved(self.common.session):
234 |             tweet.fetch_summarize()
235 |             fetched_count += 1
236 |             self.common.session.add(tweet)
237 | 
238 |         # Is this tweet a reply?
239 |         if tweet.in_reply_to_status_id:
240 |             # Do we already have the parent tweet?
241 |             parent_tweet = (
242 |                 self.common.session.query(Tweet)
243 |                 .filter_by(status_id=tweet.in_reply_to_status_id)
244 |                 .first()
245 |             )
246 |             if not parent_tweet:
247 |                 # If not, import it
248 |                 try:
249 | 
250 |                     status = self.api.get_status(
251 |                         tweet.in_reply_to_status_id, tweet_mode="extended"
252 |                     )
253 |                     fetched_count += self.import_tweet_and_thread(Tweet(status))
254 |                 except tweepy.error.TweepError:
255 |                     # If it's been deleted, ignore
256 |                     pass
257 | 
258 |         return fetched_count
259 | 
260 |     def calculate_excluded_threads(self):
261 |         """
262 |         Based on the current settings, figure out which threads should be excluded from
263 |         deletion, and which threads should have their tweets deleted
264 |         """
265 |         click.secho("Calculating which threads should be excluded", fg="cyan")
266 | 
267 |         # Reset the should_exclude flag for all threads
268 |         self.common.session.query(Thread).update({"should_exclude": False})
269 |         self.common.session.commit()
270 | 
271 |         # Set should_exclude for all threads based on the settings
272 |         if self.common.settings.get("tweets_threads_threshold"):
273 |             threads = (
274 |                 self.common.session.query(Thread)
275 |                 .join(Thread.tweets, aliased=True)
276 |                 .filter(Tweet.user_id == int(self.common.settings.get("user_id")))
277 |                 .filter(Tweet.is_deleted == 0)
278 |                 .filter(Tweet.is_retweet == 0)
279 |                 .filter(
280 |                     Tweet.retweet_count
281 |                     >= self.common.settings.get("tweets_retweet_threshold")
282 |                 )
283 |                 .filter(
284 |                     Tweet.favorite_count
285 |                     >= self.common.settings.get("tweets_like_threshold")
286 |                 )
287 |                 .all()
288 |             )
289 |             for thread in threads:
290 |                 thread.should_exclude = True
291 |             self.common.session.commit()
292 | 
293 |     def delete(self):
294 |         if not self.authenticated:
295 |             return
296 | 
297 |         # First, run fetch
298 |         click.secho("Before deleting anything, fetch", fg="cyan")
299 |         self.fetch()
300 | 
301 |         # Unretweet and unlike tweets
302 |         if self.common.settings.get("retweets_likes"):
303 |             # Unretweet
304 |             if self.common.settings.get("retweets_likes_delete_retweets"):
305 |                 datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
306 |                     days=self.common.settings.get("retweets_likes_retweets_threshold")
307 |                 )
308 |                 tweets = (
309 |                     self.common.session.query(Tweet)
310 |                     .filter(Tweet.user_id == int(self.common.settings.get("user_id")))
311 |                     .filter(Tweet.is_deleted == 0)
312 |                     .filter(Tweet.is_retweet == 1)
313 |                     .filter(Tweet.created_at < datetime_threshold)
314 |                     .order_by(Tweet.created_at)
315 |                     .all()
316 |                 )
317 | 
318 |                 click.secho(
319 |                     "Deleting {} retweets, starting with the earliest".format(
320 |                         len(tweets)
321 |                     ),
322 |                     fg="cyan",
323 |                 )
324 | 
325 |                 count = 0
326 |                 for tweet in tweets:
327 |                     try:
328 |                         self.api.destroy_status(tweet.status_id)
329 |                         tweet.unretweet_summarize()
330 |                         tweet.is_deleted = True
331 |                         self.common.session.add(tweet)
332 |                     except tweepy.error.TweepError as e:
333 |                         if e.api_code == 144:
334 |                             click.echo(
335 |                                 "Error, retweet {} is already deleted, updating database".format(
336 |                                     tweet.status_id
337 |                                 )
338 |                             )
339 |                             tweet.is_deleted = True
340 |                             self.common.session.add(tweet)
341 |                         else:
342 |                             click.echo(
343 |                                 "Error for tweet {}: {}".format(tweet.status_id, e)
344 |                             )
345 | 
346 |                     count += 1
347 |                     if count % 20 == 0:
348 |                         self.common.session.commit()
349 | 
350 |                 self.common.session.commit()
351 |                 self.common.log("Deleted %s retweets" % count)
352 | 
353 |             # Unlike
354 |             if self.common.settings.get("retweets_likes_delete_likes"):
355 |                 datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
356 |                     days=self.common.settings.get("retweets_likes_likes_threshold")
357 |                 )
358 |                 tweets = (
359 |                     self.common.session.query(Tweet)
360 |                     .filter(Tweet.user_id != int(self.common.settings.get("user_id")))
361 |                     .filter(Tweet.is_unliked == False)
362 |                     .filter(Tweet.favorited == True)
363 |                     .filter(Tweet.created_at < datetime_threshold)
364 |                     .order_by(Tweet.created_at)
365 |                     .all()
366 |                 )
367 | 
368 |                 click.secho(
369 |                     "Unliking {} tweets, starting with the earliest".format(
370 |                         len(tweets)
371 |                     ),
372 |                     fg="cyan",
373 |                 )
374 | 
375 |                 count = 0
376 |                 for tweet in tweets:
377 |                     try:
378 |                         self.api.destroy_favorite(tweet.status_id)
379 |                         tweet.unlike_summarize()
380 |                         tweet.is_unliked = True
381 |                         self.common.session.add(tweet)
382 |                     except tweepy.error.TweepError as e:
383 |                         if e.api_code == 144:
384 |                             click.echo(
385 |                                 "Error, tweet {} is already unliked, updating database".format(
386 |                                     tweet.status_id
387 |                                 )
388 |                             )
389 |                             tweet.is_unliked = True
390 |                             self.common.session.add(tweet)
391 |                         else:
392 |                             click.echo(
393 |                                 "Error for tweet {}: {}".format(tweet.status_id, e)
394 |                             )
395 | 
396 |                     count += 1
397 |                     if count % 20 == 0:
398 |                         self.common.session.commit()
399 | 
400 |                 self.common.session.commit()
401 |                 self.common.log("Unliked %s tweets" % count)
402 | 
403 |         # Deleting tweets
404 |         if self.common.settings.get("delete_tweets"):
405 |             tweets_to_delete = self.common.get_tweets_to_delete()
406 | 
407 |             click.secho(
408 |                 "Deleting {} tweets, starting with the earliest".format(
409 |                     len(tweets_to_delete)
410 |                 ),
411 |                 fg="cyan",
412 |             )
413 | 
414 |             count = 0
415 |             for tweet in tweets_to_delete:
416 |                 try:
417 |                     self.api.destroy_status(tweet.status_id)
418 |                     tweet.delete_summarize()
419 |                     tweet.is_deleted = True
420 |                     self.common.session.add(tweet)
421 |                 except tweepy.error.TweepError as e:
422 |                     if e.api_code == 144:
423 |                         click.echo(
424 |                             "Error, tweet {} is already deleted, updating database".format(
425 |                                 tweet.status_id
426 |                             )
427 |                         )
428 |                         tweet.is_deleted = True
429 |                         self.common.session.add(tweet)
430 |                     else:
431 |                         click.echo("Error for tweet {}: {}".format(tweet.status_id, e))
432 | 
433 |                 count += 1
434 |                 if count % 20 == 0:
435 |                     self.common.session.commit()
436 | 
437 |             self.common.session.commit()
438 |             self.common.log("Deleted %s tweets" % count)
439 | 
440 |         # Delete DMs
441 |         if self.common.settings.get("delete_dms"):
442 |             datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
443 |                 days=self.common.settings.get("dms_days_threshold")
444 |             )
445 | 
446 |             # Sadly, only the last 30 days worth
447 |             # https://developer.twitter.com/en/docs/direct-messages/sending-and-receiving/api-reference/list-events
448 |             click.secho(
449 |                 "Fetching direct message metadata for the last 30 days", fg="cyan"
450 |             )
451 | 
452 |             # Fetch direct messages
453 |             count = 0
454 |             for page in tweepy.Cursor(self.api.list_direct_messages).pages():
455 |                 # Twitter will return an apparently unlimited number of empty DM responses
456 |                 # in certain cases, which will hit the rate limit very quickly (15 per 15
457 |                 # minutes, currently). Exit if we get an empty response.
458 |                 if not page:
459 |                     click.secho(
460 |                         "No more accessible DMs", fg="cyan"
461 |                     )
462 |                     break
463 | 
464 |                 for dm in page:
465 |                     created_timestamp = datetime.datetime.fromtimestamp(
466 |                         int(dm.created_timestamp) / 1000
467 |                     )
468 |                     if created_timestamp <= datetime_threshold:
469 |                         self.api.destroy_direct_message(dm.id)
470 |                         click.echo(
471 |                             "Deleted DM {}, id {}".format(
472 |                                 created_timestamp.strftime("%Y-%m-%d"), dm.id
473 |                             )
474 |                         )
475 |                         count += 1
476 |                     else:
477 |                         click.secho(
478 |                             "Skipping DM {}, id {}".format(
479 |                                 created_timestamp.strftime("%Y-%m-%d"), dm.id
480 |                             ),
481 |                             dim=True,
482 |                         )
483 | 
484 |             self.common.log("Deleted %s DMs" % count)
485 | 
486 |     def unlike(self, filename):
487 |         # Validate filename
488 |         filename = os.path.abspath(filename)
489 |         if not os.path.isfile(filename):
490 |             click.echo("Invalid file")
491 |             return
492 |         if os.path.basename(filename) != "like.js":
493 |             click.echo("File should be called like.js")
494 |             return
495 | 
496 |         # Validate file format
497 |         with open(filename) as f:
498 |             expected_start = "window.YTD.like.part0 = "
499 |             js_string = f.read()
500 |             if not js_string.startswith(expected_start):
501 |                 click.echo("File expected to start with: `window.YTD.like.part0 = `")
502 |                 return
503 |             json_string = js_string[len(expected_start) :]
504 |             try:
505 |                 likes = json.loads(json_string)
506 |             except:
507 |                 click.echo("Failed parsing JSON object")
508 |                 return
509 |             if type(likes) != list:
510 |                 click.echo("JSON object expected to be a list")
511 |                 return
512 | 
513 |             for obj in likes:
514 |                 if type(obj) != dict:
515 |                     click.echo("JSON object expected to be a list of dicts")
516 |                     return
517 |                 if "like" not in obj:
518 |                     click.echo(
519 |                         "JSON object expected to be a list of dicts that contain 'like' fields"
520 |                     )
521 |                     return
522 |                 like = obj["like"]
523 |                 if "tweetId" not in like:
524 |                     click.echo(
525 |                         "JSON object expected to be a list of dicts that contain 'like' fields that contain 'tweetID' fields"
526 |                     )
527 |                     return
528 | 
529 |         if not self.authenticated:
530 |             return
531 | 
532 |         # Get permission because the following is very noisy
533 |         click.secho(
534 |             "WARNING: One does not simply unlike old tweets. According to the Twitter API, you didn't like these old tweets, so you can't unlike them -- even though they're listed in your like history. The only way to remove them from your like history is to LIKE THEM AGAIN, and then you can unlike them. This is very noisy. Every time you re-like a tweet, the user will get a notification.",
535 |             fg="red",
536 |         )
537 |         if not click.confirm("Do you want to continue?"):
538 |             return
539 | 
540 |         # Make a list of liked tweet status_ids
541 |         click.secho(
542 |             "Making a list of liked tweet status_ids".format(len(likes)), fg="cyan"
543 |         )
544 |         like_status_ids = []
545 |         for obj in likes:
546 |             status_id = int(obj["like"]["tweetId"])
547 |             if not self.common.settings.unlike_should_ignore(status_id):
548 |                 like_status_ids.append(status_id)
549 |         click.secho(
550 |             "Like history has {} tweets".format(len(like_status_ids)), fg="cyan"
551 |         )
552 | 
553 |         datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
554 |             days=self.common.settings.get("retweets_likes_likes_threshold")
555 |         )
556 |         all_tweets = []
557 | 
558 |         # Load tweets from db first
559 |         click.secho("Loading tweets from database", fg="cyan")
560 |         loaded_status_ids = []
561 |         for tweet in (
562 |             self.common.session.query(Tweet)
563 |             .filter(Tweet.status_id.in_(like_status_ids))
564 |             .order_by(Tweet.created_at.desc())
565 |             .all()
566 |         ):
567 |             if tweet.created_at < datetime_threshold:
568 |                 all_tweets.append(tweet)
569 |             loaded_status_ids.append(tweet.status_id)
570 |         click.secho("Loaded {} tweets from database".format(len(all_tweets)), fg="cyan")
571 | 
572 |         click.secho("Calculating tweets to fetch from the API", fg="cyan")
573 |         remaining_status_ids = []
574 |         for status_id in like_status_ids:
575 |             if status_id not in loaded_status_ids:
576 |                 remaining_status_ids.append(status_id)
577 | 
578 |         # Fetch remaining tweets
579 |         if len(remaining_status_ids) > 0:
580 |             click.secho(
581 |                 "Fetching remaining {} tweets from API".format(
582 |                     len(remaining_status_ids)
583 |                 ),
584 |                 fg="cyan",
585 |             )
586 |             count = 0
587 |             for status_id in remaining_status_ids:
588 |                 try:
589 |                     status = self.api.get_status(status_id, tweet_mode="extended")
590 |                     tweet = Tweet(status)
591 |                     if not tweet.already_saved(self.common.session):
592 |                         tweet.fetch_summarize()
593 |                         self.common.session.add(tweet)
594 |                         count += 1
595 | 
596 |                         if tweet.created_at < datetime_threshold:
597 |                             all_tweets.append(tweet)
598 |                 except tweepy.error.TweepError as e:
599 |                     click.secho(
600 |                         "Error importing tweet {}: {}".format(status_id, e), dim=True
601 |                     )
602 |                     self.common.settings.unlike_ignore(status_id)
603 | 
604 |                 if count % 20 == 0:
605 |                     self.common.session.commit()
606 | 
607 |             self.common.session.commit()
608 | 
609 |         # Figure out how many tweets we actually need to relike
610 |         click.secho("Calculating how many tweets to re-like and unlike", fg="cyan")
611 |         tweets = []
612 |         for tweet in all_tweets:
613 |             if tweet.created_at < datetime_threshold and not tweet.is_unliked:
614 |                 if tweet.favorited:
615 |                     try:
616 |                         self.api.destroy_favorite(tweet.status_id)
617 |                         tweet.unlike_summarize()
618 |                         tweet.is_unliked = True
619 |                         self.common.session.add(tweet)
620 |                     except tweepy.error.TweepError as e:
621 |                         click.secho(
622 |                             "Error unliking tweet {}: {}".format(tweet.status_id, e),
623 |                             dim=True,
624 |                         )
625 |                 else:
626 |                     tweets.append(tweet)
627 | 
628 |         # Leftover tweets to re-like and unlike, that we do at the end because of hitting rate limits
629 |         self.extra_tweets = []
630 | 
631 |         self.relike_unlike_tweets(datetime_threshold, tweets)
632 |         self.relike_unlike_tweets(datetime_threshold, self.extra_tweets)
633 | 
634 |     def relike_unlike_tweets(self, datetime_threshold, tweets):
635 |         # Re-like and unlike each tweet
636 |         click.secho(
637 |             "Re-liking and unliking {} liked tweets".format(len(tweets)), fg="cyan"
638 |         )
639 |         count = 0
640 |         for tweet in tweets:
641 |             if tweet.created_at < datetime_threshold and not tweet.is_unliked:
642 |                 if tweet.favorited:
643 |                     try:
644 |                         self.api.destroy_favorite(tweet.status_id)
645 |                         tweet.unlike_summarize()
646 |                         tweet.is_unliked = True
647 |                         self.common.session.add(tweet)
648 |                     except tweepy.error.TweepError as e:
649 |                         click.secho(
650 |                             "Error unliking tweet {}: {}".format(tweet.status_id, e),
651 |                             dim=True,
652 |                         )
653 |                 else:
654 |                     try:
655 |                         self.api.create_favorite(tweet.status_id)
656 |                         try:
657 |                             self.api.destroy_favorite(tweet.status_id)
658 |                             tweet.relike_unlike_summarize()
659 |                             tweet.is_unliked = True
660 |                             self.common.session.add(tweet)
661 |                         except tweepy.error.TweepError as e:
662 |                             click.secho(
663 |                                 "Error unliking tweet {}: {}".format(
664 |                                     tweet.status_id, e
665 |                                 ),
666 |                                 dim=True,
667 |                             )
668 |                     except tweepy.error.TweepError as e:
669 |                         click.secho(
670 |                             "Error liking tweet {}: {}".format(tweet.status_id, e),
671 |                             dim=True,
672 |                         )
673 | 
674 |                         if "status code = 429" in str(e) and e.api_code == None:
675 |                             self.common.session.commit()
676 |                             self.extra_tweets.append(tweet)
677 | 
678 |                             click.secho(
679 |                                 "You can only like 1000 tweets per 24 hours and you have hit the limit",
680 |                                 bold=True,
681 |                             )
682 |                             click.echo(
683 |                                 "See: https://developer.twitter.com/en/docs/basics/rate-limits"
684 |                             )
685 |                             click.echo(
686 |                                 "Current time: {}".format(
687 |                                     time.strftime("%b %d, %Y %l:%M%p %Z")
688 |                                 )
689 |                             )
690 |                             click.secho("Waiting 24 hours...", bold=True)
691 | 
692 |                             for hours_left in list(range(1, 25))[::-1]:
693 |                                 click.secho("{} hours left".format(hours_left))
694 |                                 time.sleep(3600)
695 | 
696 |                             click.secho("Waiting 2 more minutes, for good measure")
697 |                             time.sleep(120)
698 | 
699 |                 count += 1
700 | 
701 |             if count % 20 == 0:
702 |                 self.common.session.commit()
703 | 
704 |         self.common.session.commit()
705 |         self.common.log("Reliked and unliked %s tweets" % count)
706 | 
707 |     def delete_dms(self, filename):
708 |         # Validate filename
709 |         filename = os.path.abspath(filename)
710 |         if not os.path.isfile(filename):
711 |             click.echo("Invalid file")
712 |             return
713 |         if os.path.basename(filename) not in (
714 |             "direct-message.js",
715 |             "direct-messages.js",
716 |         ):
717 |             click.echo("File should be called direct-message.js or direct-messages.js")
718 |             return
719 | 
720 |         # Validate file format
721 |         with open(filename) as f:
722 |             expected_start = "window.YTD.direct_message"
723 |             js_string = f.read()
724 |             if not js_string.startswith(expected_start):
725 |                 click.echo("File expected to start with: `window.YTD.direct_message`")
726 |                 return
727 |             json_string = js_string[js_string.find("[") :]
728 |             try:
729 |                 conversations = json.loads(json_string)
730 |             except:
731 |                 click.echo("Failed parsing JSON object")
732 |                 return
733 |             if type(conversations) != list:
734 |                 click.echo("JSON object expected to be a list")
735 |                 return
736 | 
737 |             for obj in conversations:
738 |                 if type(obj) != dict:
739 |                     click.echo("JSON object expected to be a list of dicts")
740 |                     return
741 |                 if "dmConversation" not in obj:
742 |                     click.echo(
743 |                         "JSON object expected to be a list of dicts that contain 'dmConversation' fields"
744 |                     )
745 |                     return
746 |                 dm_conversation = obj["dmConversation"]
747 |                 if "messages" not in dm_conversation:
748 |                     click.echo(
749 |                         "JSON object expected to be a list of dicts that contain 'dmConversations' fields that contain 'messages' fields"
750 |                     )
751 |                     return
752 | 
753 |         if not self.authenticated:
754 |             return
755 | 
756 |         # Make a list of DM ids to delete
757 |         datetime_threshold = datetime.datetime.utcnow() - datetime.timedelta(
758 |             days=self.common.settings.get("dms_days_threshold")
759 |         )
760 |         count = 0
761 |         dm_ids = []
762 |         for obj in conversations:
763 |             for message in obj["dmConversation"]["messages"]:
764 |                 created_str = message["messageCreate"]["createdAt"]
765 |                 created_timestamp = datetime.datetime.strptime(
766 |                     created_str, "%Y-%m-%dT%H:%M:%S.%fZ"
767 |                 )
768 |                 if created_timestamp <= datetime_threshold:
769 |                     dm_id = int(message["messageCreate"]["id"])
770 |                     dm_ids.append(dm_id)
771 | 
772 |                     # Try deleting
773 |                     try:
774 |                         self.api.destroy_direct_message(dm_id)
775 |                         click.echo(
776 |                             "Deleted DM {}, id {}".format(
777 |                                 created_timestamp.strftime("%Y-%m-%d"), dm_id
778 |                             )
779 |                         )
780 |                         count += 1
781 |                     except tweepy.error.TweepError as e:
782 |                         click.secho(
783 |                             "Error deleting DM {}, id {}: {}".format(
784 |                                 created_timestamp.strftime("%Y-%m-%d"), dm_id, str(e)
785 |                             ),
786 |                             dim=True,
787 |                         )
788 | 
789 |         click.secho("Deleted {} DMs".format(count), fg="cyan")
790 |         self.common.log("Deleted {} DMs".format(count))
791 | 
792 |     def import_dump(self, filepath):
793 |         if self.common.settings.get("delete_tweets"):
794 | 
795 |             if os.path.isdir(filepath):
796 |                 # Unzipped tweet archive
797 |                 with open(os.path.join(filepath, "data", "tweet.js"), "r", encoding="UTF-8") as f:
798 |                     # Skip the JS variable assignment at the start of this file
799 |                     f.read(25)
800 |                     tweets = json.load(f)
801 |             elif os.path.splitext(filepath)[1] == ".zip":
802 |                 # Zipped tweet archive
803 |                 with ZipFile(filepath) as zipfile:
804 |                     with zipfile.open("data/tweet.js") as f:
805 |                         f = io.TextIOWrapper(f, "UTF-8")
806 |                         # Skip the JS variable assignment at the start of this file
807 |                         f.read(25)
808 |                         tweets = json.load(f)
809 |             else:
810 |                 click.echo("Path should be a zipped tweet export file or the extracted directory")
811 |                 return
812 | 
813 |             click.echo("Importing {} tweets from {}".format(len(tweets), filepath))
814 | 
815 |             current_user = self.api.me()
816 | 
817 |             def parse_tweet(tweet):
818 |                 """Parse a JSON tweet into a Tweet DB object."""
819 |                 t = Status.parse(self.api, tweet["tweet"])
820 |                 t.author = current_user
821 |                 return Tweet(t)
822 | 
823 |             for t in tweets:
824 |                 self.import_tweet_and_thread(parse_tweet(t))
825 | 
826 |         # All done, update the since_id
827 |         tweet = (
828 |             self.common.session.query(Tweet).order_by(Tweet.status_id.desc()).first()
829 |         )
830 |         if tweet:
831 |             self.common.settings.set("since_id", tweet.status_id)
832 |             self.common.settings.save()
833 | 
834 |         # Calculate which threads should be excluded from deletion
835 |         self.calculate_excluded_threads()
836 | 
837 |         self.common.settings.set(
838 |             "last_fetch", datetime.datetime.today().strftime(self.last_fetch_format)
839 |         )
840 |         self.common.settings.save()
841 | 


--------------------------------------------------------------------------------
/Pipfile.lock:
--------------------------------------------------------------------------------
  1 | {
  2 |     "_meta": {
  3 |         "hash": {
  4 |             "sha256": "dd0035c11b86591ec867f3ad3fe06fe6dcf67c5fc63c5e10d059ae5412c9367a"
  5 |         },
  6 |         "pipfile-spec": 6,
  7 |         "requires": {
  8 |             "python_version": "3"
  9 |         },
 10 |         "sources": [
 11 |             {
 12 |                 "name": "pypi",
 13 |                 "url": "https://pypi.org/simple",
 14 |                 "verify_ssl": true
 15 |             }
 16 |         ]
 17 |     },
 18 |     "default": {
 19 |         "certifi": {
 20 |             "hashes": [
 21 |                 "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
 22 |                 "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
 23 |             ],
 24 |             "version": "==2021.10.8"
 25 |         },
 26 |         "charset-normalizer": {
 27 |             "hashes": [
 28 |                 "sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0",
 29 |                 "sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405"
 30 |             ],
 31 |             "markers": "python_version >= '3'",
 32 |             "version": "==2.0.8"
 33 |         },
 34 |         "click": {
 35 |             "hashes": [
 36 |                 "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3",
 37 |                 "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"
 38 |             ],
 39 |             "index": "pypi",
 40 |             "version": "==8.0.3"
 41 |         },
 42 |         "colorama": {
 43 |             "hashes": [
 44 |                 "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
 45 |                 "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
 46 |             ],
 47 |             "index": "pypi",
 48 |             "version": "==0.4.4"
 49 |         },
 50 |         "flask": {
 51 |             "hashes": [
 52 |                 "sha256:7b2fb8e934ddd50731893bdcdb00fc8c0315916f9fcd50d22c7cc1a95ab634e2",
 53 |                 "sha256:cb90f62f1d8e4dc4621f52106613488b5ba826b2e1e10a33eac92f723093ab6a"
 54 |             ],
 55 |             "index": "pypi",
 56 |             "version": "==2.0.2"
 57 |         },
 58 |         "greenlet": {
 59 |             "hashes": [
 60 |                 "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711",
 61 |                 "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd",
 62 |                 "sha256:049fe7579230e44daef03a259faa24511d10ebfa44f69411d99e6a184fe68073",
 63 |                 "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708",
 64 |                 "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67",
 65 |                 "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23",
 66 |                 "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1",
 67 |                 "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08",
 68 |                 "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd",
 69 |                 "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa",
 70 |                 "sha256:356b3576ad078c89a6107caa9c50cc14e98e3a6c4874a37c3e0273e4baf33de8",
 71 |                 "sha256:40b951f601af999a8bf2ce8c71e8aaa4e8c6f78ff8afae7b808aae2dc50d4c40",
 72 |                 "sha256:572e1787d1460da79590bf44304abbc0a2da944ea64ec549188fa84d89bba7ab",
 73 |                 "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6",
 74 |                 "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc",
 75 |                 "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b",
 76 |                 "sha256:7418b6bfc7fe3331541b84bb2141c9baf1ec7132a7ecd9f375912eca810e714e",
 77 |                 "sha256:7cbd7574ce8e138bda9df4efc6bf2ab8572c9aff640d8ecfece1b006b68da963",
 78 |                 "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3",
 79 |                 "sha256:833e1551925ed51e6b44c800e71e77dacd7e49181fdc9ac9a0bf3714d515785d",
 80 |                 "sha256:8639cadfda96737427330a094476d4c7a56ac03de7265622fcf4cfe57c8ae18d",
 81 |                 "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28",
 82 |                 "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3",
 83 |                 "sha256:903bbd302a2378f984aef528f76d4c9b1748f318fe1294961c072bdc7f2ffa3e",
 84 |                 "sha256:93f81b134a165cc17123626ab8da2e30c0455441d4ab5576eed73a64c025b25c",
 85 |                 "sha256:95e69877983ea39b7303570fa6760f81a3eec23d0e3ab2021b7144b94d06202d",
 86 |                 "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0",
 87 |                 "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497",
 88 |                 "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee",
 89 |                 "sha256:aa5b467f15e78b82257319aebc78dd2915e4c1436c3c0d1ad6f53e47ba6e2713",
 90 |                 "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58",
 91 |                 "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a",
 92 |                 "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06",
 93 |                 "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88",
 94 |                 "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4",
 95 |                 "sha256:be5f425ff1f5f4b3c1e33ad64ab994eed12fc284a6ea71c5243fd564502ecbe5",
 96 |                 "sha256:dd0b1e9e891f69e7675ba5c92e28b90eaa045f6ab134ffe70b52e948aa175b3c",
 97 |                 "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a",
 98 |                 "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1",
 99 |                 "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43",
100 |                 "sha256:eb6ea6da4c787111adf40f697b4e58732ee0942b5d3bd8f435277643329ba627",
101 |                 "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b",
102 |                 "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168",
103 |                 "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d",
104 |                 "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5",
105 |                 "sha256:f3acda1924472472ddd60c29e5b9db0cec629fbe3c5c5accb74d6d6d14773478",
106 |                 "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf",
107 |                 "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce",
108 |                 "sha256:fa877ca7f6b48054f847b61d6fa7bed5cebb663ebc55e018fda12db09dcc664c",
109 |                 "sha256:fdcec0b8399108577ec290f55551d926d9a1fa6cad45882093a7a07ac5ec147b"
110 |             ],
111 |             "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))",
112 |             "version": "==1.1.2"
113 |         },
114 |         "idna": {
115 |             "hashes": [
116 |                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
117 |                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
118 |             ],
119 |             "markers": "python_version >= '3'",
120 |             "version": "==3.3"
121 |         },
122 |         "itsdangerous": {
123 |             "hashes": [
124 |                 "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c",
125 |                 "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"
126 |             ],
127 |             "version": "==2.0.1"
128 |         },
129 |         "jinja2": {
130 |             "hashes": [
131 |                 "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8",
132 |                 "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"
133 |             ],
134 |             "version": "==3.0.3"
135 |         },
136 |         "markupsafe": {
137 |             "hashes": [
138 |                 "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298",
139 |                 "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64",
140 |                 "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b",
141 |                 "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194",
142 |                 "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567",
143 |                 "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff",
144 |                 "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724",
145 |                 "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74",
146 |                 "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646",
147 |                 "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35",
148 |                 "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6",
149 |                 "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a",
150 |                 "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6",
151 |                 "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad",
152 |                 "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26",
153 |                 "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38",
154 |                 "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac",
155 |                 "sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7",
156 |                 "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6",
157 |                 "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047",
158 |                 "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75",
159 |                 "sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f",
160 |                 "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b",
161 |                 "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135",
162 |                 "sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8",
163 |                 "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a",
164 |                 "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a",
165 |                 "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1",
166 |                 "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9",
167 |                 "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864",
168 |                 "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914",
169 |                 "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee",
170 |                 "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f",
171 |                 "sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18",
172 |                 "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8",
173 |                 "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2",
174 |                 "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d",
175 |                 "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b",
176 |                 "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b",
177 |                 "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86",
178 |                 "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6",
179 |                 "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f",
180 |                 "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb",
181 |                 "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833",
182 |                 "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28",
183 |                 "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e",
184 |                 "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415",
185 |                 "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902",
186 |                 "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f",
187 |                 "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d",
188 |                 "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9",
189 |                 "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d",
190 |                 "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145",
191 |                 "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066",
192 |                 "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c",
193 |                 "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1",
194 |                 "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a",
195 |                 "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207",
196 |                 "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f",
197 |                 "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53",
198 |                 "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd",
199 |                 "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134",
200 |                 "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85",
201 |                 "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9",
202 |                 "sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5",
203 |                 "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94",
204 |                 "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509",
205 |                 "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51",
206 |                 "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"
207 |             ],
208 |             "version": "==2.0.1"
209 |         },
210 |         "oauthlib": {
211 |             "hashes": [
212 |                 "sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc",
213 |                 "sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3"
214 |             ],
215 |             "version": "==3.1.1"
216 |         },
217 |         "pysocks": {
218 |             "hashes": [
219 |                 "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299",
220 |                 "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5",
221 |                 "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"
222 |             ],
223 |             "version": "==1.7.1"
224 |         },
225 |         "requests": {
226 |             "hashes": [
227 |                 "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
228 |                 "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
229 |             ],
230 |             "version": "==2.26.0"
231 |         },
232 |         "requests-oauthlib": {
233 |             "hashes": [
234 |                 "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d",
235 |                 "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a",
236 |                 "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"
237 |             ],
238 |             "version": "==1.3.0"
239 |         },
240 |         "six": {
241 |             "hashes": [
242 |                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
243 |                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
244 |             ],
245 |             "version": "==1.16.0"
246 |         },
247 |         "sqlalchemy": {
248 |             "hashes": [
249 |                 "sha256:015511c52c650eebf1059ed8a21674d9d4ae567ebfd80fc73f8252faccd71864",
250 |                 "sha256:0438bccc16349db2d5203598be6073175ce16d4e53b592d6e6cef880c197333e",
251 |                 "sha256:10230364479429437f1b819a8839f1edc5744c018bfeb8d01320930f97695bc9",
252 |                 "sha256:2146ef996181e3d4dd20eaf1d7325eb62d6c8aa4dc1677c1872ddfa8561a47d9",
253 |                 "sha256:24828c5e74882cf41516740c0b150702bee4c6817d87d5c3d3bafef2e6896f80",
254 |                 "sha256:2717ceae35e71de1f58b0d1ee7e773d3aab5c403c6e79e8d262277c7f7f95269",
255 |                 "sha256:2e93624d186ea7a738ada47314701c8830e0e4b021a6bce7fbe6f39b87ee1516",
256 |                 "sha256:435b1980c1333ffe3ab386ad28d7b209590b0fa83ea8544d853e7a22f957331b",
257 |                 "sha256:486f7916ef77213103467924ef25f5ea1055ae901f385fe4d707604095fdf6a9",
258 |                 "sha256:4ac8306e04275d382d6393e557047b0a9d7ddf9f7ca5da9b3edbd9323ea75bd9",
259 |                 "sha256:4d1d707b752137e6bf45720648e1b828d5e4881d690df79cca07f7217ea06365",
260 |                 "sha256:52f23a76544ed29573c0f3ee41f0ca1aedbab3a453102b60b540cc6fa55448ad",
261 |                 "sha256:5beeff18b4e894f6cb73c8daf2c0d8768844ef40d97032bb187d75b1ec8de24b",
262 |                 "sha256:6510f4a5029643301bdfe56b61e806093af2101d347d485c42a5535847d2c699",
263 |                 "sha256:6afa9e4e63f066e0fd90a21db7e95e988d96127f52bfb298a0e9bec6999357a9",
264 |                 "sha256:771eca9872b47a629010665ff92de1c248a6979b8d1603daced37773d6f6e365",
265 |                 "sha256:78943451ab3ffd0e27876f9cea2b883317518b418f06b90dadf19394534637e9",
266 |                 "sha256:8327e468b1775c0dfabc3d01f39f440585bf4d398508fcbbe2f0d931c502337d",
267 |                 "sha256:8dbe5f639e6d035778ebf700be6d573f82a13662c3c2c3aa0f1dba303b942806",
268 |                 "sha256:9134e5810262203388b203c2022bbcbf1a22e89861eef9340e772a73dd9076fa",
269 |                 "sha256:9369f927f4d19b58322cfea8a51710a3f7c47a0e7f3398d94a4632760ecd74f6",
270 |                 "sha256:987fe2f84ceaf744fa0e48805152abe485a9d7002c9923b18a4b2529c7bff218",
271 |                 "sha256:a5881644fc51af7b232ab8d64f75c0f32295dfe88c2ee188023795cdbd4cf99b",
272 |                 "sha256:a81e40dfa50ed3c472494adadba097640bfcf43db160ed783132045eb2093cb1",
273 |                 "sha256:aadc6d1e58e14010ae4764d1ba1fd0928dbb9423b27a382ea3a1444f903f4084",
274 |                 "sha256:ad8ec6b69d03e395db48df8991aa15fce3cd23e378b73e01d46a26a6efd5c26d",
275 |                 "sha256:b02eee1577976acb4053f83d32b7826424f8b9f70809fa756529a52c6537eda4",
276 |                 "sha256:bac949be7579fed824887eed6672f44b7c4318abbfb2004b2c6968818b535a2f",
277 |                 "sha256:c035184af4e58e154b0977eea52131edd096e0754a88f7d5a847e7ccb3510772",
278 |                 "sha256:c7d0a1b1258efff7d7f2e6cfa56df580d09ba29d35a1e3f604f867e1f685feb2",
279 |                 "sha256:cc49fb8ff103900c20e4a9c53766c82a7ebbc183377fb357a8298bad216e9cdd",
280 |                 "sha256:d768359daeb3a86644f3854c6659e4496a3e6bba2b4651ecc87ce7ad415b320c",
281 |                 "sha256:d81c84c9d2523b3ea20f8e3aceea68615768a7464c0f9a9899600ce6592ec570",
282 |                 "sha256:ec1c908fa721f2c5684900cc8ff75555b1a5a2ae4f5a5694eb0e37a5263cea44",
283 |                 "sha256:fa52534076394af7315306a8701b726a6521b591d95e8f4e5121c82f94790e8d",
284 |                 "sha256:fd421a14edf73cfe01e8f51ed8966294ee3b3db8da921cacc88e497fd6e977af"
285 |             ],
286 |             "index": "pypi",
287 |             "version": "==1.4.27"
288 |         },
289 |         "tweepy": {
290 |             "hashes": [
291 |                 "sha256:5e22003441a11f6f4c2ea4d05ec5532f541e9f5d874c3908270f0c28e649b53a",
292 |                 "sha256:76e6954b806ca470dda877f57db8792fff06a0beba0ed43efc3805771e39f06a"
293 |             ],
294 |             "index": "pypi",
295 |             "version": "==3.10.0"
296 |         },
297 |         "urllib3": {
298 |             "hashes": [
299 |                 "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece",
300 |                 "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"
301 |             ],
302 |             "version": "==1.26.7"
303 |         },
304 |         "werkzeug": {
305 |             "hashes": [
306 |                 "sha256:63d3dc1cf60e7b7e35e97fa9861f7397283b75d765afcaefd993d6046899de8f",
307 |                 "sha256:aa2bb6fc8dee8d6c504c0ac1e7f5f7dc5810a9903e793b6f715a9f015bdadb9a"
308 |             ],
309 |             "version": "==2.0.2"
310 |         }
311 |     },
312 |     "develop": {
313 |         "black": {
314 |             "hashes": [
315 |                 "sha256:802c6c30b637b28645b7fde282ed2569c0cd777dbe493a41b6a03c1d903f99ac",
316 |                 "sha256:a042adbb18b3262faad5aff4e834ff186bb893f95ba3a8013f09de1e5569def2"
317 |             ],
318 |             "index": "pypi",
319 |             "version": "==21.11b1"
320 |         },
321 |         "bleach": {
322 |             "hashes": [
323 |                 "sha256:0900d8b37eba61a802ee40ac0061f8c2b5dee29c1927dd1d233e075ebf5a71da",
324 |                 "sha256:4d2651ab93271d1129ac9cbc679f524565cc8a1b791909c4a51eac4446a15994"
325 |             ],
326 |             "version": "==4.1.0"
327 |         },
328 |         "certifi": {
329 |             "hashes": [
330 |                 "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
331 |                 "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
332 |             ],
333 |             "version": "==2021.10.8"
334 |         },
335 |         "cffi": {
336 |             "hashes": [
337 |                 "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3",
338 |                 "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2",
339 |                 "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636",
340 |                 "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20",
341 |                 "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728",
342 |                 "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27",
343 |                 "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66",
344 |                 "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443",
345 |                 "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0",
346 |                 "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7",
347 |                 "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39",
348 |                 "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605",
349 |                 "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a",
350 |                 "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37",
351 |                 "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029",
352 |                 "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139",
353 |                 "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc",
354 |                 "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df",
355 |                 "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14",
356 |                 "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880",
357 |                 "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2",
358 |                 "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a",
359 |                 "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e",
360 |                 "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474",
361 |                 "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024",
362 |                 "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8",
363 |                 "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0",
364 |                 "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e",
365 |                 "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a",
366 |                 "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e",
367 |                 "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032",
368 |                 "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6",
369 |                 "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e",
370 |                 "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b",
371 |                 "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e",
372 |                 "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954",
373 |                 "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962",
374 |                 "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c",
375 |                 "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4",
376 |                 "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55",
377 |                 "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962",
378 |                 "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023",
379 |                 "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c",
380 |                 "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6",
381 |                 "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8",
382 |                 "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382",
383 |                 "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7",
384 |                 "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc",
385 |                 "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997",
386 |                 "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"
387 |             ],
388 |             "version": "==1.15.0"
389 |         },
390 |         "charset-normalizer": {
391 |             "hashes": [
392 |                 "sha256:735e240d9a8506778cd7a453d97e817e536bb1fc29f4f6961ce297b9c7a917b0",
393 |                 "sha256:83fcdeb225499d6344c8f7f34684c2981270beacc32ede2e669e94f7fa544405"
394 |             ],
395 |             "markers": "python_version >= '3'",
396 |             "version": "==2.0.8"
397 |         },
398 |         "click": {
399 |             "hashes": [
400 |                 "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3",
401 |                 "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b"
402 |             ],
403 |             "index": "pypi",
404 |             "version": "==8.0.3"
405 |         },
406 |         "colorama": {
407 |             "hashes": [
408 |                 "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
409 |                 "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
410 |             ],
411 |             "index": "pypi",
412 |             "version": "==0.4.4"
413 |         },
414 |         "cryptography": {
415 |             "hashes": [
416 |                 "sha256:2049f8b87f449fc6190350de443ee0c1dd631f2ce4fa99efad2984de81031681",
417 |                 "sha256:231c4a69b11f6af79c1495a0e5a85909686ea8db946935224b7825cfb53827ed",
418 |                 "sha256:24469d9d33217ffd0ce4582dfcf2a76671af115663a95328f63c99ec7ece61a4",
419 |                 "sha256:2deab5ec05d83ddcf9b0916319674d3dae88b0e7ee18f8962642d3cde0496568",
420 |                 "sha256:494106e9cd945c2cadfce5374fa44c94cfadf01d4566a3b13bb487d2e6c7959e",
421 |                 "sha256:4c702855cd3174666ef0d2d13dcc879090aa9c6c38f5578896407a7028f75b9f",
422 |                 "sha256:52f769ecb4ef39865719aedc67b4b7eae167bafa48dbc2a26dd36fa56460507f",
423 |                 "sha256:5c49c9e8fb26a567a2b3fa0343c89f5d325447956cc2fc7231c943b29a973712",
424 |                 "sha256:684993ff6f67000a56454b41bdc7e015429732d65a52d06385b6e9de6181c71e",
425 |                 "sha256:6fbbbb8aab4053fa018984bb0e95a16faeb051dd8cca15add2a27e267ba02b58",
426 |                 "sha256:8982c19bb90a4fa2aad3d635c6d71814e38b643649b4000a8419f8691f20ac44",
427 |                 "sha256:9511416e85e449fe1de73f7f99b21b3aa04fba4c4d335d30c486ba3756e3a2a6",
428 |                 "sha256:97199a13b772e74cdcdb03760c32109c808aff7cd49c29e9cf4b7754bb725d1d",
429 |                 "sha256:a776bae1629c8d7198396fd93ec0265f8dd2341c553dc32b976168aaf0e6a636",
430 |                 "sha256:aa94d617a4cd4cdf4af9b5af65100c036bce22280ebb15d8b5262e8273ebc6ba",
431 |                 "sha256:b17d83b3d1610e571fedac21b2eb36b816654d6f7496004d6a0d32f99d1d8120",
432 |                 "sha256:d73e3a96c38173e0aa5646c31bf8473bc3564837977dd480f5cbeacf1d7ef3a3",
433 |                 "sha256:d91bc9f535599bed58f6d2e21a2724cb0c3895bf41c6403fe881391d29096f1d",
434 |                 "sha256:ef216d13ac8d24d9cd851776662f75f8d29c9f2d05cdcc2d34a18d32463a9b0b",
435 |                 "sha256:f6a5a85beb33e57998dc605b9dbe7deaa806385fdf5c4810fb849fcd04640c81",
436 |                 "sha256:f92556f94e476c1b616e6daec5f7ddded2c082efa7cee7f31c7aeda615906ed8"
437 |             ],
438 |             "version": "==36.0.0"
439 |         },
440 |         "docutils": {
441 |             "hashes": [
442 |                 "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c",
443 |                 "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"
444 |             ],
445 |             "version": "==0.18.1"
446 |         },
447 |         "idna": {
448 |             "hashes": [
449 |                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
450 |                 "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
451 |             ],
452 |             "markers": "python_version >= '3'",
453 |             "version": "==3.3"
454 |         },
455 |         "importlib-metadata": {
456 |             "hashes": [
457 |                 "sha256:53ccfd5c134223e497627b9815d5030edf77d2ed573922f7a0b8f8bb81a1c100",
458 |                 "sha256:75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb"
459 |             ],
460 |             "version": "==4.8.2"
461 |         },
462 |         "jeepney": {
463 |             "hashes": [
464 |                 "sha256:1b5a0ea5c0e7b166b2f5895b91a08c14de8915afda4407fb5022a195224958ac",
465 |                 "sha256:fa9e232dfa0c498bd0b8a3a73b8d8a31978304dcef0515adc859d4e096f96f4f"
466 |             ],
467 |             "markers": "sys_platform == 'linux'",
468 |             "version": "==0.7.1"
469 |         },
470 |         "keyring": {
471 |             "hashes": [
472 |                 "sha256:3dc0f66062a4f8f6f2ce30d6a516e6e623e6c3c2e76864204ceaf64695408f07",
473 |                 "sha256:88f206024295e3c6fb16bb0a60fb4bb7ec1185629dc5a729f12aa7c236d01387"
474 |             ],
475 |             "version": "==23.4.0"
476 |         },
477 |         "mypy-extensions": {
478 |             "hashes": [
479 |                 "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
480 |                 "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
481 |             ],
482 |             "version": "==0.4.3"
483 |         },
484 |         "packaging": {
485 |             "hashes": [
486 |                 "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
487 |                 "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
488 |             ],
489 |             "version": "==21.3"
490 |         },
491 |         "pathspec": {
492 |             "hashes": [
493 |                 "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a",
494 |                 "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"
495 |             ],
496 |             "version": "==0.9.0"
497 |         },
498 |         "pkginfo": {
499 |             "hashes": [
500 |                 "sha256:65175ffa2c807220673a41c371573ac9a1ea1b19ffd5eef916278f428319934f",
501 |                 "sha256:bb55a6c017d50f2faea5153abc7b05a750e7ea7ae2cbb7fb3ad6f1dcf8d40988"
502 |             ],
503 |             "version": "==1.8.1"
504 |         },
505 |         "platformdirs": {
506 |             "hashes": [
507 |                 "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2",
508 |                 "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"
509 |             ],
510 |             "version": "==2.4.0"
511 |         },
512 |         "pycparser": {
513 |             "hashes": [
514 |                 "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
515 |                 "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
516 |             ],
517 |             "version": "==2.21"
518 |         },
519 |         "pygments": {
520 |             "hashes": [
521 |                 "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380",
522 |                 "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"
523 |             ],
524 |             "version": "==2.10.0"
525 |         },
526 |         "pyparsing": {
527 |             "hashes": [
528 |                 "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4",
529 |                 "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81"
530 |             ],
531 |             "version": "==3.0.6"
532 |         },
533 |         "readme-renderer": {
534 |             "hashes": [
535 |                 "sha256:3286806450d9961d6e3b5f8a59f77e61503799aca5155c8d8d40359b4e1e1adc",
536 |                 "sha256:8299700d7a910c304072a7601eafada6712a5b011a20139417e1b1e9f04645d8"
537 |             ],
538 |             "version": "==30.0"
539 |         },
540 |         "regex": {
541 |             "hashes": [
542 |                 "sha256:0416f7399e918c4b0e074a0f66e5191077ee2ca32a0f99d4c187a62beb47aa05",
543 |                 "sha256:05b7d6d7e64efe309972adab77fc2af8907bb93217ec60aa9fe12a0dad35874f",
544 |                 "sha256:0617383e2fe465732af4509e61648b77cbe3aee68b6ac8c0b6fe934db90be5cc",
545 |                 "sha256:07856afef5ffcc052e7eccf3213317fbb94e4a5cd8177a2caa69c980657b3cb4",
546 |                 "sha256:0f594b96fe2e0821d026365f72ac7b4f0b487487fb3d4aaf10dd9d97d88a9737",
547 |                 "sha256:139a23d1f5d30db2cc6c7fd9c6d6497872a672db22c4ae1910be22d4f4b2068a",
548 |                 "sha256:162abfd74e88001d20cb73ceaffbfe601469923e875caf9118333b1a4aaafdc4",
549 |                 "sha256:2207ae4f64ad3af399e2d30dde66f0b36ae5c3129b52885f1bffc2f05ec505c8",
550 |                 "sha256:2409b5c9cef7054dde93a9803156b411b677affc84fca69e908b1cb2c540025d",
551 |                 "sha256:2fee3ed82a011184807d2127f1733b4f6b2ff6ec7151d83ef3477f3b96a13d03",
552 |                 "sha256:30ab804ea73972049b7a2a5c62d97687d69b5a60a67adca07eb73a0ddbc9e29f",
553 |                 "sha256:3598893bde43091ee5ca0a6ad20f08a0435e93a69255eeb5f81b85e81e329264",
554 |                 "sha256:3b5df18db1fccd66de15aa59c41e4f853b5df7550723d26aa6cb7f40e5d9da5a",
555 |                 "sha256:3c5fb32cc6077abad3bbf0323067636d93307c9fa93e072771cf9a64d1c0f3ef",
556 |                 "sha256:416c5f1a188c91e3eb41e9c8787288e707f7d2ebe66e0a6563af280d9b68478f",
557 |                 "sha256:42b50fa6666b0d50c30a990527127334d6b96dd969011e843e726a64011485da",
558 |                 "sha256:432bd15d40ed835a51617521d60d0125867f7b88acf653e4ed994a1f8e4995dc",
559 |                 "sha256:473e67837f786404570eae33c3b64a4b9635ae9f00145250851a1292f484c063",
560 |                 "sha256:4aaa4e0705ef2b73dd8e36eeb4c868f80f8393f5f4d855e94025ce7ad8525f50",
561 |                 "sha256:50a7ddf3d131dc5633dccdb51417e2d1910d25cbcf842115a3a5893509140a3a",
562 |                 "sha256:529801a0d58809b60b3531ee804d3e3be4b412c94b5d267daa3de7fadef00f49",
563 |                 "sha256:537ca6a3586931b16a85ac38c08cc48f10fc870a5b25e51794c74df843e9966d",
564 |                 "sha256:53db2c6be8a2710b359bfd3d3aa17ba38f8aa72a82309a12ae99d3c0c3dcd74d",
565 |                 "sha256:5537f71b6d646f7f5f340562ec4c77b6e1c915f8baae822ea0b7e46c1f09b733",
566 |                 "sha256:563d5f9354e15e048465061509403f68424fef37d5add3064038c2511c8f5e00",
567 |                 "sha256:5d408a642a5484b9b4d11dea15a489ea0928c7e410c7525cd892f4d04f2f617b",
568 |                 "sha256:61600a7ca4bcf78a96a68a27c2ae9389763b5b94b63943d5158f2a377e09d29a",
569 |                 "sha256:6650f16365f1924d6014d2ea770bde8555b4a39dc9576abb95e3cd1ff0263b36",
570 |                 "sha256:666abff54e474d28ff42756d94544cdfd42e2ee97065857413b72e8a2d6a6345",
571 |                 "sha256:68a067c11463de2a37157930d8b153005085e42bcb7ad9ca562d77ba7d1404e0",
572 |                 "sha256:6e1d2cc79e8dae442b3fa4a26c5794428b98f81389af90623ffcc650ce9f6732",
573 |                 "sha256:74cbeac0451f27d4f50e6e8a8f3a52ca074b5e2da9f7b505c4201a57a8ed6286",
574 |                 "sha256:780b48456a0f0ba4d390e8b5f7c661fdd218934388cde1a974010a965e200e12",
575 |                 "sha256:788aef3549f1924d5c38263104dae7395bf020a42776d5ec5ea2b0d3d85d6646",
576 |                 "sha256:7ee1227cf08b6716c85504aebc49ac827eb88fcc6e51564f010f11a406c0a667",
577 |                 "sha256:7f301b11b9d214f83ddaf689181051e7f48905568b0c7017c04c06dfd065e244",
578 |                 "sha256:83ee89483672b11f8952b158640d0c0ff02dc43d9cb1b70c1564b49abe92ce29",
579 |                 "sha256:85bfa6a5413be0ee6c5c4a663668a2cad2cbecdee367630d097d7823041bdeec",
580 |                 "sha256:9345b6f7ee578bad8e475129ed40123d265464c4cfead6c261fd60fc9de00bcf",
581 |                 "sha256:93a5051fcf5fad72de73b96f07d30bc29665697fb8ecdfbc474f3452c78adcf4",
582 |                 "sha256:962b9a917dd7ceacbe5cd424556914cb0d636001e393b43dc886ba31d2a1e449",
583 |                 "sha256:96fc32c16ea6d60d3ca7f63397bff5c75c5a562f7db6dec7d412f7c4d2e78ec0",
584 |                 "sha256:98ba568e8ae26beb726aeea2273053c717641933836568c2a0278a84987b2a1a",
585 |                 "sha256:a3feefd5e95871872673b08636f96b61ebef62971eab044f5124fb4dea39919d",
586 |                 "sha256:a955b747d620a50408b7fdf948e04359d6e762ff8a85f5775d907ceced715129",
587 |                 "sha256:b43c2b8a330a490daaef5a47ab114935002b13b3f9dc5da56d5322ff218eeadb",
588 |                 "sha256:b483c9d00a565633c87abd0aaf27eb5016de23fed952e054ecc19ce32f6a9e7e",
589 |                 "sha256:b9ed0b1e5e0759d6b7f8e2f143894b2a7f3edd313f38cf44e1e15d360e11749b",
590 |                 "sha256:ba05430e819e58544e840a68b03b28b6d328aff2e41579037e8bab7653b37d83",
591 |                 "sha256:ca49e1ab99593438b204e00f3970e7a5f70d045267051dfa6b5f4304fcfa1dbf",
592 |                 "sha256:ca5f18a75e1256ce07494e245cdb146f5a9267d3c702ebf9b65c7f8bd843431e",
593 |                 "sha256:cd410a1cbb2d297c67d8521759ab2ee3f1d66206d2e4328502a487589a2cb21b",
594 |                 "sha256:ce298e3d0c65bd03fa65ffcc6db0e2b578e8f626d468db64fdf8457731052942",
595 |                 "sha256:d5ca078bb666c4a9d1287a379fe617a6dccd18c3e8a7e6c7e1eb8974330c626a",
596 |                 "sha256:d5fd67df77bab0d3f4ea1d7afca9ef15c2ee35dfb348c7b57ffb9782a6e4db6e",
597 |                 "sha256:da1a90c1ddb7531b1d5ff1e171b4ee61f6345119be7351104b67ff413843fe94",
598 |                 "sha256:dba70f30fd81f8ce6d32ddeef37d91c8948e5d5a4c63242d16a2b2df8143aafc",
599 |                 "sha256:dc07f021ee80510f3cd3af2cad5b6a3b3a10b057521d9e6aaeb621730d320c5a",
600 |                 "sha256:dd33eb9bdcfbabab3459c9ee651d94c842bc8a05fabc95edf4ee0c15a072495e",
601 |                 "sha256:e0538c43565ee6e703d3a7c3bdfe4037a5209250e8502c98f20fea6f5fdf2965",
602 |                 "sha256:e1f54b9b4b6c53369f40028d2dd07a8c374583417ee6ec0ea304e710a20f80a0",
603 |                 "sha256:e32d2a2b02ccbef10145df9135751abea1f9f076e67a4e261b05f24b94219e36",
604 |                 "sha256:e6096b0688e6e14af6a1b10eaad86b4ff17935c49aa774eac7c95a57a4e8c296",
605 |                 "sha256:e71255ba42567d34a13c03968736c5d39bb4a97ce98188fafb27ce981115beec",
606 |                 "sha256:ed2e07c6a26ed4bea91b897ee2b0835c21716d9a469a96c3e878dc5f8c55bb23",
607 |                 "sha256:eef2afb0fd1747f33f1ee3e209bce1ed582d1896b240ccc5e2697e3275f037c7",
608 |                 "sha256:f23222527b307970e383433daec128d769ff778d9b29343fb3496472dc20dabe",
609 |                 "sha256:f341ee2df0999bfdf7a95e448075effe0db212a59387de1a70690e4acb03d4c6",
610 |                 "sha256:f5be7805e53dafe94d295399cfbe5227f39995a997f4fd8539bf3cbdc8f47ca8",
611 |                 "sha256:f7f325be2804246a75a4f45c72d4ce80d2443ab815063cdf70ee8fb2ca59ee1b",
612 |                 "sha256:f8af619e3be812a2059b212064ea7a640aff0568d972cd1b9e920837469eb3cb",
613 |                 "sha256:fa8c626d6441e2d04b6ee703ef2d1e17608ad44c7cb75258c09dd42bacdfc64b",
614 |                 "sha256:fbb9dc00e39f3e6c0ef48edee202f9520dafb233e8b51b06b8428cfcb92abd30",
615 |                 "sha256:fff55f3ce50a3ff63ec8e2a8d3dd924f1941b250b0aac3d3d42b687eeff07a8e"
616 |             ],
617 |             "version": "==2021.11.10"
618 |         },
619 |         "requests": {
620 |             "hashes": [
621 |                 "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
622 |                 "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
623 |             ],
624 |             "version": "==2.26.0"
625 |         },
626 |         "requests-toolbelt": {
627 |             "hashes": [
628 |                 "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
629 |                 "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
630 |             ],
631 |             "version": "==0.9.1"
632 |         },
633 |         "rfc3986": {
634 |             "hashes": [
635 |                 "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835",
636 |                 "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"
637 |             ],
638 |             "version": "==1.5.0"
639 |         },
640 |         "secretstorage": {
641 |             "hashes": [
642 |                 "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f",
643 |                 "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"
644 |             ],
645 |             "markers": "sys_platform == 'linux'",
646 |             "version": "==3.3.1"
647 |         },
648 |         "six": {
649 |             "hashes": [
650 |                 "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
651 |                 "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
652 |             ],
653 |             "version": "==1.16.0"
654 |         },
655 |         "tomli": {
656 |             "hashes": [
657 |                 "sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee",
658 |                 "sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade"
659 |             ],
660 |             "version": "==1.2.2"
661 |         },
662 |         "tqdm": {
663 |             "hashes": [
664 |                 "sha256:8dd278a422499cd6b727e6ae4061c40b48fce8b76d1ccbf5d34fca9b7f925b0c",
665 |                 "sha256:d359de7217506c9851b7869f3708d8ee53ed70a1b8edbba4dbcb47442592920d"
666 |             ],
667 |             "version": "==4.62.3"
668 |         },
669 |         "twine": {
670 |             "hashes": [
671 |                 "sha256:4caad5ef4722e127b3749052fcbffaaf71719b19d4fd4973b29c469957adeba2",
672 |                 "sha256:916070f8ecbd1985ebed5dbb02b9bda9a092882a96d7069d542d4fc0bb5c673c"
673 |             ],
674 |             "index": "pypi",
675 |             "version": "==3.6.0"
676 |         },
677 |         "typing-extensions": {
678 |             "hashes": [
679 |                 "sha256:2cdf80e4e04866a9b3689a51869016d36db0814d84b8d8a568d22781d45d27ed",
680 |                 "sha256:829704698b22e13ec9eaf959122315eabb370b0884400e9818334d8b677023d9"
681 |             ],
682 |             "version": "==4.0.0"
683 |         },
684 |         "urllib3": {
685 |             "hashes": [
686 |                 "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece",
687 |                 "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"
688 |             ],
689 |             "version": "==1.26.7"
690 |         },
691 |         "webencodings": {
692 |             "hashes": [
693 |                 "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
694 |                 "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
695 |             ],
696 |             "version": "==0.5.1"
697 |         },
698 |         "zipp": {
699 |             "hashes": [
700 |                 "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832",
701 |                 "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"
702 |             ],
703 |             "version": "==3.6.0"
704 |         }
705 |     }
706 | }
707 | 


--------------------------------------------------------------------------------