├── data └── .keep ├── etc └── .keep ├── .gitignore ├── Gemfile ├── config.yml ├── README.md ├── LICENSE ├── Gemfile.lock └── lib ├── generate.rb ├── database.rb └── fetch.rb /data/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /etc/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | venv/ 3 | data/recent.json 4 | data/simulator.db 5 | config.py 6 | dings.txt 7 | ff.txt 8 | config.yml 9 | *.db 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'twittbot', '~> 0.6' 4 | gem 'sqlite3' 5 | gem 'ruby_markovify', git: 'https://github.com/meew0/ruby_markovify' 6 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | :consumer_key: FYRuQcDbPAXAyVjuPZMuw 3 | :consumer_secret: KiLCYTftPdxNebl5DNcj7Ey2Y8YVZu7hfqiFRYkcg 4 | :access_token: '' 5 | :access_token_secret: '' 6 | :track: [] 7 | :admins: [] 8 | :dm_command_prefix: "!" 9 | :debug: true 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # timeline 2 | 3 | Takes tweets from a bot's followings and markovifies them. 4 | 5 | This is the Ruby port of the [original code](https://github.com/sneaksnake/timeline), with some enhancements: 6 | 7 | * Database migrations 8 | * Does not use deleted tweets for generation 9 | 10 | ## Instructions 11 | 12 | ### Requirements 13 | 14 | * Ruby 2.3 or newer 15 | * Bundler (install it using `gem install bundler` if you don't have it already) 16 | * FreeBSD, OpenBSD, macOS or any other decent Unix-like system 17 | 18 | ### Installation 19 | 20 | 1. Clone this git repo 21 | 2. Run `bundle install` to install the dependencies 22 | 3. Authenticate with Twitter: `twittbot auth` 23 | 4. Run the bot to start fetching tweets: `twittbot start` 24 | 5. Create an entry in your crontab which runs `twittbot cron generate` 25 | 26 | Have fun! 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | timeline 2 | 3 | Copyright (c) 2016 martin, Georg G. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GIT 2 | remote: https://github.com/meew0/ruby_markovify 3 | revision: 5b6d7771fd85060050ca9d85f7115fe42363ed9e 4 | specs: 5 | ruby_markovify (0.1.0) 6 | unidecode 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | addressable (2.5.2) 12 | public_suffix (>= 2.0.2, < 4.0) 13 | buftok (0.2.0) 14 | domain_name (0.5.20180417) 15 | unf (>= 0.0.5, < 1.0.0) 16 | equalizer (0.0.10) 17 | erubis (2.7.0) 18 | faraday (0.9.2) 19 | multipart-post (>= 1.2, < 3) 20 | http (1.0.4) 21 | addressable (~> 2.3) 22 | http-cookie (~> 1.0) 23 | http-form_data (~> 1.0.1) 24 | http_parser.rb (~> 0.6.0) 25 | http-cookie (1.0.3) 26 | domain_name (~> 0.5) 27 | http-form_data (1.0.3) 28 | http_parser.rb (0.6.0) 29 | json (1.8.6) 30 | memoizable (0.4.2) 31 | thread_safe (~> 0.3, >= 0.3.1) 32 | multipart-post (2.0.0) 33 | naught (1.1.0) 34 | oauth (0.5.4) 35 | public_suffix (3.0.2) 36 | simple_oauth (0.3.1) 37 | sqlite3 (1.3.12) 38 | thor (0.20.0) 39 | thread_safe (0.3.6) 40 | twittbot (0.6.0) 41 | erubis (~> 2.7, >= 2.7.0) 42 | oauth (~> 0.5) 43 | thor (~> 0.19) 44 | twitter (~> 5.16) 45 | twitter (5.17.0) 46 | addressable (~> 2.3) 47 | buftok (~> 0.2.0) 48 | equalizer (= 0.0.10) 49 | faraday (~> 0.9.0) 50 | http (~> 1.0) 51 | http_parser.rb (~> 0.6.0) 52 | json (~> 1.8) 53 | memoizable (~> 0.4.0) 54 | naught (~> 1.0) 55 | simple_oauth (~> 0.3.0) 56 | unf (0.1.4) 57 | unf_ext 58 | unf_ext (0.0.7.5) 59 | unidecode (1.0.0) 60 | 61 | PLATFORMS 62 | ruby 63 | 64 | DEPENDENCIES 65 | ruby_markovify! 66 | sqlite3 67 | twittbot (~> 0.6) 68 | 69 | BUNDLED WITH 70 | 1.16.3 71 | -------------------------------------------------------------------------------- /lib/generate.rb: -------------------------------------------------------------------------------- 1 | require 'ruby_markovify' 2 | require 'cgi' 3 | 4 | Twittbot::BotPart.new :generate do 5 | task :generate, desc: 'Tweets something from the markov chain' do 6 | dataset = fetch_tweets 7 | model = RubyMarkovify::ArrayText.new(dataset, state_size = 2) 8 | retries = 10 9 | while retries > 0 10 | tweet = CGI.unescapeHTML(model.make_short_sentence(280, tries: 100)) 11 | # TODO: move the if conditions for avg word length etc. to own method 12 | break if average_word_length(tweet) > 2.0 && unique?(tweet) && unique_words(tweet).length > 2 13 | retries -= 1 14 | end 15 | next if retries == 0 && average_word_length(tweet) <= 2.0 && exists?(tweet) && unique_words(tweet).length > 2 16 | 17 | tweet_obj = bot.tweet(tweet) 18 | update_post(tweet_obj) 19 | end 20 | 21 | def fetch_tweets 22 | rows = dosql("SELECT id, text FROM tweets WHERE deleted_at IS NULL AND user_id NOT IN (SELECT id FROM users WHERE protected = 1) ORDER BY id DESC LIMIT 150", nil, "Tweet Load") 23 | rows.map{ |row| row[1] } 24 | end 25 | 26 | def unique?(text) 27 | return false if text.nil? 28 | return false if exists?(text) 29 | dosql("INSERT INTO posts (text, created_at) VALUES (?, ?);", 30 | [text, Time.now.to_i], 31 | "Post Insert") 32 | true 33 | end 34 | 35 | def exists?(text) 36 | return true if text.nil? 37 | row = dosql("SELECT 1 AS one FROM posts WHERE text = ? LIMIT 1", [text], "Post Exists") 38 | !row.empty? 39 | end 40 | 41 | def update_post(tweet) 42 | id = dosql("SELECT id FROM posts WHERE text = ? LIMIT 1", [tweet.text], "Post Load").first.first 43 | dosql("UPDATE posts SET tweet_id = ? WHERE id = ?", [tweet.id, id], "Post Update") 44 | end 45 | 46 | def average_word_length(text) 47 | word_lengths = text.split(/\s+/).map(&:length) 48 | word_lengths.reduce(:+) / word_lengths.count.to_f 49 | end 50 | 51 | def unique_words(text) 52 | text.downcase.split(/\s+/).uniq 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/database.rb: -------------------------------------------------------------------------------- 1 | require 'sqlite3' 2 | 3 | $db = SQLite3::Database.new(File.expand_path("../../data/simulator.db", __FILE__)) 4 | $db_output_color = 35 5 | 6 | def dosql(sql, args = nil, desc = "SQL") 7 | puts "\033[34;1m #{desc} \033[#{$db_output_color};1m #{sql} \033[0;1m#{args.nil? ? '' : args.inspect}\033[0m" if $bot[:config][:debug] 8 | $db_output_color = $db_output_color == 35 ? 36 : 35 9 | $db.execute(sql, args) 10 | end 11 | 12 | Twittbot::BotPart.new :database do 13 | 14 | TARGET_SCHEMA = 3 15 | 16 | on :load do 17 | do_migration if current_schema_version < TARGET_SCHEMA 18 | end 19 | 20 | task :migrate, desc: 'Migrate the database to the current version' do 21 | do_migration 22 | end 23 | 24 | task :reset_tweets, desc: 'Reset the tweets database' do 25 | dosql("DELETE FROM tweets;") 26 | end 27 | 28 | task :reset_posts, desc: 'Reset the posts database' do 29 | dosql("DELETE FROM posts;") 30 | end 31 | 32 | def do_migration 33 | currver = current_schema_version 34 | while currver < TARGET_SCHEMA 35 | migrate_db(currver + 1) 36 | currver = current_schema_version 37 | end 38 | end 39 | 40 | def current_schema_version 41 | return dosql("SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1") 42 | .flatten.first || 0 43 | rescue SQLite3::SQLException => _ 44 | return 0 45 | end 46 | 47 | def migrate_db(target_version) 48 | should_update = false 49 | start = Time.now 50 | puts "======== Migrating to version #{target_version}" 51 | case target_version 52 | when 1 53 | should_update = true 54 | dosql <<-SQL 55 | CREATE TABLE IF NOT EXISTS schema_migrations ( 56 | version INTEGER PRIMARY KEY 57 | ); 58 | SQL 59 | dosql <<-SQL 60 | CREATE TABLE IF NOT EXISTS users ( 61 | id INTEGER PRIMARY KEY, 62 | screen_name TEXT, 63 | created_at INTEGER 64 | ); 65 | SQL 66 | dosql <<-SQL 67 | CREATE TABLE IF NOT EXISTS tweets ( 68 | id INTEGER PRIMARY KEY, 69 | user_id INTEGER, 70 | text TEXT, 71 | created_at INTEGER 72 | ); 73 | SQL 74 | dosql <<-SQL 75 | CREATE TABLE IF NOT EXISTS posts ( 76 | id INTEGER PRIMARY KEY, 77 | text TEXT, 78 | created_at INTEGER, 79 | tweet_id INTEGER 80 | ); 81 | SQL 82 | when 2 83 | should_update = true 84 | dosql("ALTER TABLE tweets ADD COLUMN deleted_at INTEGER;") 85 | when 3 86 | should_update = true 87 | dosql <<-SQL 88 | ALTER TABLE users 89 | ADD COLUMN protected INTEGER 90 | CONSTRAINT users_protected_not_null NOT NULL 91 | CONSTRAINT users_protected_default DEFAULT 0; 92 | SQL 93 | dosql <<-SQL 94 | CREATE INDEX index_users_on_protected ON users (protected); 95 | SQL 96 | end 97 | update_schema_migrations(target_version) if should_update 98 | puts "======== Migration successful. Took #{(Time.now - start).round(3).to_s.ljust(5, '0')}s" 99 | end 100 | 101 | def update_schema_migrations(target_version) 102 | dosql("INSERT INTO schema_migrations (version) VALUES (?)", [target_version]) 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/fetch.rb: -------------------------------------------------------------------------------- 1 | Twittbot::BotPart.new :fetch do 2 | if $bot[:stream] 3 | puts "streaming mode ON" 4 | 5 | on :tweet do |tweet, opts| 6 | # only require tweets from user stream 7 | next unless opts[:stream_type] == :user 8 | next unless clean?(tweet, opts) 9 | 10 | upsert_user(tweet.user) 11 | 12 | begin 13 | dosql("INSERT INTO tweets (id, user_id, text, created_at) VALUES (?, ?, ?, ?)", 14 | [tweet.id, tweet.user.id, tweet.expanded_text, tweet.created_at.to_i], 15 | "Tweet Insert") 16 | rescue => e 17 | puts "Exception while inserting tweet: #{e.message}" 18 | end 19 | end 20 | 21 | on :deleted do |tweet, opts| 22 | next unless opts[:stream_type] == :user 23 | 24 | begin 25 | dosql("UPDATE tweets SET deleted_at = ? WHERE id = ?", 26 | [Time.now.to_i, tweet.id], 27 | "Tweet Delete") 28 | rescue => e 29 | puts "Exception while marking tweet as deleted: #{e.message}" 30 | end 31 | end 32 | else 33 | puts ":: streaming mode OFF, will fetch periodically (every minute or so)" 34 | 35 | every 1, :minute do 36 | fetch_last_tweets 37 | end 38 | end 39 | 40 | task :fetch, desc: "Fetches the latest tweets from the home timeline" do 41 | fetch_last_tweets 42 | end 43 | 44 | def fetch_last_tweets 45 | until (tweets = fetch_tweets_since(last_id)).empty? 46 | clean_tweets = tweets.select { |tweet| clean?(tweet, retweet: tweet.retweet?) } 47 | puts "fetched #{tweets.count} tweets, out of which #{clean_tweets.count} are clean" 48 | return if clean_tweets.empty? 49 | 50 | clean_tweets.each do |tweet| 51 | upsert_user(tweet.user) 52 | 53 | begin 54 | dosql("INSERT INTO tweets (id, user_id, text, created_at) VALUES (?, ?, ?, ?)", 55 | [tweet.id, tweet.user.id, tweet.expanded_text, tweet.created_at.to_i], 56 | "Tweet Insert") 57 | rescue => e 58 | puts "Exception while inserting tweet: #{e.message}" 59 | end 60 | end 61 | end 62 | end 63 | 64 | def last_id 65 | dosql("SELECT MAX(id) FROM tweets;", nil, "Tweet Load").first[0] 66 | end 67 | 68 | def fetch_tweets_since(id) 69 | $bot[:client].home_timeline(since_id: id, count: 800, include_rts: false) 70 | rescue => e 71 | puts "exception while fetching tweets: #{e.class} (#{e.message})" 72 | puts "returning empty set" 73 | [] 74 | end 75 | 76 | def clean?(tweet, opts) 77 | return false if opts[:retweet] 78 | 79 | filter_users = [@config[:screen_name], 'SolideSchlange'] 80 | filter_users.each do |user| 81 | return false if tweet.user.screen_name.downcase == user.downcase 82 | end 83 | 84 | filter_regexps = [ 85 | /Wordle/i, # Wordle share output 86 | /@/, # Mentions 87 | /t\.co/, # Links 88 | /\ART\s+/, # Old-style retweets 89 | /[\u5350\u534d]/, # "Friendship Windmill" 90 | /\u262d/ # Communist propaganda 91 | ] 92 | filter_regexps.each do |regexp| 93 | return false if tweet.expanded_text =~ regexp 94 | end 95 | 96 | true 97 | end 98 | 99 | def upsert_user(user) 100 | dosql("INSERT OR REPLACE INTO users (id, screen_name, created_at, protected) VALUES (?, ?, ?, ?);", 101 | [user.id, user.screen_name, user.created_at.to_i, (user.protected? ? 1 : 0)], 102 | "User Upsert") 103 | rescue => e 104 | puts "Exception while upserting user: #{e.message}" 105 | end 106 | end 107 | --------------------------------------------------------------------------------