├── .gitignore ├── .replit ├── LICENSE ├── README.md ├── drivers └── chromedriver ├── main.py ├── modules ├── compare.py ├── file_io.py ├── scraper.py ├── stats.py └── utils.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | exports/ 2 | 3 | __pycache__/ 4 | *.pyc 5 | -------------------------------------------------------------------------------- /.replit: -------------------------------------------------------------------------------- 1 | language = "python3" 2 | run = "pip3 install -r requirements.txt" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Francesco Bonomi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Instagram followers scraper 2 | 3 | This tool uses Python and [Selenium](http://www.seleniumhq.org/) to scrape the list of followers and people following a user and see the difference with a previously recorded list. 4 | 5 | ## Table of Contents 6 | 7 | - [Requirements](#requirements) 8 | - [Installation](#installation) 9 | - [Usage](#usage) 10 | - [TODO](#todo) 11 | 12 | ## Requirements 13 | 14 | You need to have Python 3 and PIP installed. You can follow [these installation instructions](http://python-guide-pt-br.readthedocs.io/en/latest/starting/install/osx/). You also need to have Chrome installed as Selenium uses the `chromedriver` contained in the `drivers` folder. 15 | 16 | Finally, you'll need your Instagram credentials to log in. At this time the dialog with the list of followers cannot be opened as an anonymous user. 17 | 18 | 19 | ## Installation 20 | 21 | Download this project manually or clone the repo with git: 22 | 23 | ```bash 24 | git clone git@github.com:frabonomi/instagram-followers-scraper.git 25 | ``` 26 | 27 | Then go to the directory and install the required dependencies 28 | 29 | ```bash 30 | cd instagram-followers-scraper 31 | pip3 install -r requirements.txt 32 | ``` 33 | 34 | ## Usage 35 | 36 | After installing the dependencies run the `main.py` file with Python 3: 37 | 38 | ```bash 39 | python3 main.py 40 | ``` 41 | 42 | You'll be asked to input the username that you want to analyze and your Instagram credentials. You can get info about followers, following or both. The data will be stored in the `exports` folder. 43 | 44 | ## TODO 45 | 46 | - Speed up scraping of the users. Right now scraping is quite slow and can be improved 47 | - Handle wrong credentials or missing username 48 | [![Run on Repl.it](https://repl.it/badge/github/tonoli/instagram-followers-scraper)](https://repl.it/github/tonoli/instagram-followers-scraper) -------------------------------------------------------------------------------- /drivers/chromedriver: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tonoli/instagram-followers-scraper/4649047ee090f2a2cae83ed1c4aabe8187b23b78/drivers/chromedriver -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from modules import compare 3 | from modules import file_io 4 | from modules import stats 5 | 6 | from modules.scraper import Scraper 7 | from modules.utils import ask_input, ask_multiple_option 8 | 9 | 10 | groups = ['followers', 'following'] 11 | 12 | # Ask for input 13 | target = ask_input('Enter the target username: ') 14 | group = ask_multiple_option(options = groups + ['both']); 15 | print('\nEnter your Instagram credentials') 16 | username = ask_input('Username: ') 17 | password = ask_input(is_password = True) 18 | 19 | def scrape(group): 20 | differs = False 21 | scraper = Scraper(target) 22 | startTime = datetime.now() 23 | 24 | scraper.authenticate(username, password) 25 | users = scraper.get_users(group, verbose=True) 26 | scraper.close() 27 | 28 | last_users = file_io.read_last(target, group) 29 | if last_users: 30 | differs = bool(compare.get_diffs(users, last_users)) 31 | 32 | if (differs or not last_users): 33 | file_io.store(target, group, users) 34 | # Stats 35 | stats.numbers(len(users), scraper.expected_number) 36 | if (differs): stats.diff(users, last_users) 37 | print('Took ' + str(datetime.now() - startTime)) 38 | 39 | if (group == 'both'): 40 | for group in groups: 41 | scrape(group) 42 | else: 43 | scrape(group) 44 | -------------------------------------------------------------------------------- /modules/compare.py: -------------------------------------------------------------------------------- 1 | def get_diffs(list1, list2): 2 | """Finds new users and lost users and returns a tuple containing them in 3 | this order. If there's no difference return false.""" 4 | 5 | new = list(set(list1) - set(list2)) 6 | lost = list(set(list2) - set(list1)) 7 | 8 | if bool(new + lost): 9 | return (new, lost) 10 | else: 11 | return False 12 | -------------------------------------------------------------------------------- /modules/file_io.py: -------------------------------------------------------------------------------- 1 | """Utility functions to read and write the exports of a user.""" 2 | 3 | from datetime import datetime 4 | import glob 5 | import ntpath 6 | import os 7 | import pickle 8 | import time 9 | 10 | 11 | def _base_directory(username): 12 | """Get the directory where the file that needs to be read or written is.""" 13 | 14 | return 'exports/{}'.format(username) 15 | 16 | 17 | def _write(path, data): 18 | """Write a pickle file at a specified path. If the folder does not exist 19 | yet, create it.""" 20 | 21 | os.makedirs(os.path.dirname(path), exist_ok = True) 22 | try: 23 | pickle.dump(data, open(path, 'wb')) 24 | return True 25 | except: 26 | return False 27 | 28 | 29 | def _read(path): 30 | """Read a pickle file at a specified path.""" 31 | 32 | with (open(path, 'rb')) as file: 33 | try: 34 | return pickle.load(file) 35 | except: 36 | return False 37 | 38 | 39 | def store(username, group, usersList): 40 | """Store the scraped users list in a file saved into a folder that has the 41 | same name as the username.""" 42 | 43 | date = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%dT%H:%M:%S') 44 | path = '{}/{}{}.pkl'.format(_base_directory(username), group, date) 45 | return _write(path, usersList) 46 | 47 | 48 | def read_last(username, group, before_last=1): 49 | """Read the last file that was stored for a specific username.""" 50 | 51 | files = glob.glob('{}/*.pkl'.format(_base_directory(username))) 52 | group_files = list( 53 | filter(lambda path: group in ntpath.basename(path), files)) 54 | try: 55 | return _read(group_files[-before_last]) 56 | except: 57 | return False 58 | -------------------------------------------------------------------------------- /modules/scraper.py: -------------------------------------------------------------------------------- 1 | import re 2 | import time 3 | 4 | from selenium import webdriver 5 | from selenium.webdriver.common.keys import Keys 6 | from selenium.webdriver.common.by import By 7 | from selenium.webdriver.support.ui import WebDriverWait 8 | from selenium.webdriver.support import expected_conditions as EC 9 | 10 | 11 | class Scraper(object): 12 | """Able to start up a browser, to authenticate to Instagram and get 13 | followers and people following a specific user.""" 14 | 15 | 16 | def __init__(self, target): 17 | self.target = target 18 | self.driver = webdriver.Chrome('drivers/chromedriver') 19 | 20 | 21 | def close(self): 22 | """Close the browser.""" 23 | 24 | self.driver.close() 25 | 26 | 27 | def authenticate(self, username, password): 28 | """Log in to Instagram with the provided credentials.""" 29 | 30 | print('\nLogging in…') 31 | self.driver.get('https://www.instagram.com') 32 | 33 | # Go to log in 34 | login_link = WebDriverWait(self.driver, 5).until( 35 | EC.presence_of_element_located((By.LINK_TEXT, 'Log in')) 36 | ) 37 | login_link.click() 38 | 39 | # Authenticate 40 | username_input = self.driver.find_element_by_xpath( 41 | '//input[@placeholder="Username"]' 42 | ) 43 | password_input = self.driver.find_element_by_xpath( 44 | '//input[@placeholder="Password"]' 45 | ) 46 | 47 | username_input.send_keys(username) 48 | password_input.send_keys(password) 49 | password_input.send_keys(Keys.RETURN) 50 | time.sleep(1) 51 | 52 | 53 | def get_users(self, group, verbose = False): 54 | """Return a list of links to the users profiles found.""" 55 | 56 | self._open_dialog(self._get_link(group)) 57 | 58 | print('\nGetting {} users…{}'.format( 59 | self.expected_number, 60 | '\n' if verbose else '' 61 | )) 62 | 63 | links = [] 64 | last_user_index = 0 65 | updated_list = self._get_updated_user_list() 66 | initial_scrolling_speed = 5 67 | retry = 2 68 | 69 | # While there are more users scroll and save the results 70 | while updated_list[last_user_index] is not updated_list[-1] or retry > 0: 71 | self._scroll(self.users_list_container, initial_scrolling_speed) 72 | 73 | for index, user in enumerate(updated_list): 74 | if index < last_user_index: 75 | continue 76 | 77 | try: 78 | link_to_user = user.find_element(By.TAG_NAME, 'a').get_attribute('href') 79 | last_user_index = index 80 | if link_to_user not in links: 81 | links.append(link_to_user) 82 | if verbose: 83 | print( 84 | '{0:.2f}% {1:s}'.format( 85 | round(index / self.expected_number * 100, 2), 86 | link_to_user 87 | ) 88 | ) 89 | except: 90 | if (initial_scrolling_speed > 1): 91 | initial_scrolling_speed -= 1 92 | pass 93 | 94 | updated_list = self._get_updated_user_list() 95 | if updated_list[last_user_index] is updated_list[-1]: 96 | retry -= 1 97 | 98 | print('100% Complete') 99 | return links 100 | 101 | 102 | def _open_dialog(self, link): 103 | """Open a specific dialog and identify the div containing the users 104 | list.""" 105 | 106 | link.click() 107 | self.expected_number = int( 108 | re.search('(\d+)', link.text).group(1) 109 | ) 110 | time.sleep(1) 111 | self.users_list_container = self.driver.find_element_by_xpath( 112 | '//div[@role="dialog"]//ul/parent::div' 113 | ) 114 | 115 | 116 | def _get_link(self, group): 117 | """Return the element linking to the users list dialog.""" 118 | 119 | print('\nNavigating to %s profile…' % self.target) 120 | self.driver.get('https://www.instagram.com/%s/' % self.target) 121 | try: 122 | return WebDriverWait(self.driver, 5).until( 123 | EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, group)) 124 | ) 125 | except: 126 | self._get_link(self.target, group) 127 | 128 | 129 | def _get_updated_user_list(self): 130 | """Return all the list items included in the users list.""" 131 | 132 | return self.users_list_container.find_elements(By.XPATH, 'ul//li') 133 | 134 | 135 | def _scroll(self, element, times = 1): 136 | """Scroll a specific element one or more times with small delay between 137 | them.""" 138 | 139 | while times > 0: 140 | self.driver.execute_script( 141 | 'arguments[0].scrollTop = arguments[0].scrollHeight', 142 | element 143 | ) 144 | time.sleep(.2) 145 | times -= 1 146 | -------------------------------------------------------------------------------- /modules/stats.py: -------------------------------------------------------------------------------- 1 | """Utilities to print stats about the collected set of users.""" 2 | 3 | from modules import compare 4 | 5 | 6 | def numbers(found, expected): 7 | """Print number statistics about the collected list of users.""" 8 | 9 | print('\n\nNumber of users: %i' % found) 10 | if found < expected: 11 | mean_users = expected - found 12 | print( 13 | 'Expected {} users but only found {}. {} {} probably blocked you.'.format( 14 | expected, 15 | found, 16 | mean_users, 17 | 'people' if mean_users != 1 else 'person' 18 | ) 19 | ) 20 | 21 | 22 | def diff(current = [], previous = []): 23 | """Print information about the diff between the last collected list of 24 | users and the previous one.""" 25 | 26 | new_users, lost_users = compare.get_diffs(current, previous) 27 | new_length = len(new_users) 28 | lost_length = len(lost_users) 29 | 30 | if (new_length + lost_length) > 0: 31 | def print_users(users, lost=False): 32 | print('{} {} user{}:'.format( 33 | len(users), 34 | 'removed' if lost else 'new', 35 | 's' if len(users) != 1 else '') 36 | ) 37 | for user in users: 38 | print(user) 39 | 40 | print('\n' + '-' * 10 + '\n') 41 | if (new_length is not 0): 42 | print_users(new_users) 43 | if (lost_length is not 0): 44 | print_users(lost_users, True) 45 | -------------------------------------------------------------------------------- /modules/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions.""" 2 | 3 | import getpass 4 | 5 | 6 | def ask_input(prompt = '', is_password = False): 7 | """Keep asking for input until it's empty.""" 8 | 9 | while True: 10 | answer = getpass.getpass() if is_password == True else input(prompt) 11 | if answer is not '': 12 | return answer 13 | 14 | 15 | def ask_multiple_option(options, prefix = 'Choose between', prompt = ': '): 16 | """Keep asking for input until it's empty or not in range.""" 17 | 18 | def exists(index): 19 | return 0 <= index < len(options) 20 | 21 | while True: 22 | print(prefix) 23 | for index, option in enumerate(options): 24 | print(' {} - {}'.format(index + 1, option)) 25 | answer = input(prompt).strip() 26 | if answer is not '': 27 | index = int(answer) - 1 28 | if exists(index): 29 | return options[index] 30 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | selenium==3.4.3 2 | --------------------------------------------------------------------------------