├── .gitignore ├── Battery └── charging.py ├── Bitcoin └── bitcoin.py ├── Cricket_Calender └── cricket_calender.py ├── Cricket_Rankings ├── cricket_rankings.py └── demo.md ├── Cricket_scores └── cricket_scores.py ├── Crypto ├── crypto.csv ├── crypto.py ├── crypto.xlsx ├── crypto2.csv ├── crypto2.py ├── crypto2.xlsx └── requirements.txt ├── File_organiser ├── clean_folder.py └── demo.md ├── Geeks For Geeks └── geeks_for_geeks_dp.py ├── Hackathon ├── hackathon1.py ├── hackathon2.py ├── hackerearth_hackathons.py ├── hackerearth_improved.py └── hackerearth_improved_table.py ├── IMDB └── charts.py ├── Img-Download └── img.py ├── Is it up ├── json_version.py └── text_version.py ├── LICENSE ├── Location └── location.py ├── News └── news.py ├── PDF └── pdf_to_text.py ├── Quotes ├── quote1.py └── quote_of_the_day.py ├── README.md ├── Saavn └── charts.py ├── Synonyms └── synonyms.py ├── Twitter ├── favorites_count.py ├── followers_count.py ├── following_count.py ├── tweet_count.py └── tweet_scrape.py ├── Wallpaper ├── bing.py └── demo.md ├── Weather └── weather.py ├── Whatsapp ├── message.py └── updated.py ├── Wordcloud ├── twitter_mask.png ├── wordcloud.py ├── wordcloud2.py └── wordcloud3.py └── Youtube Downloader └── saavn.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | -------------------------------------------------------------------------------- /Battery/charging.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from time import sleep 3 | import notify2 4 | notify2.init("charge") 5 | 6 | battery_status_file = "/sys/class/power_supply/BAT1/uevent" 7 | 8 | try: 9 | while True: 10 | with open(battery_status_file) as state: 11 | 12 | for s in state: 13 | if s.split('=')[0] == 'POWER_SUPPLY_STATUS': 14 | status = s.split('=')[1].strip('\n') 15 | 16 | if s.split('=')[0] == 'POWER_SUPPLY_CAPACITY': 17 | battery = s.split('=')[1].strip('\n') 18 | 19 | battery = int(battery) 20 | 21 | if battery < 40 and status == 'Discharging': 22 | head = 'Connect Charger' 23 | message = 'Charge remaining: ' + str(battery) + '%' 24 | notify2.Notification(head, message).show() 25 | 26 | if battery > 80 and (status == 'Charging' or status == 'Full'): 27 | head = 'Disconnect Charger' 28 | message = 'Charge remaining: ' + str(battery) + '%' 29 | notify2.Notification(head, message).show() 30 | 31 | sleep(15) 32 | 33 | 34 | except KeyboardInterrupt: 35 | print("\nCtrl+C pressed. Exiting.") 36 | -------------------------------------------------------------------------------- /Bitcoin/bitcoin.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | res = requests.get('https://www.zebapi.com/api/v1/market/ticker/btc/inr') 4 | 5 | data = res.json() 6 | 7 | print("Buy at : Rs", data['buy']) 8 | print("Sell at : Rs", data['sell']) 9 | -------------------------------------------------------------------------------- /Cricket_Calender/cricket_calender.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from terminaltables import DoubleTable 3 | from terminaltables import DoubleTable 4 | from colorclass import Color 5 | import requests 6 | 7 | 8 | def scrape(): 9 | res=requests.get('http://synd.cricbuzz.com/j2me/1.0/sch_calender.xml') 10 | 11 | while(res.status_code!=200): 12 | res=requests.get('http://synd.cricbuzz.com/j2me/1.0/sch_calender.xml') 13 | 14 | soup=BeautifulSoup(res.content,'lxml') 15 | data=soup.find_all('mch') 16 | 17 | return data 18 | 19 | 20 | def make_table(team1,team2=''): 21 | data=scrape() 22 | table_data=[['Match','Series','Date','Month','Time']] 23 | 24 | for i in data[:]: 25 | row=[] 26 | if team1.strip(' ') in i.get('desc') and team2.strip(' ') in i.get('desc') : 27 | row.extend((Color('{autoyellow}'+i.get('desc')+'{/autoyellow}'),Color('{autocyan}'+i.get('srs')[5:]+'{/autocyan}'),Color('{autored}'+i.get('ddt')+'{/autored}'),Color('{autogreen}'+i.get('mnth_yr')+'{/autogreen}'),Color('{autoyellow}'+i.get('tm')+'{/autoyellow}'))) 28 | table_data.append(row) 29 | 30 | table_instance = DoubleTable(table_data) 31 | table_instance.inner_row_border = True 32 | 33 | print(table_instance.table) 34 | print() 35 | 36 | 37 | def menu(): 38 | print(' Search by team') 39 | print(' 1.One team: ') 40 | print(' 2.Two teams: ') 41 | choice=input('\n Enter your choice:') 42 | 43 | while(choice not in ['1','2']): 44 | print(Color('{autored}\n Wrong choice{/autored}')) 45 | choice=input('\n Enter your choice:') 46 | 47 | teams=['Ind','SL','Aus','Ban','RSA','ZIM','NZ','Eng','WI'] 48 | print() 49 | print(teams) 50 | 51 | return int(choice) 52 | 53 | 54 | def main(): 55 | 56 | choice=menu() 57 | 58 | if(choice==1) : 59 | team=input("\n Enter team's name:") 60 | while(len(team)<=1): 61 | print(Color('{autored}\n Enter valid team name{/autored}')) 62 | team=input("\n Enter team's name:") 63 | 64 | make_table(team) 65 | 66 | else: 67 | team1=input("\n Enter name of team 1:") 68 | team2=input(" Enter name of team 2:") 69 | while(len(team1)<=1 or len(team2)<=1 or team1.strip(' ')==team2.strip(' ')): 70 | print(Color('{autored}\n Enter valid team names{/autored}')) 71 | team1=input("\n Enter name of team 1:") 72 | team2=input(" Enter name of team 2:") 73 | make_table(team1,team2) 74 | 75 | 76 | if __name__ == '__main__': 77 | main() 78 | -------------------------------------------------------------------------------- /Cricket_Rankings/cricket_rankings.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | from prettytable import PrettyTable 4 | 5 | 6 | def Menu(): 7 | print('\n1. Men \n2. Women\n') 8 | gen=Gender() 9 | print('\n1. Team Rankings \n2. Player Ranking\n') 10 | tp=TeamOrPlayer() 11 | 12 | mode='' 13 | val='' 14 | 15 | if gen=='mens': 16 | print('\n1. Test\n2. ODI\n3. T20\n') 17 | mode=Mode() 18 | 19 | if tp=='player-rankings': 20 | if mode=='': 21 | print('\n1. ODI\n2. T20\n') 22 | mode=Mode2() 23 | print('\n1. Batting\n2. Bowling\n3. All-Rounder\n') 24 | val=Value() 25 | 26 | return gen,tp,mode,val 27 | 28 | 29 | def Gender(): 30 | gender=input('Enter your choice:') 31 | code={'1':'mens','2':'womens'} 32 | 33 | if gender in code: 34 | return code[gender] 35 | 36 | else: 37 | print('\nInvalid Input\nTry Again\n') 38 | return Gender(); 39 | 40 | 41 | def TeamOrPlayer(): 42 | choice=input('Enter your choice:') 43 | tp={'1':'team-rankings','2':'player-rankings'} 44 | 45 | if choice in tp: 46 | return tp[choice] 47 | 48 | else: 49 | print('\nInvalid Input\nTry Again\n') 50 | return TeamOrPlayer(); 51 | 52 | 53 | def Mode(): 54 | choice=input('Enter your choice:') 55 | word={'1':'/test','2':'/odi','3':'/t20i'} 56 | 57 | if choice in word: 58 | return word[choice] 59 | 60 | else: 61 | print('\nInvalid Input\nTry Again\n') 62 | return Choice(); 63 | 64 | 65 | def Mode2(): 66 | choice=input('Enter your choice:') 67 | word={'1':'/odi','2':'/t20i'} 68 | 69 | if choice in word: 70 | return word[choice] 71 | 72 | else: 73 | print('\nInvalid Input\nTry Again\n') 74 | return Choice(); 75 | 76 | 77 | def Value(): 78 | choice=input('Enter your choice:') 79 | val={'1':'batting','2':'bowling','3':'all-rounder'} 80 | 81 | if choice in val: 82 | return val[choice] 83 | 84 | else: 85 | print('\nInvalid Input\nTry Again\n') 86 | return Value() 87 | 88 | 89 | def URL(): 90 | gen,tp,mode,val=Menu() 91 | url='https://www.icc-cricket.com/rankings/'+gen+'/'+tp+mode+'/'+val 92 | header=gen.upper() +' ' +mode[1:].upper() + ' ' + val.upper() 93 | print('\n{:<15} {:<30}\n{:<15} {:<30}'.format('',tp.upper(),'',header)) 94 | return url,tp 95 | 96 | 97 | def SOUP(url,tp): 98 | try: 99 | res=requests.get(url) 100 | soup=BeautifulSoup(res.text,'lxml') 101 | a= soup.find_all('tr',{'class':'table-body'}) 102 | data={} 103 | 104 | for i in a : 105 | team=[] 106 | name='' 107 | rating='' 108 | 109 | try: 110 | rank=int(i.contents[1].text) 111 | except: 112 | pass 113 | 114 | try: 115 | name=i.contents[3].text.replace('\n','') 116 | name=" ".join(name.split()) 117 | if rank==1 and tp=='player-rankings': 118 | name=name[0:-3] 119 | except: 120 | pass 121 | 122 | try: 123 | rating=i.contents[9].text 124 | except: 125 | if rank==1 : 126 | rating=i.contents[5].text 127 | else: 128 | rating=i.contents[7].text 129 | 130 | team.extend([name,rating]) 131 | data[rank]=team 132 | 133 | return data 134 | 135 | except: 136 | return SOUP(url,tp) 137 | 138 | 139 | def Print(data): 140 | print('\nRANKING \t TEAM\t\t\t\tRATING') 141 | for i in sorted(data): 142 | print('{:<10} '.format(i),end='') 143 | for j in range(len(data[i])): 144 | print('{:<26}'.format(data[i][j]),end=' ') 145 | print() 146 | 147 | 148 | def main(): 149 | 150 | url,tp=URL() 151 | data=SOUP(url,tp) 152 | Print(data) 153 | 154 | 155 | if __name__=='__main__': 156 | main() 157 | -------------------------------------------------------------------------------- /Cricket_Rankings/demo.md: -------------------------------------------------------------------------------- 1 | Here is a little demo of working script 2 | 3 | ![cricket](https://user-images.githubusercontent.com/22851705/28251597-835488b2-6a9e-11e7-89ee-785c6e3338b0.gif) 4 | -------------------------------------------------------------------------------- /Cricket_scores/cricket_scores.py: -------------------------------------------------------------------------------- 1 | from pycricbuzz import Cricbuzz 2 | from pprint import pprint 3 | import json 4 | 5 | c=Cricbuzz() 6 | 7 | def match_id(desc): 8 | all_matches = c.matches() 9 | for match in all_matches: 10 | if match['mchdesc'].title() == desc: 11 | return match['id'] 12 | else: 13 | return None 14 | 15 | 16 | def all_matches(): 17 | match_data = c.matches() 18 | matches = [] 19 | for match in match_data: 20 | matches.append(match['mchdesc']) 21 | return matches 22 | 23 | 24 | def live_score(desc): 25 | mid = match_id(desc) 26 | data = c.livescore(mid) 27 | score = {} 28 | score['matchinfo'] = "{}, {}".format(data['matchinfo']['mnum'],data['matchinfo']['mchdesc']) 29 | score['status'] = "{}, {}".format(data['matchinfo']['mchstate'].title(),data['matchinfo']['status']) 30 | score['bowling'] = data['bowling'] 31 | score['batting'] = data['batting'] 32 | 33 | text = '' 34 | text += score['matchinfo'] + '\n' + score['status'] + '\n\n' 35 | text += score['batting']['team'] + '\n' 36 | 37 | for scr in reversed(score['batting']['score']): 38 | text += "{} :- {}/{} in {} overs\n".format(scr['desc'],scr['runs'],scr['wickets'],scr['overs']) 39 | for b in reversed(score['batting']['batsman']): 40 | text += "{} : {}({}) \n".format(b['name'].strip('*'),b['runs'],b['balls']) 41 | text += "\n" + score['bowling']['team'] + '\n' 42 | for scr in reversed(score['bowling']['score']): 43 | text += "{} :- {}/{} in {} overs\n".format(scr['desc'],scr['runs'],scr['wickets'],scr['overs']) 44 | for b in reversed(score['bowling']['bowler']): 45 | text += "{} : {}/{} \n".format(b['name'].strip('*'),b['wickets'],b['runs']) 46 | 47 | return text 48 | 49 | 50 | def commentary(desc): 51 | mid = match_id(desc) 52 | data = c.commentary(mid) 53 | comm ={} 54 | comm['matchinfo'] = "{}, {}".format(data['matchinfo']['mnum'],data['matchinfo']['mchdesc']) 55 | comm['status'] = "{}, {}".format(data['matchinfo']['mchstate'].title(),data['matchinfo']['status']) 56 | comm['commentary'] = data['commentary'] 57 | text ='' 58 | text += comm['matchinfo'] + '\n' + comm['status'] + '\n\n' 59 | for com in comm['commentary']: 60 | text += "{}\n\n".format(com) 61 | 62 | return text 63 | 64 | 65 | def scorecard(desc): 66 | mid = match_id(desc) 67 | data = c.scorecard(mid) 68 | card = {} 69 | card['matchinfo'] = "{}, {}".format(data['matchinfo']['mnum'],data['matchinfo']['mchdesc']) 70 | card['status'] = "{}, {}".format(data['matchinfo']['mchstate'].title(),data['matchinfo']['status']) 71 | card['scorecard'] = data['scorecard'] 72 | text = '' 73 | text += card['matchinfo'] + '\n' + card['status'] + '\n\n' 74 | text +='*'*35 +'\n\n' 75 | 76 | for scr in reversed(card['scorecard']): 77 | text += "{} {}\n{}/{} in {} overs\n\n".format(scr['batteam'],scr['inngdesc'],scr['runs'],scr['wickets'],scr['overs']) 78 | text += "Batting\n" 79 | text += "{:<17} {:<3} {:<3} {:<3} {}\n\n".format('Name','R','B','4','6') 80 | for b in scr['batcard']: 81 | text += "{:<17} {:<3} {:<3} {:<3} {}\n{}\n\n".format(b['name'], b['runs'], b['balls'], b['fours'], b['six'], b['dismissal']) 82 | text +="-"*35 +"\n\n" 83 | text += "Bowling\n" 84 | text += "{:<17} {:<5} {:<3} {:<3} {}\n\n".format('Name','O','M','R','W') 85 | for b in scr['bowlcard']: 86 | text += "{:<17} {:<5} {:<3} {:<3} {}\n\n".format(b['name'], b['overs'], b['maidens'], b['runs'], b['wickets']) 87 | text +='*'*35 +'\n\n' 88 | return text 89 | 90 | 91 | def main(): 92 | matches = all_matches() 93 | print("\nALL MATCHES\n") 94 | for i,m in enumerate(matches,1): 95 | print("{}. {}".format(str(i),m)) 96 | choice = int(input('\nEnter choice (number): ')) 97 | while choice <1 or choice > len(matches): 98 | print('\nWrong choice') 99 | choice = int(input('\nEnter choice again: ')) 100 | 101 | desc = matches[choice-1].title() 102 | print('\n') 103 | print('1. Live Score') 104 | print('2. Full Score Card') 105 | print('3. Commentary') 106 | choice = int(input('\nEnter choice (number): ')) 107 | while choice <1 or choice > 3: 108 | print('\nWrong choice') 109 | choice = int(input('\nEnter choice again: ')) 110 | print('\n') 111 | if choice ==1: 112 | ref = 'y' 113 | while ref =='y': 114 | print(live_score(desc)) 115 | ref = input('\n\nDo you want to refresh:(y/n) ') 116 | print('\n') 117 | 118 | elif choice ==2: 119 | ref = 'y' 120 | while ref =='y': 121 | print(scorecard(desc)) 122 | ref = input('\n\nDo you want to refresh:(y/n) ') 123 | print('\n') 124 | 125 | else: 126 | ref = 'y' 127 | while ref =='y': 128 | print(commentary(desc)) 129 | ref = input('\n\nDo you want to refresh:(y/n) ') 130 | print('\n') 131 | 132 | 133 | if __name__ == '__main__': 134 | main() 135 | -------------------------------------------------------------------------------- /Crypto/crypto.csv: -------------------------------------------------------------------------------- 1 | Altcoin,Buying Price (INR),Selling Price(INR),24 hour Change (%) 2 | Bitcoin,1020930,885369,-3 3 | Ethereum,101754,77107,+1 4 | Bitcoin Cash,168628,130328,+2 5 | Ripple,136.22,102.95,-3.84 6 | Litecoin,18338,13822,0 7 | NEM,102.91,74.52,-1.44 8 | Neo,13023,9488,0 9 | Dash,77605,60271,-3 10 | OmiseGO,1722,1249,-2 11 | Ethereum Classic,3045,2167,0 12 | Qtum,4365,3221,+17 13 | Lisk,2186,1612,-3 14 | Monero,33878,25723,-2 15 | Stratis,1339,968,-8 16 | Ark,612,435,-7 17 | Zcash,47008,36637,-3 18 | Steem,516,369,0 19 | TenX,260,182,0 20 | Augur,9063,6764,+16 21 | Golem,62.66,45.39,-6.61 22 | Basic Attention Token,56.96,39.23,-5.71 23 | Factom,4955,3643,-4 24 | BitShares,38.45,26.33,+2.44 25 | PIVX,850,629,-10 26 | Siacoin,4.85,3.29,-1.09 27 | Digibyte,6.18,4.24,-2.33 28 | Civic,72.99,54.1,+0.14 29 | Dogecoin,0.78,0.51,-3.33 30 | NxT,31.59,21.24,-3.59 31 | CloakCoin,2054,1456,+26 32 | -------------------------------------------------------------------------------- /Crypto/crypto.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | import pandas as pd 4 | 5 | url = 'https://www.buyucoin.com/altcoin-rate-inr-india' 6 | 7 | 8 | def get_data(): 9 | res = requests.get(url) 10 | soup = BeautifulSoup(res.text, 'lxml') 11 | 12 | table = soup.find('table', {'id': 'inr_rate'}) 13 | 14 | all_rows = table.find_all('tr') 15 | 16 | data = [] 17 | 18 | for row in all_rows: 19 | row_data = [item.text.strip(' ') for item in row.find_all('td')] 20 | data.append(row_data) 21 | 22 | return data 23 | 24 | 25 | def print_data(data): 26 | for row in data: 27 | print(" {:<25} {:<20} {:<20} {}".format(*row)) 28 | 29 | 30 | def store_csv(data): 31 | df = pd.DataFrame(data[1:], columns=data[0], index=None) 32 | df.to_csv('Crypto.csv', index=None) 33 | 34 | 35 | def store_excel(data): 36 | df = pd.DataFrame(data[1:], columns=data[0], index=None) 37 | df.to_excel('Crypto.xlsx', sheet_name='Prices', index=None) 38 | 39 | 40 | def main(): 41 | data = get_data() 42 | print_data(data) 43 | store_csv(data) 44 | store_excel(data) 45 | 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /Crypto/crypto.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivan-0224/Python-scraping/9f7b0c2cb1649ae8dcd4f467577c1e5038719c57/Crypto/crypto.xlsx -------------------------------------------------------------------------------- /Crypto/crypto2.csv: -------------------------------------------------------------------------------- 1 | #,Name,Symbol,Market Cap,Price,Circulating Supply,Volume (24h),% 1h,% 24h,% 7d 2 | 1,BTCBitcoin,BTC,"$204,652,962,595",$12169.60,"16,816,737","$10,559,300,000",1.81%,-3.81%,-11.16% 3 | 2,ETHEthereum,ETH,"$107,128,835,124",$1103.08,"97,117,920","$3,459,250,000",2.38%,-1.63%,-17.49% 4 | 3,XRPRipple,XRP,"$57,298,678,740",$1.48,"38,739,142,811","$2,612,120,000",3.85%,-5.48%,-21.31% 5 | 4,BCHBitcoin Cash,BCH,"$31,593,962,674",$1866.85,"16,923,675","$1,060,260,000",2.66%,-1.36%,-27.55% 6 | 5,ADACardano,ADA,"$16,845,880,738",$0.649741,"25,927,070,538","$713,747,000",2.91%,-5.89%,-17.10% 7 | 6,LTCLitecoin,LTC,"$10,870,134,259",$198.16,"54,854,233","$485,364,000",2.10%,-2.66%,-18.32% 8 | 7,XEMNEM,XEM,"$10,104,299,999",$1.12,"8,999,999,999","$121,698,000",3.56%,-1.95%,-19.09% 9 | 8,NEONEO,NEO,"$9,261,330,000",$142.48,"65,000,000","$443,721,000",3.53%,0.02%,0.44% 10 | 9,XLMStellar,XLM,"$8,873,510,418",$0.495934,"17,892,522,831","$173,643,000",3.23%,-4.27%,-19.24% 11 | 10,EOSEOS,EOS,"$8,632,427,441",$13.98,"617,563,595","$2,978,960,000",3.58%,1.83%,-3.00% 12 | 11,MIOTAIOTA,MIOTA,"$8,104,304,241",$2.92,"2,779,530,283","$109,090,000",1.98%,-3.12%,-21.37% 13 | 12,DASHDash,DASH,"$6,768,834,528",$864.35,"7,831,136","$124,321,000",1.82%,-4.63%,-16.63% 14 | 13,XMRMonero,XMR,"$5,738,987,845",$367.22,"15,628,158","$173,471,000",2.09%,-5.99%,-9.06% 15 | 14,TRXTRON,TRX,"$5,172,568,098",$0.078672,"65,748,192,475","$621,809,000",2.35%,-5.18%,-16.77% 16 | 15,ICXICON,ICX,"$3,615,584,749",$9.51,"380,045,004","$107,891,000",2.59%,-2.98%,-2.78% 17 | 16,BTGBitcoin Gold,BTG,"$3,586,877,312",$213.80,"16,777,024","$231,405,000",1.31%,-4.18%,-31.31% 18 | 17,QTUMQtum,QTUM,"$3,417,828,541",$46.30,"73,823,496","$1,847,800,000",3.44%,11.45%,-9.01% 19 | 18,ETCEthereum Classic,ETC,"$3,216,213,256",$32.40,"99,276,872","$426,673,000",2.31%,-3.01%,-25.69% 20 | 19,LSKLisk,LSK,"$2,749,829,382",$23.46,"117,227,520","$50,003,800",2.01%,-4.72%,-15.23% 21 | 20,XRBRaiBlocks,XRB,"$2,541,617,843",$19.07,"133,248,289","$17,593,500",-0.30%,-0.08%,-11.83% 22 | 21,VENVeChain,VEN,"$2,191,538,801",$7.91,"277,162,633","$230,077,000",6.06%,8.08%,25.75% 23 | 22,OMGOmiseGO,OMG,"$1,857,562,204",$18.20,"102,042,552","$69,385,100",2.03%,-5.65%,-21.82% 24 | 23,PPTPopulous,PPT,"$1,689,914,702",$45.67,"37,004,027","$2,861,210",3.56%,-2.78%,-7.54% 25 | 24,XVGVerge,XVG,"$1,640,304,905",$0.113151,"14,496,601,043","$200,377,000",5.74%,3.74%,-20.12% 26 | 25,USDTTether,USDT,"$1,621,602,080",$1.00,"1,618,090,823","$3,192,660,000",-0.48%,-0.39%,-1.41% 27 | 26,ZECZcash,ZEC,"$1,596,126,007",$514.13,"3,104,506","$113,321,000",2.18%,-5.90%,-24.29% 28 | 27,SCSiacoin,SC,"$1,524,647,092",$0.048562,"31,396,146,174","$131,455,000",4.15%,-3.07%,-19.49% 29 | 28,BNBBinance Coin,BNB,"$1,464,040,807",$14.79,"99,014,000","$154,078,000",1.96%,-6.14%,-27.99% 30 | 29,STRATStratis,STRAT,"$1,412,657,364",$14.31,"98,709,227","$37,403,000",1.42%,-9.68%,-22.32% 31 | 30,BCNBytecoin,BCN,"$1,357,621,561",$0.007391,"183,676,579,755","$7,402,960",1.03%,-5.69%,-29.65% 32 | 31,STEEMSteem,STEEM,"$1,344,916,192",$5.44,"247,439,657","$87,567,700",9.90%,-0.80%,-5.08% 33 | 32,ARDRArdor,ARDR,"$1,210,108,068",$1.21,"998,999,495","$71,630,400",3.31%,-15.04%,? 34 | 33,REPAugur,REP,"$1,074,714,300",$97.70,"11,000,000","$54,482,300",4.84%,12.66%,1.13% 35 | 34,SNTStatus,SNT,"$1,074,135,555",$0.309506,"3,470,483,788","$324,048,000",2.63%,-8.19%,-27.80% 36 | 35,MKRMaker,MKR,"$1,043,766,309",$1688.32,"618,228","$863,330",0.80%,3.77%,26.53% 37 | 36,BTSBitShares,BTS,"$1,016,982,580",$0.390052,"2,607,300,000","$50,750,400",4.33%,-0.95%,-28.73% 38 | 37,KCSKuCoin Shares,KCS,"$935,239,998",$10.27,"91,043,076","$7,091,100",2.71%,1.65%,-33.17% 39 | 38,ZRX0x,ZRX,"$900,422,935",$1.81,"497,776,501","$20,015,500",3.02%,-6.84%,-12.31% 40 | 39,WAVESWaves,WAVES,"$897,613,000",$8.98,"100,000,000","$40,839,700",1.83%,-5.59%,-18.27% 41 | 40,DOGEDogecoin,DOGE,"$880,578,772",$0.007802,"112,867,799,719","$21,062,800",2.47%,-7.50%,-35.91% 42 | 41,ETNElectroneum,ETN,"$833,709,615",$0.141039,"5,911,199,136","$7,148,680",2.93%,-5.73%,-13.90% 43 | 42,VERIVeritaseum,VERI,"$816,350,628",$400.83,"2,036,645","$1,542,410",3.81%,-11.01%,-3.48% 44 | 43,KMDKomodo,KMD,"$751,405,698",$7.31,"102,859,018","$27,581,200",4.29%,-11.61%,-8.36% 45 | 44,DCRDecred,DCR,"$703,341,451",$106.57,"6,599,683","$1,520,180",0.43%,-4.14%,-4.10% 46 | 45,DRGNDragonchain,DRGN,"$701,494,569",$2.94,"238,421,940","$4,401,720",1.66%,-1.66%,-22.55% 47 | 46,WTCWalton,WTC,"$681,087,169",$27.35,"24,898,178","$58,996,600",6.25%,10.22%,2.28% 48 | 47,LRCLoopring,LRC,"$659,932,880",$1.18,"561,167,415","$31,467,200",4.89%,-4.53%,3.24% 49 | 48,DCNDentacoin,DCN,"$650,663,346",$0.002001,"325,190,215,376","$1,521,080",3.02%,-12.57%,-44.76% 50 | 49,ARKArk,ARK,"$639,540,497",$6.53,"97,981,284","$15,135,400",2.73%,-6.40%,-20.29% 51 | 50,SALTSALT,SALT,"$620,312,954",$8.72,"71,105,820","$15,545,900",2.92%,-2.96%,-19.88% 52 | 51,QASHQASH,QASH,"$604,562,000",$1.73,"350,000,000","$10,306,800",2.04%,-3.16%,-21.90% 53 | 52,DGBDigiByte,DGB,"$598,854,117",$0.061558,"9,728,227,451","$15,026,900",2.31%,-4.54%,-34.23% 54 | 53,BATBasic Attenti...,BAT,"$581,564,000",$0.581564,"1,000,000,000","$10,707,900",0.83%,-8.30%,-14.29% 55 | 54,GNTGolem,GNT,"$557,654,926",$0.668441,"834,262,000","$10,449,800",2.25%,-8.10%,-25.32% 56 | 55,HSRHshare,HSR,"$557,307,965",$13.11,"42,524,414","$103,112,000",2.73%,-3.01%,-30.47% 57 | 56,GASGas,GAS,"$528,546,230",$55.31,"9,555,638","$7,882,890",2.28%,-4.78%,-6.52% 58 | 57,KNCKyber Network,KNC,"$526,918,839",$3.93,"134,132,697","$33,754,000",1.49%,-9.62%,-6.48% 59 | 58,WAXWAX,WAX,"$515,196,646",$1.05,"492,954,537","$2,391,620",1.57%,-14.75%,-30.48% 60 | 59,ETHOSEthos,ETHOS,"$514,299,657",$6.82,"75,405,089","$3,038,580",3.14%,-3.40%,-7.22% 61 | 60,AIONAion,AION,"$497,077,023",$6.38,"77,953,424","$14,238,700",8.81%,10.84%,-17.97% 62 | 61,PIVXPIVX,PIVX,"$497,037,647",$8.97,"55,402,585","$6,944,520",1.18%,-10.94%,-24.18% 63 | 62,SMARTSmartCash,SMART,"$492,103,111",$0.800950,"614,399,290","$1,204,770",10.30%,-23.46%,-60.19% 64 | 63,FUNFunFair,FUN,"$491,596,757",$0.111244,"4,419,085,589","$19,228,300",3.55%,-9.57%,-13.25% 65 | 64,GBYTEByteball Bytes,GBYTE,"$489,877,101",$759.24,"645,222","$1,714,270",0.59%,-12.96%,-25.31% 66 | 65,RHOCRChain,RHOC,"$473,783,179",$1.94,"244,040,764","$813,890",3.47%,-7.40%,-25.96% 67 | 66,ZCLZClassic,ZCL,"$470,399,655",$149.81,"3,139,975","$11,079,700",3.29%,-9.34%,-24.95% 68 | 67,FCTFactom,FCT,"$466,899,247",$53.39,"8,745,102","$15,430,500",5.22%,-6.29%,-18.00% 69 | 68,DENTDent,DENT,"$442,240,663",$0.041663,"10,614,760,961","$11,885,600",2.92%,-4.21%,-26.64% 70 | 69,MONAMonaCoin,MONA,"$434,910,226",$7.65,"56,869,000","$21,848,600",5.30%,-0.78%,-1.89% 71 | 70,ELFaelf,ELF,"$422,620,000",$1.69,"250,000,000","$97,507,000",5.16%,-6.78%,6.42% 72 | 71,POWRPower Ledger,POWR,"$408,429,207",$1.13,"360,621,601","$50,929,900",3.08%,-8.18%,-18.27% 73 | 72,DGDDigixDAO,DGD,"$399,996,000",$200.00,"2,000,000","$4,567,320",2.67%,-2.72%,-2.38% 74 | 73,KINKin,KIN,"$398,612,366",$0.000527,"756,097,560,976","$968,183",2.03%,-6.98%,-16.97% 75 | 74,RDDReddCoin,RDD,"$381,802,784",$0.013289,"28,730,522,337","$6,385,570",3.31%,-8.26%,-30.20% 76 | 75,BTMBytom,BTM,"$379,132,362",$0.384126,"987,000,000","$17,499,300",3.60%,-6.15%,-35.36% 77 | 76,NASNebulas,NAS,"$376,775,700",$10.61,"35,500,000","$18,368,500",4.99%,8.09%,-2.43% 78 | 77,AEAeternity,AE,"$372,581,093",$1.60,"233,020,472","$1,831,420",1.23%,-6.48%,-24.08% 79 | 78,SYSSyscoin,SYS,"$365,966,947",$0.689988,"530,396,103","$7,427,050",1.40%,-3.04%,-22.27% 80 | 79,LINKChainLink,LINK,"$360,899,000",$1.03,"350,000,000","$27,480,500",6.55%,15.88%,1.20% 81 | 80,NEBLNeblio,NEBL,"$353,076,381",$27.71,"12,742,116","$14,432,500",3.56%,-5.27%,3.67% 82 | 81,ENGEnigma,ENG,"$349,547,033",$4.67,"74,836,171","$11,832,400",3.27%,-9.14%,-20.26% 83 | 82,REQRequest Network,REQ,"$342,099,399",$0.533659,"641,044,935","$18,133,400",0.60%,-8.97%,-24.55% 84 | 83,MAIDMaidSafeCoin,MAID,"$328,126,747",$0.725058,"452,552,412","$5,815,430",2.18%,-7.39%,-24.87% 85 | 84,GXSGXShares,GXS,"$326,732,400",$5.45,"60,000,000","$10,037,600",0.97%,-10.40%,-38.30% 86 | 85,SUBSubstratum,SUB,"$326,586,837",$1.44,"226,091,449","$24,573,500",2.23%,1.91%,-21.38% 87 | 86,XZCZCoin,XZC,"$325,603,837",$83.43,"3,902,649","$3,538,950",1.88%,-4.37%,-15.81% 88 | 87,NXSNexus,NXS,"$325,035,434",$5.89,"55,139,817","$2,168,020",1.78%,-12.85%,-32.32% 89 | 88,MEDMediBloc,MED,"$320,847,071",$0.108161,"2,966,384,100","$8,536,780",2.79%,-8.49%,-34.80% 90 | 89,NXTNxt,NXT,"$320,638,022",$0.320959,"998,999,942","$13,924,900",2.36%,-5.45%,-25.14% 91 | 90,XPExperience Po...,XP,"$311,484,104",$0.001472,"211,548,562,853","$2,142,670",-5.97%,-15.96%,-35.96% 92 | 91,EMCEmercoin,EMC,"$307,443,370",$7.46,"41,217,055","$2,371,990",1.18%,-18.02%,-21.39% 93 | 92,BTXBitcore,BTX,"$304,834,256",$28.32,"10,763,427","$3,435,250",0.76%,-5.31%,12.19% 94 | 93,BNTBancor,BNT,"$302,983,610",$7.43,"40,772,871","$31,287,100",2.19%,-1.33%,-22.96% 95 | 94,CNDCindicator,CND,"$295,600,994",$0.204430,"1,445,976,590","$34,619,800",3.04%,-2.96%,89.02% 96 | 95,QSPQuantstamp,QSP,"$290,803,125",$0.471078,"617,314,171","$9,293,550",2.60%,-11.04%,-5.64% 97 | 96,GAMEGameCredits,GAME,"$278,136,752",$4.32,"64,355,352","$4,806,210",1.57%,-7.87%,-30.22% 98 | 97,CNXCryptonex,CNX,"$277,775,400",$6.16,"45,075,701","$287,328",-3.27%,3.36%,10.94% 99 | 98,PAYTenX,PAY,"$277,652,849",$2.65,"104,661,310","$6,420,240",3.00%,-3.07%,-27.37% 100 | 99,ICNIconomi,ICN,"$277,599,116",$2.78,"99,788,314","$4,438,220",1.69%,-4.93%,-31.40% 101 | 100,GNOGnosis,GNO,"$271,615,367",$245.90,"1,104,590","$3,184,540",2.18%,-5.54%,-31.62% 102 | -------------------------------------------------------------------------------- /Crypto/crypto2.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | import pandas as pd 4 | 5 | url = 'https://coinmarketcap.com/all/views/all/' 6 | 7 | 8 | def get_data(): 9 | res = requests.get(url) 10 | soup = BeautifulSoup(res.text, 'lxml') 11 | 12 | table = soup.find('table', {'id': 'currencies-all'}) 13 | 14 | all_rows = table.find_all('tr') 15 | 16 | data = [[item.text.strip(' ') for item in all_rows[0].find_all('th')]] 17 | 18 | for row in all_rows[1:101]: 19 | row_data = [item.text.replace('\n', '').replace('*', '').strip(' ') for item in row.find_all('td')] 20 | data.append(row_data) 21 | 22 | return data 23 | 24 | 25 | def print_data(data): 26 | for row in data: 27 | print(*row) 28 | 29 | 30 | def store_csv(data): 31 | df = pd.DataFrame(data[1:], columns=data[0], index=None) 32 | df.to_csv('crypto2.csv', index=None) 33 | 34 | 35 | def store_excel(data): 36 | df = pd.DataFrame(data[1:], columns=data[0], index=None) 37 | df.to_excel('crypto2.xlsx', sheet_name='Prices', index=None) 38 | 39 | 40 | def main(): 41 | data = get_data() 42 | print_data(data) 43 | store_csv(data) 44 | store_excel(data) 45 | 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /Crypto/crypto2.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivan-0224/Python-scraping/9f7b0c2cb1649ae8dcd4f467577c1e5038719c57/Crypto/crypto2.xlsx -------------------------------------------------------------------------------- /Crypto/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.6.0 2 | bs4==0.0.1 3 | certifi==2018.1.18 4 | chardet==3.0.4 5 | et-xmlfile==1.0.1 6 | idna==2.6 7 | jdcal==1.3 8 | lxml==4.1.1 9 | numpy==1.14.0 10 | openpyxl==2.4.10 11 | pandas==0.22.0 12 | python-dateutil==2.6.1 13 | pytz==2017.3 14 | requests==2.18.4 15 | six==1.11.0 16 | urllib3==1.22 17 | -------------------------------------------------------------------------------- /File_organiser/clean_folder.py: -------------------------------------------------------------------------------- 1 | from colorama import Fore 2 | import os 3 | 4 | all_paths=[] 5 | dir_name = input( 'Enter the name of directory you want to clear: ') 6 | extension = set() 7 | 8 | 9 | def source_path(dir_name): 10 | for root in os.walk("/home"): 11 | if dir_name == root[0].split('/')[-1]: 12 | all_paths.append(root[0]) 13 | 14 | for i in range(len(all_paths)): 15 | print() 16 | print("{}. {}".format(i+1,all_paths[i])) 17 | 18 | if len(all_paths) == 0: 19 | print(Fore.LIGHTRED_EX + 'No directory found') 20 | exit() 21 | 22 | choice = int(input('\nEnter the option number: ')) 23 | 24 | if choice < 1 or choice > len(all_paths): 25 | print(Fore.LIGHTRED_EX +'Wrong choice entered') 26 | exit() 27 | 28 | else: 29 | path = all_paths[choice-1] 30 | 31 | return path 32 | 33 | 34 | def print_before(path): 35 | print("Cleaning {} located at {}\n".format(path.split('/')[-1],path)) 36 | 37 | print(Fore.LIGHTBLUE_EX + "Before cleaning\n" + Fore.RESET) 38 | 39 | for files in os.listdir(path): 40 | print(files,end='\t') 41 | print() 42 | 43 | 44 | def destination_path(path): 45 | os.chdir(path) 46 | 47 | for f in os.listdir(): 48 | name = (os.path.splitext(f))[0] 49 | ext = (os.path.splitext(f))[1] 50 | 51 | extension.add(ext[1:]) 52 | 53 | new_dir = "New" + path.split('/')[-1] 54 | new_dir_path = os.path.join(path,new_dir) 55 | 56 | if not os.path.exists(new_dir_path): 57 | os.mkdir(new_dir_path) 58 | 59 | return new_dir_path,new_dir 60 | 61 | 62 | def organise(new_dir_path,new_dir,path): 63 | for ext in extension: 64 | folder = os.path.join(new_dir_path,ext) 65 | 66 | if not os.path.exists(folder): 67 | os.mkdir(folder) 68 | 69 | if ext !='': 70 | for f in os.listdir(): 71 | if os.path.splitext(f)[1].strip('.') == ext: 72 | os.rename(f,os.path.join(folder,f)) 73 | 74 | else: 75 | for f in os.listdir(): 76 | if f!=new_dir and os.path.splitext(f)[1].strip('.') == ext: 77 | print(f) 78 | inner_folder = os.path.join(new_dir_path,f) 79 | 80 | if os.path.exists(inner_folder): 81 | os.chdir(os.path.join(path,f)) 82 | for file in os.listdir(): 83 | new_path = os.path.join(inner_folder,file) 84 | os.rename(file,new_path) 85 | os.rmdir(os.path.join(path,f)) 86 | 87 | else: 88 | os.rename(f,inner_folder) 89 | 90 | 91 | def print_after(path): 92 | 93 | print(Fore.LIGHTBLUE_EX + "\nAfter cleaning\n" + Fore.RESET) 94 | 95 | for files in os.listdir(path): 96 | print(files,end='\t') 97 | 98 | print(Fore.LIGHTMAGENTA_EX + "\n\nCLEANED\n" + Fore.RESET) 99 | 100 | 101 | def file_manage(): 102 | path = source_path(dir_name) 103 | print_before(path) 104 | new_dir_path, new_dir = destination_path(path) 105 | organise(new_dir_path, new_dir,path) 106 | print_after(path) 107 | 108 | 109 | file_manage() 110 | -------------------------------------------------------------------------------- /File_organiser/demo.md: -------------------------------------------------------------------------------- 1 | Here is a small preview of working script 2 | 3 | ![demo2](https://user-images.githubusercontent.com/22851705/28751921-276cefac-7503-11e7-9716-9fa17e673dde.gif) 4 | 5 | After running this script have a look at the contents of folder 6 | 7 | ![screenshot from 2017-07-30 13-32-18](https://user-images.githubusercontent.com/22851705/28751691-db5e6bd6-74fd-11e7-85c0-29da6f9b238f.png) 8 | -------------------------------------------------------------------------------- /Geeks For Geeks/geeks_for_geeks_dp.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from collections import OrderedDict 3 | import pdfkit 4 | import requests 5 | import os 6 | 7 | URL = 'http://www.geeksforgeeks.org/dynamic-programming/' 8 | config = pdfkit.configuration() 9 | 10 | data = OrderedDict() 11 | 12 | match = { 13 | 'Basic Problems': 'basicProblems', 14 | 'Medium Problems': 'mediumProblems', 15 | 'Hard Problems': 'hardProblems' 16 | } 17 | 18 | 19 | def make_soup(url): 20 | res = requests.get(url) 21 | soup = BeautifulSoup(res.text, 'lxml') 22 | return soup 23 | 24 | 25 | def get_data(type, soup): 26 | 27 | content = soup.find('div', {'class': '' + match[type]}) 28 | links = content.find_all('a') 29 | 30 | list_problems = OrderedDict() 31 | for i, link in enumerate(links[:7], 1): 32 | list_problems[link.text] = link['href'] 33 | 34 | data[type] = list_problems 35 | 36 | 37 | def pdf(type, data): 38 | current_dir = os.getcwd() 39 | folder = os.path.join(current_dir, type) 40 | if not os.path.exists(folder): 41 | os.mkdir(folder) 42 | 43 | for problem_name in data[type]: 44 | link = data[type][problem_name] 45 | pdf_name = problem_name + ".pdf" 46 | try: 47 | pdfkit.from_url(link, os.path.join(folder, pdf_name), configuration=config) 48 | except: 49 | pass 50 | 51 | 52 | def main(): 53 | soup = make_soup(URL) 54 | for type in match: 55 | get_data(type, soup) 56 | pdf(type, data) 57 | 58 | 59 | if __name__ == '__main__': 60 | main() 61 | -------------------------------------------------------------------------------- /Hackathon/hackathon1.py: -------------------------------------------------------------------------------- 1 | ''' Web Scraped from https://hackevents.co site ''' 2 | 3 | import requests 4 | from bs4 import BeautifulSoup 5 | 6 | res = requests.get('https://hackevents.co/hackathons') 7 | soup = BeautifulSoup(res.text, 'lxml') 8 | hacks = soup.find_all('div',{'class':'hackathon '}) 9 | 10 | for i,f in enumerate(hacks,1): 11 | month = f.find('div',{'class':'date'}).find('div',{'class':'date-month'}).text.strip() 12 | date = f.find('div',{'class':'date'}).find('div',{'class':'date-day-number'}).text.strip() 13 | days = f.find('div',{'class':'date'}).find('div',{'class':'date-week-days'}).text.strip() 14 | final_date = "{} {}, {} ".format(date, month, days ) 15 | 16 | name = f.find('div',{'class':'info'}).find('h2').text.strip() 17 | 18 | city = f.find('div',{'class':'info'}).find('p').find('span',{'class':'city'}).text.strip() 19 | country = f.find('div',{'class':'info'}).find('p').find('span',{'class':'country'}).text.strip() 20 | 21 | print("{:<5} {:<15} : {:<90} : {}, {}\n ".format(str(i)+')',final_date, name.title(), city, country)) 22 | -------------------------------------------------------------------------------- /Hackathon/hackathon2.py: -------------------------------------------------------------------------------- 1 | """ Web scraped from http://www.hackathon.io/ """ 2 | 3 | import requests 4 | from bs4 import BeautifulSoup 5 | 6 | res = requests.get('http://www.hackathon.io/events') 7 | soup = BeautifulSoup(res.text, 'lxml') 8 | hacks = soup.find_all('div',{'class':'event-teaser'}) 9 | 10 | for i,f in enumerate(hacks,1): 11 | print("[{}] {}\n{}\n\n".format(i,f.find('h4').text,f.find('h5').text)) 12 | -------------------------------------------------------------------------------- /Hackathon/hackerearth_hackathons.py: -------------------------------------------------------------------------------- 1 | """ Hackerearth hackathons """ 2 | 3 | from selenium import webdriver 4 | from bs4 import BeautifulSoup 5 | 6 | driver = webdriver.Chrome() 7 | driver.get('https://www.hackerearth.com/challenges/') 8 | res = driver.execute_script("return document.documentElement.outerHTML") 9 | driver.quit() 10 | soup = BeautifulSoup(res, 'lxml') 11 | upcoming = soup.find('div',{'class':'upcoming challenge-list'}) 12 | all_hackathons = upcoming.find_all('div',{'class':'challenge-content'}) 13 | 14 | for i,hackathon in enumerate(all_hackathons,1): 15 | challenge_type = hackathon.find('div',{'class':'challenge-type'}).text.replace("\n"," ").strip() 16 | challenge_name = hackathon.find('div',{'class':'challenge-name'}).text.replace("\n"," ").strip() 17 | date_time = hackathon.find('div',{'class':'challenge-list-meta challenge-card-wrapper'}).text.replace("\n"," ").strip() 18 | print("[{}] {}\n{}\n{}\n\n".format(str(i), challenge_name, challenge_type, date_time)) 19 | 20 | -------------------------------------------------------------------------------- /Hackathon/hackerearth_improved.py: -------------------------------------------------------------------------------- 1 | # Install PhantomJS to use this script 2 | 3 | from selenium import webdriver 4 | from bs4 import BeautifulSoup 5 | from time import sleep 6 | 7 | print('--- Fetching hackathons--- \n') 8 | driver = webdriver.PhantomJS() 9 | driver.get('https://www.hackerearth.com/challenges/') 10 | res = driver.page_source 11 | soup = BeautifulSoup(res, 'lxml') 12 | upcoming = soup.find('div',{'class':'upcoming challenge-list'}) 13 | all_hackathons = upcoming.find_all('div',{'class':'challenge-content'}) 14 | 15 | for i,hackathon in enumerate(all_hackathons,1): 16 | challenge_type = hackathon.find('div',{'class':'challenge-type'}).text.replace("\n"," ").strip() 17 | challenge_name = hackathon.find('div',{'class':'challenge-name'}).text.replace("\n"," ").strip() 18 | date_time = hackathon.find('div',{'class':'challenge-list-meta challenge-card-wrapper'}).text.replace("\n"," ").strip() 19 | print("[{}] {}\n{}\n{}\n\n".format(str(i), challenge_name, challenge_type, date_time)) 20 | 21 | -------------------------------------------------------------------------------- /Hackathon/hackerearth_improved_table.py: -------------------------------------------------------------------------------- 1 | # Install PhantomJS to use this script 2 | 3 | from selenium import webdriver 4 | from bs4 import BeautifulSoup 5 | from time import sleep 6 | from terminaltables import DoubleTable 7 | from colorclass import Color 8 | 9 | print('--- Fetching hackathons--- \n') 10 | driver = webdriver.PhantomJS() 11 | driver.get('https://www.hackerearth.com/challenges/') 12 | res = driver.page_source 13 | soup = BeautifulSoup(res, 'lxml') 14 | upcoming = soup.find('div',{'class':'upcoming challenge-list'}) 15 | all_hackathons = upcoming.find_all('div',{'class':'challenge-content'}) 16 | 17 | table_data = [['S.No', 'Name', 'Type', 'Timings']] 18 | 19 | for s_no,hackathon in enumerate(all_hackathons,1): 20 | row = [] 21 | challenge_type = hackathon.find('div',{'class':'challenge-type'}).text.replace("\n"," ").strip() 22 | challenge_name = hackathon.find('div',{'class':'challenge-name'}).text.replace("\n"," ").strip() 23 | date_time = hackathon.find('div',{'class':'challenge-list-meta challenge-card-wrapper'}).text.replace("\n"," ").strip() 24 | row.extend((Color('{autoyellow}' + str(s_no) + '.' + '{/autoyellow}'), 25 | Color('{autocyan}' + challenge_name + '{/autogreen}'), 26 | Color('{autogreen}' + challenge_type + '{/autoyellow}'), 27 | Color('{autoyellow}' + date_time + '{/autoyellow}'))) 28 | table_data.append(row) 29 | 30 | 31 | table_instance = DoubleTable(table_data) 32 | table_instance.inner_row_border = True 33 | 34 | print(table_instance.table) 35 | print() 36 | 37 | 38 | -------------------------------------------------------------------------------- /IMDB/charts.py: -------------------------------------------------------------------------------- 1 | ''' Charts and rating from IMDBB ''' 2 | 3 | from bs4 import BeautifulSoup 4 | from terminaltables import DoubleTable 5 | from colorclass import Color 6 | import requests 7 | 8 | def select(): 9 | options = { 10 | 1: ('Top movies' , 'top'), 11 | 2: ('Most Popular Movies' , 'moviemeter'), 12 | 3: ('Top English Movies' , 'top-english-movies'), 13 | 4: ('Top TV Shows' , 'toptv'), 14 | 5: ('Most Popular TV Shows' , 'tvmeter'), 15 | 6: ('Low Rated Movies', 'bottom'), 16 | 7: ('Top Box Office collection', 'boxoffice') 17 | } 18 | 19 | for i,option in enumerate(options,1): 20 | print("{}) {}".format(i,options[option][0])) 21 | 22 | choice = int(input('\nEnter your choice: ')) 23 | while(choice<1 or choice>len(options)): 24 | print('Wrong choice') 25 | choice = int(input('\nEnter your choice: ')) 26 | 27 | print() 28 | return options[choice][1] 29 | 30 | 31 | def get_data(option): 32 | res = requests.get('http://m.imdb.com/chart/'+option) 33 | soup = BeautifulSoup(res.text, 'lxml') 34 | card_list = soup.find_all('span',{'class':'media-body media-vertical-align'}) 35 | result = [] 36 | for card in card_list: 37 | try: 38 | name = card.find('h4').text.replace("\n"," ").lstrip("0123456789.- ") 39 | except: 40 | pass 41 | try: 42 | rating = card.find('p').text.strip() 43 | except: 44 | pass 45 | 46 | result.append((name,rating)) 47 | 48 | return result 49 | 50 | 51 | def make_table(result): 52 | table_data = [['S.No', 'Name', 'Rating']] 53 | 54 | for s_no,res in enumerate(result,1): 55 | row = [] 56 | row.extend((Color('{autoyellow}' + str(s_no) + '.' + '{/autoyellow}'), 57 | Color('{autogreen}' + res[0] + '{/autogreen}'), 58 | Color('{autoyellow}' + res[1] + '{/autoyellow}'))) 59 | table_data.append(row) 60 | 61 | table_instance = DoubleTable(table_data) 62 | table_instance.inner_row_border = True 63 | 64 | print(table_instance.table) 65 | print() 66 | 67 | def main(): 68 | option = select() 69 | data = get_data(option) 70 | make_table(data) 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /Img-Download/img.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | 3 | url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/69/Muralitharan_bowling_to_Adam_Gilchrist.jpg/499px-Muralitharan_bowling_to_Adam_Gilchrist.jpg' 4 | 5 | urllib.request.urlretrieve(url, 'cricket.jpg') 6 | -------------------------------------------------------------------------------- /Is it up/json_version.py: -------------------------------------------------------------------------------- 1 | """ Check if site is up """ 2 | 3 | import requests 4 | 5 | domain = input('Enter the name of site to check (format google.com): ') 6 | res = requests.get('https://isitup.org/' + domain + '.json') 7 | 8 | data = res.json() 9 | status_code = data['status_code'] 10 | 11 | if status_code == 1: 12 | print(domain, 'is up') 13 | 14 | elif status_code == 2: 15 | print(domain, 'is down') 16 | 17 | elif status_code == 3: 18 | print('Please enter a valid domain to check availability.') 19 | 20 | else: 21 | print('Something unexpected happened!') 22 | -------------------------------------------------------------------------------- /Is it up/text_version.py: -------------------------------------------------------------------------------- 1 | ''' Check if a site is up or not ''' 2 | 3 | from bs4 import BeautifulSoup 4 | import requests 5 | 6 | url = input('Enter the name of site to check (format google.com): ') 7 | res = requests.get('https://isitup.org/' + url) 8 | soup = BeautifulSoup(res.text, 'lxml') 9 | 10 | container = soup.find('div', {'id': 'container'}) 11 | status = container.find('p') 12 | print(status.text) 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Umang Ahuja 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Location/location.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | res = requests.get('https://ipinfo.io/') 4 | data = res.json() 5 | 6 | city = data['city'] 7 | 8 | location = data['loc'].split(',') 9 | latitude = location[0] 10 | longitude = location[1] 11 | 12 | print("Latitude : ", latitude) 13 | print("Longitude : ", longitude) 14 | print("City : ", city) 15 | -------------------------------------------------------------------------------- /News/news.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from ntfy import notify 3 | import requests 4 | 5 | def get_data(): 6 | url='http://indiatoday.intoday.in/section/120/1/top-stories.html' 7 | res=requests.get(url) 8 | 9 | while(res.status_code!=200): 10 | try: 11 | res=requests.get('url') 12 | except: 13 | pass 14 | 15 | soup=BeautifulSoup(res.text,'lxml') 16 | 17 | short=soup.find('ul',{'class':'topstr-list gap topmarging'}).find_all('a') 18 | head_lines=soup.find_all('div',{'class':'innerbox'}) 19 | return short,head_lines 20 | 21 | def top_news(short): 22 | for top in short: 23 | print(top.text,end='\n\n') 24 | 25 | def main_news(head_lines): 26 | for news in head_lines: 27 | for head in news.find_all('a'): 28 | print(head.text) 29 | for cont in news.find_all('p'): 30 | print(cont.text,end='\n\n') 31 | 32 | def top_not(title): 33 | for i in title: 34 | notify('',i.text) 35 | 36 | def main_not(head_lines): 37 | for news in head_lines: 38 | for head in news.find_all('a'): 39 | title=head.text 40 | for cont in news.find_all('p'): 41 | msg=cont.text 42 | notify(msg,title) 43 | 44 | def menu(): 45 | print() 46 | print(' -----------------------') 47 | print(' | Menu |') 48 | print(' |1. Top News |') 49 | print(' |2. Top News Notify |') 50 | print(' |3. Main News |') 51 | print(' |4. Main News Notify |') 52 | print(' -----------------------') 53 | 54 | def main(): 55 | ch='y' 56 | while(ch=='y'): 57 | menu() 58 | choice=input('\nEnter your choice:') 59 | print() 60 | short,head_lines=get_data() 61 | out={'1':'Top News','2':'Top Notify','3':'Main News','4':'Main Notify'} 62 | 63 | if choice in out: 64 | print('\t\t\t\t{}\n'.format(out[choice])) 65 | 66 | if choice=='1': 67 | top_news(short) 68 | elif choice=='2': 69 | top_not(short) 70 | elif choice=='3': 71 | main_news(head_lines) 72 | elif choice=='4': 73 | main_not(head_lines) 74 | else: 75 | print('Wrong Input') 76 | 77 | ch=input('Do you want to continue (y/n) :') 78 | print() 79 | 80 | if __name__=='__main__': 81 | main() 82 | -------------------------------------------------------------------------------- /PDF/pdf_to_text.py: -------------------------------------------------------------------------------- 1 | import PyPDF2 2 | 3 | pdf_name = 'poem.pdf' 4 | pdfObj = open(pdf_name, 'rb') 5 | pdfReader = PyPDF2.PdfFileReader(pdfObj) 6 | 7 | for pageNum in range(pdfReader.numPages): 8 | pageObj = pdfReader.getPage(pageNum) 9 | text = pageObj.extractText() 10 | print(text) 11 | 12 | -------------------------------------------------------------------------------- /Quotes/quote1.py: -------------------------------------------------------------------------------- 1 | ''' Scraped from http://quotes.toscrape.com ''' 2 | 3 | from bs4 import BeautifulSoup 4 | import requests 5 | 6 | URL = 'http://quotes.toscrape.com/page/' 7 | i = 1 8 | 9 | def get_soup(url): 10 | res = requests.get(url) 11 | soup = BeautifulSoup(res.text,'lxml') 12 | return soup 13 | 14 | def quotes(soup): 15 | global i 16 | all_quotes = soup.find_all('div',{'class':'quote'}) 17 | 18 | for quote_box in all_quotes: 19 | quote = quote_box.find('span',{'class':'text'}).text 20 | author = quote_box.find('small',{'class':'author'}).text 21 | print("{:<4} {}\n-{:>5}\n\n".format(str(i)+')',quote,author)) 22 | i +=1 23 | 24 | def main(): 25 | pages = 10 26 | for p in range(1,pages+1): 27 | soup = get_soup(URL + str(p)) 28 | quotes(soup) 29 | 30 | if __name__ == '__main__': 31 | main() 32 | -------------------------------------------------------------------------------- /Quotes/quote_of_the_day.py: -------------------------------------------------------------------------------- 1 | ''' Webscraped from https://www.brainyquote.com ''' 2 | 3 | import requests 4 | from bs4 import BeautifulSoup 5 | 6 | res = requests.get('https://www.brainyquote.com/quotes_of_the_day.html') 7 | soup = BeautifulSoup(res.text, 'lxml') 8 | 9 | quote = soup.find('img',{'class':'p-qotd'}) 10 | print(quote['alt']) 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web-Scraping 2 | Web scraping projects using python 3 | 4 | 5 | -------------------------------------------------------------------------------- /Saavn/charts.py: -------------------------------------------------------------------------------- 1 | #Get top charts from Saavn 2 | 3 | from bs4 import BeautifulSoup 4 | import requests 5 | 6 | def songs_info(res): 7 | soup = BeautifulSoup(res.text, 'lxml') 8 | data = soup.find('ol', {'class': 'content-list'}) 9 | return data 10 | 11 | def get_songs(data, limit=10): 12 | song_list = [] 13 | count = 0 14 | for i, count in zip(data.find_all('div', {'class': 'details'}), range(1, int(limit) + 1)): 15 | song = i.find('p', {'class': 'song-name'}).text 16 | album = i.find('p', {'class': 'album-name'}).text 17 | count += 1 18 | item = song 19 | if album != song: 20 | item = item + " (" + album + ")" 21 | song_list.append(item) 22 | return song_list 23 | 24 | def saavn_tops(lang): 25 | res = requests.get("https://www.saavn.com/s/featured/" + lang + "/Weekly+Top+Songs") 26 | data = songs_info(res) 27 | return get_songs(data) 28 | 29 | def hindi_chartbusters(): 30 | res = requests.get("https://www.saavn.com/s/charts/Hindi-Chartbusters/u-75xwHI4ks_?&utm_content=wap%3Ahome%3Atop_charts%3Aplay%3Aclick&utm_page=home&utm_button=top_charts") 31 | data = songs_info(res) 32 | return get_songs(data) 33 | 34 | def english_chartbusters(): 35 | res = requests.get("https://www.saavn.com/s/charts/English-Chartbusters/9J4ePDXBp8k_?utm_content=wap%3Aall_top_charts%3Atop_charts%3Aplay%3Aclick&utm_page=all_top_charts&utm_button=top_charts&") 36 | data = songs_info(res) 37 | return get_songs(data) 38 | 39 | def menu(commands): 40 | print() 41 | for i,com in enumerate(commands,1): 42 | print("{}. {}".format(i,com)) 43 | choice = int(input('\nEnter your choice (number): ')) 44 | while(choice<1 or choice>len(commands)): 45 | print('\nWrong choice entered') 46 | choice = int(input('\nEnter your choice again (number): ')) 47 | return commands[choice-1] 48 | 49 | def main(): 50 | commands =['Saavn Weekly Top','Hindi Chartbusters','English Chartbusters'] 51 | choice = menu(commands) 52 | 53 | if choice =='Saavn Weekly Top': 54 | message = '' 55 | commands =['Hindi','English'] 56 | option = menu(commands) 57 | lang = option.lower() 58 | songs = saavn_tops(lang) 59 | for i,item in enumerate(songs,1): 60 | message+= str(i)+". "+item +'\n\n' 61 | print("\n\n\t\t{} - {}".format(choice,lang.title())) 62 | print() 63 | print(message) 64 | 65 | elif choice =='Hindi Chartbusters': 66 | message = '' 67 | songs = hindi_chartbusters() 68 | for i,item in enumerate(songs,1): 69 | message+= str(i)+". "+item +'\n\n' 70 | print("\n\n\t\t{}".format(choice)) 71 | print() 72 | print(message) 73 | 74 | elif choice =='English Chartbusters': 75 | message = '' 76 | songs = english_chartbusters() 77 | for i,item in enumerate(songs,1): 78 | message+= str(i)+". "+item +'\n\n' 79 | print("\n\n\t\t{}".format(choice)) 80 | print() 81 | print(message) 82 | 83 | if __name__ == '__main__': 84 | main() 85 | -------------------------------------------------------------------------------- /Synonyms/synonyms.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | from ntfy import notify 3 | import requests 4 | 5 | 6 | def get_data(query): 7 | res=requests.get('http://www.thesaurus.com/browse/'+query+'?s=t') 8 | soup=BeautifulSoup(res.text,'lxml') 9 | des=soup.find('div',{'class':'synonym-description'}).find('strong') 10 | syn=soup.find('div',{'class':'relevancy-list'}).find_all('span',{'class':'text'}) 11 | return des,syn 12 | 13 | 14 | def notification(des,synonym,query): 15 | notify(des.text+"\n"+", ".join(synonym[:3]),query.upper()) 16 | 17 | 18 | def file_out(word,des): 19 | fa=open('synonym.txt','a') 20 | for i in word: 21 | fa.write("WORD: {} \n".format(i)) 22 | fa.write("MEANING: {}\n".format(des.text)) 23 | fa.write("SYNONYMS: {}\n\n".format(word[i])) 24 | 25 | 26 | def data(des,syn,query): 27 | word={} 28 | synonym=[] 29 | 30 | for i in syn[:10]: 31 | synonym.append(i.text) 32 | 33 | word[query]=", ".join(synonym) 34 | return synonym,word 35 | 36 | 37 | def main(): 38 | query=input('Enter word:') 39 | des,syn=get_data(query) 40 | synonym,word=data(des,syn,query) 41 | notification(des,synonym,query) 42 | file_out(word,des) 43 | 44 | 45 | if __name__=='__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /Twitter/favorites_count.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | 4 | handle = input('Enter twitter handle to check favorite/liked post of a user : ') 5 | res = requests.get('https://twitter.com/'+handle) 6 | soup = BeautifulSoup(res.text,'lxml') 7 | 8 | try: 9 | favorite_box = soup.find('li',{'class':'ProfileNav-item ProfileNav-item--favorites'}) 10 | favorite = favorite_box.find('a').find('span',{'class':'ProfileNav-value'}) 11 | print("Number of post {} liked are {}: ".format(handle,favorite.get('data-count'))) 12 | 13 | except: 14 | print('Cannot find the handle right now') 15 | -------------------------------------------------------------------------------- /Twitter/followers_count.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | 4 | handle = input('Enter twitter handle to check followers of a user : ') 5 | res = requests.get('https://twitter.com/'+handle) 6 | soup = BeautifulSoup(res.text,'lxml') 7 | 8 | try: 9 | follow_box = soup.find('li',{'class':'ProfileNav-item ProfileNav-item--followers'}) 10 | followers = follow_box.find('a').find('span',{'class':'ProfileNav-value'}) 11 | print("Number of people following {} are {}: ".format(handle,followers.get('data-count'))) 12 | 13 | except: 14 | print('Cannot find the handle right now') 15 | -------------------------------------------------------------------------------- /Twitter/following_count.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | 4 | handle = input('Enter twitter handle : ') 5 | res = requests.get('https://twitter.com/'+handle) 6 | soup = BeautifulSoup(res.text,'lxml') 7 | 8 | try: 9 | following_box = soup.find('li',{'class':'ProfileNav-item ProfileNav-item--following'}) 10 | following = following_box.find('a').find('span',{'class':'ProfileNav-value'}) 11 | print("Number of people {} is following are {}: ".format(handle,following.get('data-count'))) 12 | 13 | except: 14 | print('Cannot find the handle right now') 15 | 16 | -------------------------------------------------------------------------------- /Twitter/tweet_count.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | 4 | handle = input('Enter twitter handle : ') 5 | res = requests.get('https://twitter.com/'+handle) 6 | soup = BeautifulSoup(res.text,'lxml') 7 | 8 | try: 9 | tweet_box = soup.find('li',{'class':'ProfileNav-item ProfileNav-item--tweets is-active'}) 10 | tweets= tweet_box.find('a').find('span',{'class':'ProfileNav-value'}) 11 | print("Number of tweets of {} are {}: ".format(handle,tweets.get('data-count'))) 12 | 13 | except: 14 | print('Cannot find the handle right now') 15 | -------------------------------------------------------------------------------- /Twitter/tweet_scrape.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import requests 3 | 4 | username = input('Enter username to scrape : ') 5 | count = int(input('Number of tweets to scrape : ')) 6 | 7 | res=requests.get('https://twitter.com/'+ username) 8 | soup=BeautifulSoup(res.content,'lxml') 9 | 10 | all_tweets = soup.find_all('div',{'class':'tweet'}) 11 | 12 | for tweet in all_tweets[:count]: 13 | 14 | context = tweet.find('div',{'class':'context'}).text.replace("\n"," ").strip() 15 | content = tweet.find('div',{'class':'content'}) 16 | 17 | header = content.find('div',{'class':'stream-item-header'}) 18 | user = header.find('a',{'class':'account-group js-account-group js-action-profile js-user-profile-link js-nav'}).text.replace("\n"," ").strip() 19 | time = header.find('a',{'class':'tweet-timestamp js-permalink js-nav js-tooltip'}).find('span').text.replace("\n"," ").strip() 20 | 21 | message = content.find('div',{'class':'js-tweet-text-container'}).text.replace("\n"," ").strip() 22 | 23 | footer = content.find('div',{'class':'stream-item-footer'}) 24 | stat = footer.find('div',{'class':'ProfileTweet-actionCountList u-hiddenVisually'}).text.replace("\n"," ").strip() 25 | 26 | if context: 27 | print(context) 28 | print(user,time) 29 | print(message) 30 | print(stat) 31 | print() 32 | -------------------------------------------------------------------------------- /Wallpaper/bing.py: -------------------------------------------------------------------------------- 1 | from urllib import request 2 | from bs4 import BeautifulSoup 3 | from datetime import datetime as dt 4 | import requests,os 5 | 6 | 7 | def get_image(): 8 | res=requests.get('https://bingwallpaper.com/') 9 | soup=BeautifulSoup(res.text,"lxml") 10 | image=soup.find('a',{'class':'cursor_zoom'}).find('img') 11 | link=image.get('src') 12 | return link 13 | 14 | def download(): 15 | link=get_image() 16 | file_name = dt.now().strftime("%Y-%m-%d") 17 | user = os.getenv('USER') 18 | path='/home/'+user+'/Pictures/BingWallpapers' 19 | full_path=os.path.join(path,file_name) 20 | 21 | if not os.path.exists(path): 22 | os.mkdir(path) 23 | 24 | with open(full_path, 'wb') as f: 25 | f.write(request.urlopen(link).read()) 26 | return full_path 27 | 28 | def change_wall(): 29 | full_path=download() 30 | os.system("/usr/bin/gsettings set org.gnome.desktop.background picture-uri file:///"+full_path) 31 | 32 | 33 | change_wall() 34 | -------------------------------------------------------------------------------- /Wallpaper/demo.md: -------------------------------------------------------------------------------- 1 | ![wallpaper](https://user-images.githubusercontent.com/22851705/28993355-ef5a2120-79a3-11e7-9b25-758d22907be4.gif) 2 | -------------------------------------------------------------------------------- /Weather/weather.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from pprint import pprint 3 | 4 | def location(): 5 | res=requests.get('http://ipinfo.io/') 6 | data=res.json() 7 | loc=[]; 8 | loc=data['loc'].split(',') 9 | lat=loc[0] 10 | lon=loc[1] 11 | return lat,lon,data['city'] 12 | 13 | def location_coordinates(): 14 | lat,lon,city=location() 15 | print("Latitude: {}\nLongitude: {}\nCity: {}".format(lat,lon,city)) 16 | 17 | def weather_data(query): 18 | res=requests.get('http://api.openweathermap.org/data/2.5/weather?'+query+'&appid=ac7c75b9937a495021393024d0a90c44&units=metric'); 19 | return res.json(); 20 | 21 | def print_temp(result,city): 22 | print("{}'s temperature : {}°C ".format(city,result['main']['temp'])) 23 | print("Wind speed:{} m/s".format(result['wind']['speed'])) 24 | print("Weather:{}".format(result['weather'][0]['main'])) 25 | print("Description:{}".format(result['weather'][0]['description'])) 26 | 27 | 28 | def current_temperature(): 29 | lat,lon,city=location(); 30 | query='lat='+lat+'&lon='+lon; 31 | data=weather_data(query); 32 | print_temp(data,city); 33 | 34 | def temp_by_city(): 35 | city=input('Enter the city:') 36 | print() 37 | query='q='+city; 38 | data=weather_data(query); 39 | print_temp(data,city) 40 | 41 | def menu(): 42 | print() 43 | print('1. Location coordinates') 44 | print('2. Current Temperature') 45 | print('3. Temperature by city') 46 | choice=input('\nEnter your choice:') 47 | while (choice not in ['1','2','3']): 48 | print('Wrong choice') 49 | choice=input('\nEnter your choice:') 50 | print() 51 | 52 | return choice; 53 | 54 | def call(ch): 55 | if ch=='1': 56 | location_coordinates() 57 | if ch=='2': 58 | current_temperature() 59 | if ch=='3': 60 | temp_by_city() 61 | 62 | def main(): 63 | choice=menu() 64 | call(choice) 65 | print() 66 | 67 | if __name__=='__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /Whatsapp/message.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | 3 | driver = webdriver.Chrome() 4 | driver.get('http://web.whatsapp.com') 5 | 6 | name = input('Enter the name of user or group : ') 7 | msg = input('Enter the message : ') 8 | count = int(input('Enter the count : ')) 9 | 10 | #Scan the code before proceeding further 11 | input('Enter anything after scanning QR code') 12 | 13 | user = driver.find_element_by_xpath('//span[@title = "{}"]'.format(name)) 14 | user.click() 15 | 16 | msg_box = driver.find_element_by_class_name('input-container') 17 | 18 | for i in range(count): 19 | msg_box.send_keys(msg) 20 | driver.find_element_by_class_name('compose-btn-send').click() 21 | -------------------------------------------------------------------------------- /Whatsapp/updated.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | 3 | driver = webdriver.Chrome() 4 | driver.get('https://web.whatsapp.com/') 5 | 6 | name = input('Enter the name of user or group : ') 7 | msg = input('Enter your message : ') 8 | count = int(input('Enter the count : ')) 9 | 10 | input('Enter anything after scanning QR code') 11 | 12 | user = driver.find_element_by_xpath('//span[@title = "{}"]'.format(name)) 13 | user.click() 14 | 15 | msg_box = driver.find_element_by_class_name('_2S1VP') 16 | 17 | for i in range(count): 18 | msg_box.send_keys(msg) 19 | button = driver.find_element_by_class_name('_2lkdt') 20 | button.click() 21 | -------------------------------------------------------------------------------- /Wordcloud/twitter_mask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ivan-0224/Python-scraping/9f7b0c2cb1649ae8dcd4f467577c1e5038719c57/Wordcloud/twitter_mask.png -------------------------------------------------------------------------------- /Wordcloud/wordcloud.py: -------------------------------------------------------------------------------- 1 | from wordcloud import WordCloud 2 | import matplotlib.pyplot as plt 3 | 4 | text = "keys coding competetive development Python webscraping automation getsetpython projects cool awesome blockchain data machinelearning regex loops datascience opencv ML list dictionary analysis sentiment variable set counter enumerate try except if-else for function class object instance GUI encryption security" 5 | 6 | cloud = WordCloud(background_color="white").generate(text) 7 | 8 | plt.imshow(cloud) 9 | plt.a 10 | xis('off') 11 | plt.show() 12 | -------------------------------------------------------------------------------- /Wordcloud/wordcloud2.py: -------------------------------------------------------------------------------- 1 | from wordcloud import WordCloud 2 | import matplotlib.pyplot as plt 3 | 4 | text = input('Enter your string : ') 5 | 6 | cloud = WordCloud(background_color="white").generate(text) 7 | 8 | plt.imshow(cloud) 9 | plt.axis('off') 10 | plt.show() 11 | -------------------------------------------------------------------------------- /Wordcloud/wordcloud3.py: -------------------------------------------------------------------------------- 1 | from wordcloud import WordCloud 2 | import matplotlib.pyplot as plt 3 | from PIL import Image 4 | import numpy as np 5 | 6 | text = "keys coding competetive development Python webscraping automation getsetpython projects cool awesome blockchain data machinelearning regex loops datascience opencv ML list dictionary analysis sentiment variable set counter enumerate try except if-else for function class object instance GUI encryption security" 7 | 8 | our_mask = np.array(Image.open('twitter_mask.png')) 9 | 10 | cloud = WordCloud(background_color="white", mask = our_mask).generate(text) 11 | 12 | plt.imshow(cloud) 13 | plt.axis('off') 14 | plt.show() 15 | -------------------------------------------------------------------------------- /Youtube Downloader/saavn.py: -------------------------------------------------------------------------------- 1 | #download saavn songs and video songs 2 | 3 | from subprocess import call 4 | from bs4 import BeautifulSoup 5 | import requests 6 | from colorclass import Color 7 | 8 | 9 | def songs_info(res): 10 | 11 | soup = BeautifulSoup(res.text, 'lxml') 12 | data = soup.find('ol', {'class': 'content-list'}) 13 | return data 14 | 15 | 16 | def song_print(data, limit='20'): 17 | song_list = [] 18 | 19 | for i, count in zip(data.find_all('div', {'class': 'details'}), range(1, int(limit) + 1)): 20 | song = i.find('p', {'class': 'song-name'}).text 21 | album = i.find('p', {'class': 'album-name'}).text 22 | 23 | print("{:>2}.".format(count), end='') 24 | count += 1 25 | print(song, end='') 26 | if song != album: 27 | print(" (" + album + ")") 28 | else: 29 | print() 30 | item = song 31 | if album != song: 32 | item = item + " (" + album + ")" 33 | song_list.append(item) 34 | 35 | return song_list 36 | 37 | 38 | 39 | def download_menu(): 40 | print('1.Song') 41 | print('2.Video Song') 42 | print() 43 | choice = input('Enter your choice:') 44 | print() 45 | while choice not in ['1', '2']: 46 | print(Color('{autored}Wrong Choice{/autored}')) 47 | choice = input('\nEnter choice: ') 48 | print() 49 | return choice 50 | 51 | 52 | def subtitles(): 53 | subtitles_list = ['hi', 'en'] 54 | lang = input('Enter subtitles language ( hindi -> hi , english -> en ): ') 55 | while lang not in subtitles_list: 56 | print('\n Wrong choice:') 57 | lang = input('Enter subtitles language ( hindi -> hi , english -> en ): ') 58 | return ["--write-sub", "--sub-lang", lang] 59 | 60 | 61 | def song_download_by_name(name, sub, sub_data, url=''): 62 | extras = ["--embed-thumbnail"] 63 | search = ["ytsearch:" + name] 64 | 65 | if url != '': 66 | search = [url] 67 | 68 | if sub == 'y' or sub == 'Y': 69 | extras.extend(sub_data) 70 | 71 | print("\nDownloading {}\n".format(name)) 72 | 73 | command = ['youtube-dl', '-o', '/home/umang/Music/%(title)s.%(ext)s+'] 74 | audio_format = "-x --audio-format mp3".split() 75 | search.extend(audio_format) 76 | command.extend(search) 77 | command.extend(extras) 78 | call(command, shell=False) 79 | 80 | 81 | def video_song_download_by_name(name, sub, sub_data, url=''): 82 | extras = ['--embed-subs'] 83 | search = ["ytsearch:" + name] 84 | 85 | if url != '': 86 | search = [url] 87 | 88 | if sub == 'y' or sub == 'Y': 89 | extras.extend(sub_data) 90 | 91 | print("\nDownloading {}\n".format(name)) 92 | 93 | command = ['youtube-dl', '-o', '/home/umang/Videos/%(title)s.%(ext)s+'] 94 | command.extend(search) 95 | command.extend(extras) 96 | call(command, shell=False) 97 | 98 | 99 | def mix_song_download_by_name(name, sub, sub_data, url=''): 100 | print("\nSelect options for {}".format(name)) 101 | print() 102 | choice = download_menu() 103 | 104 | if choice == '1': 105 | song_download_by_name(name, sub, sub_data, url) 106 | 107 | if choice == '2': 108 | video_song_download_by_name(name, sub, sub_data, url) 109 | 110 | 111 | def menu(): 112 | print() 113 | print('1. Display Saavn Weekly Top') 114 | print('2. Display Hindi Chartbusters') 115 | print('3. Display English Chartbusters') 116 | print('4. Download Weekly Top Mix (video/song )') 117 | print('5. Download Weekly Top Songs') 118 | print('6. Download Weekly Top Video Songs') 119 | print('7. Download by Search') 120 | print('8. Download by Ranking in table') 121 | print('9. Download by URL', end='\n\n') 122 | choice = input('Enter choice: ') 123 | print() 124 | while choice not in ['1', '2', '3', '4', '5', '6', '7','8','9']: 125 | print(Color('{autored}Wrong Choice{/autored}')) 126 | choice = input('\nEnter choice: ') 127 | print() 128 | 129 | return choice 130 | 131 | 132 | def song_list_menu(): 133 | print('Select from the above lists\n') 134 | print('1. Saavn Weekly Top') 135 | print('2. Hindi Chartbusters') 136 | print('3. English Chartbusters') 137 | print() 138 | choice = input('Enter choice: ') 139 | print() 140 | while choice not in ['1', '2', '3']: 141 | print(Color('{autored}Wrong Choice{/autored}')) 142 | choice = input('\nEnter choice: ') 143 | print() 144 | 145 | return choice 146 | 147 | 148 | def song_list_options(choice): 149 | 150 | if choice == '1': 151 | lang = input("Enter songs' lanuguage (hindi/english) : ") 152 | print() 153 | res = requests.get("https://www.saavn.com/s/featured/" + lang + "/Weekly+Top+Songs") 154 | 155 | if choice == '2': 156 | res = requests.get("https://www.saavn.com/s/charts/Hindi-Chartbusters/u-75xwHI4ks_?&utm_content=wap%3Ahome%3Atop_charts%3Aplay%3Aclick&utm_page=home&utm_button=top_charts") 157 | 158 | if choice == '3': 159 | res = requests.get("https://www.saavn.com/s/charts/English-Chartbusters/9J4ePDXBp8k_?utm_content=wap%3Aall_top_charts%3Atop_charts%3Aplay%3Aclick&utm_page=all_top_charts&utm_button=top_charts&") 160 | 161 | return res 162 | 163 | 164 | def main(): 165 | choice = menu() 166 | 167 | if choice == '1': 168 | limit = input('Enter the number of songs: ') 169 | print() 170 | res=song_list_options(choice) 171 | song_print(songs_info(res), limit) 172 | 173 | if choice == '2': 174 | limit = input('Enter the number of songs: ') 175 | print() 176 | res=song_list_options(choice) 177 | song_print(songs_info(res), limit) 178 | 179 | if choice == '3': 180 | limit = input('Enter the number of songs: ') 181 | print() 182 | res=song_list_options(choice) 183 | song_print(songs_info(res), limit) 184 | 185 | 186 | if choice in ['4', '5', '6']: 187 | ch=song_list_menu() 188 | res=song_list_options(ch) 189 | limit = input('Enter the number of songs or videos: ') 190 | print() 191 | song_list = song_print(songs_info(res), limit) 192 | print() 193 | sub = input('Want to download external subtitles file (y/n): ') 194 | sub_data = [] 195 | if sub == 'y' or sub == 'Y': 196 | sub_data = subtitles() 197 | print() 198 | for song in song_list: 199 | 200 | if choice == '4': 201 | mix_song_download_by_name(song, sub, sub_data) 202 | if choice == '5': 203 | song_download_by_name(song, sub, sub_data) 204 | if choice == '6': 205 | video_song_download_by_name(song, sub, sub_data) 206 | print() 207 | 208 | if choice == '7': 209 | name = input('Enter the name of song or videos: ') 210 | print() 211 | sub = input('Want to download external subtitles file (y/n): ') 212 | sub_data = [] 213 | if sub == 'y' or sub == 'Y': 214 | sub_data = subtitles() 215 | mix_song_download_by_name(name, sub, sub_data) 216 | 217 | if choice == '8': 218 | ch=song_list_menu() 219 | res=song_list_options(ch) 220 | song_list = song_print(songs_info(res)) 221 | print() 222 | rank = input('Enter the number from the table: ') 223 | print() 224 | sub = input('Want to download external subtitles file (y/n): ') 225 | sub_data = [] 226 | if sub == 'y' or sub == 'Y': 227 | sub_data = subtitles() 228 | mix_song_download_by_name(song_list[int(rank) - 1], sub, sub_data) 229 | 230 | if choice == '9': 231 | song_url = input('Enter the url: ') 232 | name = '' 233 | print() 234 | sub = input('Want to download external subtitles file (y/n): ') 235 | sub_data = [] 236 | if sub == 'y' or sub == 'Y': 237 | sub_data = subtitles() 238 | mix_song_download_by_name(name, sub, sub_data, song_url) 239 | print() 240 | 241 | if __name__ == '__main__': 242 | main() 243 | 244 | 245 | --------------------------------------------------------------------------------