2 |
3 | |
4 | 1
5 | |
6 | Praise Him, praise Him, all ye little children,
God is love, God is love;
Praise Him, praise Him, all ye little children,
God is love, God is love. |
7 |
8 |
9 | |
10 | 2
11 | |
12 | Love Him, love Him, all ye little children,
God is love, God is love;
Love Him, love Him, all ye little children,
God is love, God is love. |
13 |
14 |
15 | |
16 | 3
17 | |
18 | Thank Him, thank Him, all ye little children,
God is love, God is love;
Thank Him, thank Him, all ye little children,
God is love, God is love. |
19 |
20 |
--------------------------------------------------------------------------------
/5-track-pianoroll/compile.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 |
3 | print('[*] loading...')
4 | proc_list = np.load('segments.npy')
5 | print('[*] processing...')
6 | num_item = len(proc_list)
7 | print(num_item)
8 | compiled_list = []
9 | for lidx in range(num_item):
10 | multi_track = proc_list[lidx]
11 | pianorolls = []
12 |
13 | for tracks in multi_track.tracks:
14 | pianorolls.append(tracks.pianoroll[:, :, np.newaxis])
15 |
16 | pianoroll_compiled = np.reshape(np.concatenate(pianorolls, axis=2)[:, 24:108, :], (8, 48, 84, 5))
17 | pianoroll_compiled = pianoroll_compiled[np.newaxis, :] > 0
18 | compiled_list.append(pianoroll_compiled.astype(bool))
19 |
20 | final = np.concatenate(compiled_list, axis=0)
21 | print(final.shape)
22 | print('[*] saving...')
23 | np.save('x_lpd_5_phr.npy', final)
24 | print('Done!!')
--------------------------------------------------------------------------------
/theorytab/utils/youtube_crawler.py:
--------------------------------------------------------------------------------
1 | import pafy
2 | import ffmpy
3 | from time import gmtime, strftime
4 | import json
5 | import os
6 |
7 | def video_crawler(y_id, filepath=''):
8 |
9 | try:
10 | video = pafy.new(y_id)
11 |
12 | video_info = dict()
13 | video_info['YouTubeID'] = y_id
14 | video_info['title'] = video.title
15 | video_info['rating'] = video.rating
16 | video_info['viewcount'] = video.viewcount
17 | video_info['author'] = video.author
18 | video_info['length'] = video.length
19 | video_info['duration'] = video.duration
20 | video_info['likes'] = video.likes
21 | video_info['dislikes'] = video.dislikes
22 | video_info['crawl_time'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
23 | video_info['description'] = video.description
24 |
25 | with open(os.path.join(filepath,'video_info.json'), 'w') as f:
26 | json.dump(video_info, f)
27 |
28 | best = video.getbest(preftype="mp4")
29 | best.download(quiet=True, filepath=os.path.join(filepath,'video.mp4'))
30 |
31 | return True
32 | except:
33 | print('Download Failed: %s ' % y_id)
34 | return False
35 |
36 |
37 |
38 | if __name__ == '__main__':
39 | y_id = 'n1BtExxkP0M'
40 | video_crawler(y_id, '')
41 | video_crawler(y_id, '')
--------------------------------------------------------------------------------
/5-track-pianoroll/readme.md:
--------------------------------------------------------------------------------
1 | # 5-track pianoroll dataset
2 |
3 | This repository contains pre-processing codes and processed datasets of [LPD](https://github.com/salu133445/lakh-pianoroll-dataset) dataset.
4 |
5 | ## Source Codes for Pre-processing
6 |
7 | 1. Download 'lpd_cleansed' from [here](https://github.com/salu133445/lakh-pianoroll-dataset)
8 |
9 | 2. run *parse.py*
10 | 3. run *compile.py*
11 |
12 | You can change the setting to customize your own piano-roll dataset.
13 |
14 | ## Processed 5-track Piano-roll Datasets
15 |
16 | #### Latest Versoin (ver.2)
17 |
18 | * num of tracks: **5**
19 | * *Bass, Drum, Guitar, String and Piano*
20 | * time resolution: **48**
21 | * pitch range: **84**
22 | * num of bar: **8**
23 | * num of phrases: **34126**
24 | * the shape of the tensor is **34126 x 8 x 48 x 84 x 5**
25 | * 5.12 GB
26 | * select segments with higher qulity. One instrument for one track.
27 | * [Download](https://drive.google.com/file/d/17FBw7c_vrK33_mEgsA919GTSlHoJ7M6T/view?usp=sharing)
28 |
29 | #### Old Version for MuseGAN (ver.1)
30 |
31 | * num of tracks: **5**
32 | * *Bass, Drum, Guitar, String and Piano*
33 | * time resolution: **96**
34 | * pitch range: **84**
35 | * num of bar: **4**
36 | * num of phrases: **50266**
37 | * the shape of the tensor is **50266 x 384 x 84 x 5**
38 | * 7.54 GB
39 | * Compress instruments in the same midi family into one track. See [here](https://github.com/salu133445/musegan/tree/master/v1/training)
40 | * [Download](https://drive.google.com/file/d/1yj-5CsAwSoj1LHk4QwEQ09VB5fS69Vnq/view?usp=sharing)
41 |
42 | Generally, version 2 has richer but clear textures.
43 |
44 | --------------
45 | Sample image of 5-track Piano-roll Datasets (ver.2):
46 |
47 | 
48 |
49 | The generated samples of version 2 on MuseGAN is available [here](sample).
50 |
--------------------------------------------------------------------------------
/hymnal/utils/log.txt:
--------------------------------------------------------------------------------
1 | =================================================
2 | https://www.hymnal.net/en/song-index/h/A
3 | https://www.hymnal.net/en/song-index/h/B
4 | https://www.hymnal.net/en/song-index/h/C
5 | https://www.hymnal.net/en/song-index/h/D
6 | https://www.hymnal.net/en/song-index/h/E
7 | https://www.hymnal.net/en/song-index/h/F
8 | https://www.hymnal.net/en/song-index/h/G
9 | https://www.hymnal.net/en/song-index/h/H
10 | https://www.hymnal.net/en/song-index/h/I
11 | https://www.hymnal.net/en/song-index/h/J
12 | https://www.hymnal.net/en/song-index/h/K
13 | https://www.hymnal.net/en/song-index/h/L
14 | https://www.hymnal.net/en/song-index/h/M
15 | https://www.hymnal.net/en/song-index/h/N
16 | https://www.hymnal.net/en/song-index/h/O
17 | https://www.hymnal.net/en/song-index/h/P
18 | https://www.hymnal.net/en/song-index/h/R
19 | https://www.hymnal.net/en/song-index/h/S
20 | https://www.hymnal.net/en/song-index/h/T
21 | https://www.hymnal.net/en/song-index/h/U
22 | https://www.hymnal.net/en/song-index/h/V
23 | https://www.hymnal.net/en/song-index/h/W
24 | https://www.hymnal.net/en/song-index/h/Y
25 | https://www.hymnal.net/en/song-index/nt
26 | https://www.hymnal.net/en/song-index/ns/1
27 | https://www.hymnal.net/en/song-index/ns/2
28 | https://www.hymnal.net/en/song-index/ns/A
29 | https://www.hymnal.net/en/song-index/ns/B
30 | https://www.hymnal.net/en/song-index/ns/C
31 | https://www.hymnal.net/en/song-index/ns/D
32 | https://www.hymnal.net/en/song-index/ns/E
33 | https://www.hymnal.net/en/song-index/ns/F
34 | https://www.hymnal.net/en/song-index/ns/G
35 | https://www.hymnal.net/en/song-index/ns/H
36 | https://www.hymnal.net/en/song-index/ns/I
37 | https://www.hymnal.net/en/song-index/ns/J
38 | https://www.hymnal.net/en/song-index/ns/K
39 | https://www.hymnal.net/en/song-index/ns/L
40 | https://www.hymnal.net/en/song-index/ns/M
41 | https://www.hymnal.net/en/song-index/ns/N
42 | https://www.hymnal.net/en/song-index/ns/O
43 | https://www.hymnal.net/en/song-index/ns/P
44 | https://www.hymnal.net/en/song-index/ns/R
45 | https://www.hymnal.net/en/song-index/ns/S
46 | https://www.hymnal.net/en/song-index/ns/T
47 | https://www.hymnal.net/en/song-index/ns/U
48 | https://www.hymnal.net/en/song-index/ns/V
49 | https://www.hymnal.net/en/song-index/ns/W
50 | https://www.hymnal.net/en/song-index/ns/Y
51 | https://www.hymnal.net/en/song-index/ns/Z
52 | https://www.hymnal.net/en/song-index/c
53 | > classic
54 | (1/1950) https://www.hymnal.net/en/hymn/h/877
55 | (2/1950) https://www.hymnal.net/en/hymn/h/292
56 | (3/1950) https://www.hymnal.net/en/hymn/h/509
57 | (4/1950) https://www.hymnal.net/en/hymn/h/952
58 | (5/1950) https://www.hymnal.net/en/hymn/h/724
59 | (6/1950) https://www.hymnal.net/en/hymn/h/886
60 | (7/1950) https://www.hymnal.net/en/hymn/h/1198
61 |
--------------------------------------------------------------------------------
/doug-mckenzie-jazz/codes/DMjazzCrawler.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from bs4 import BeautifulSoup
3 | import os
4 | import sys
5 | import time
6 | import json
7 | import re
8 |
9 | class DMjazzCrawler():
10 | BASE_URL = 'http://www.bushgrafts.com/jazz'
11 | ROOT = 'archive'
12 |
13 | def __init__(self, sleep_time=0.1, log=True):
14 | self.sleep_time = sleep_time
15 | self.log = log
16 |
17 | def _request_url(self, url, doctype='html'):
18 | # set header
19 | response = requests.get(url, headers = {"Cache-Control":"max-age=0"})
20 |
21 | # sleep
22 | time.sleep(self.sleep_time)
23 |
24 | # return
25 | if doctype =='html':
26 | soup = BeautifulSoup(response.text, 'html.parser')
27 | return soup
28 | elif doctype =='content':
29 | return response.content
30 | else:
31 | return response
32 |
33 | def _log_print(self, log, quite=False):
34 | if not quite:
35 | print(log)
36 |
37 | if self.log:
38 | with open("log.txt", "a") as f:
39 | print(log, file=f)
40 |
41 | def fetch_song(self):
42 | self.soup = self._request_url(self.BASE_URL+'/midi.htm')
43 | a_list = dmc.soup.find_all('a')
44 | midi_list = []
45 | name_list = []
46 |
47 | cnt = 0
48 | for idx, a in enumerate(a_list):
49 | str_ = a.get('href')
50 | if str_ and (str_ not in midi_list) and ('.mid' in str_):
51 | song_name = re.sub( '\s+', ' ' , a.text.replace( '\r\n' , '' )).strip(' ')
52 | if song_name:
53 | midi_fn = str_.split('/')[1]
54 | midi_list.append(midi_fn )
55 | name_list.append(song_name)
56 | print('%3d | %-40s %s'%(idx, song_name, midi_fn))
57 | cnt += 1
58 |
59 | self._log_print('Total: %d'%cnt)
60 |
61 | return dict(zip(midi_list, name_list))
62 |
63 | def crawl_song(self, song_dict):
64 | for idx, k in enumerate(song_dict.keys()):
65 |
66 | url = self.BASE_URL + '/Midi%20site/' + k
67 | print('%3d %s' %(idx, url))
68 | content = self._request_url(url, doctype='content')
69 |
70 | with open(os.path.join(self.ROOT,k), "wb") as f:
71 | f.write(content)
72 |
73 | def run(self):
74 |
75 | song_dict = self.fetch_song()
76 |
77 | if not os.path.exists(self.ROOT):
78 | os.makedirs(self.ROOT)
79 | with open(os.path.join(self.ROOT, 'archive.json'), "w") as f:
80 | json.dump(song_dict, f)
81 |
82 | self.crawl_song(song_dict)
83 |
84 |
85 | if __name__ == '__main__':
86 | dmc = DMjazzCrawler()
87 | dmc.run()
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # List of Symbolic Musical Datasets
2 |
3 | This repository aims to collect accessible symolic musical datasets on the Net.
4 | Generally, each dataset is organized as the following way:
5 | * archive: samples from the dataset
6 | * utils: codes for crawling or processing
7 |
8 | ## Contents
9 | * Piano-roll
10 | * Lead Sheet
11 | * MIDI
12 | * MISC
13 |
14 | ---
15 |
16 | ## Piano Roll
17 | ### 5 track piano-roll dataset
18 | 
19 |
20 | This dataset is derived from [LPD](https://github.com/salu133445/lakh-pianoroll-dataset) with new pre-processing policy.
21 |
22 | ### lead sheet dataset
23 | 
24 |
25 | This dataset is derived from [Theorytab]. However, it also has potentials to incoperate with other lead sheet datasets. For further understanding, please refer to this [repo](https://github.com/wayne391/Lead-Sheet-Analysis/tree/master/lead_sheet_dataset).
26 |
27 | ---
28 |
29 | ## Lead Sheets
30 | one melody track accompanied with one chord track
31 |
32 | ### Crawled Datasets
33 | | Source | Genre | Format | Chord | Melody | Songs | Src |
34 | |-----------------------|:----------:|:------:|:-----:|:------:|:------:|:----:|
35 | | [Theorytab] | pop | XML | V | V | 10148 | [O](https://drive.google.com/file/d/13AEVD9xaZIaicEgd8tF1l6aOiRTymJxL/view?usp=sharing)
36 | | [Wikifonia] | pop | XML | V | V | 6675 | [O](https://drive.google.com/file/d/155FZ9Uq7QLySv9y2bAtk5LD37XZDo0DF/view?usp=sharing)
37 | | [Hymnal] | hymn | MIDI | Δ | V | 3358 | [O](https://drive.google.com/drive/folders/1fP9OmQa9amz-nwaaaITggCEWs3ewz1_8?usp=sharing)
38 |
39 |
40 | #### Links
41 |
42 | * WJazzD: http://jazzomat.hfm-weimar.de/dbformat/dboverview.html
43 | * MIDI format of Theorytab is now available: [Link](https://drive.google.com/file/d/1K1t8L9IRTHnQ1ozRIMRGEyxk_yhN6kLr/view?usp=sharing).
44 | --------------
45 |
46 | ## Midi
47 | ### Crawled Datasets
48 | | Source | Genre | Multi-track | Format |Songs | src |
49 | |-----------------------|:----------:|:-----------:|:------:|:------:|:---:|
50 | | [VGMdb] | game | V | MIDI | 28419 | [O](https://drive.google.com/drive/folders/1IW83MmH-RJ81yog6sbOUOTHimobE4FuK?usp=sharing)
51 | | [Doug McKenzie Jazz] | jazz | V | MIDI | 297 | [O](https://drive.google.com/drive/folders/1wVVDpcov5VV6Govhn1-CT0BOifqoF-Od?usp=sharing)
52 | | [Piano-e-Competition] | classical | | MIDI | 1573 | [O](https://drive.google.com/drive/folders/17yAGt3AR6txSZv8DBcbAbT3luTMkrkIb?usp=sharing)
53 |
54 | ### Online Resources
55 | #### Jazz
56 | * [profesordepiano](http://www.profesordepiano.com/Real%20Book/Realbook.htm?fbclid=IwAR09XcuMD6PMEyUFq0gXAIVFsJVPw8uQSXq5s-o46JFv7OlYVQnwArFOmSk)
57 | * [minor9](http://bhs.minor9.com)
58 |
59 | #### Drum
60 | * [Groove MIDI Dataset (Magenta)](https://magenta.tensorflow.org/datasets/groove)
61 |
62 | ### MIDI MAN (on reddit)
63 | * [Midi Man](https://www.reddit.com/r/WeAreTheMusicMakers/comments/3anwu8/the_drum_percussion_midi_archive_800k/)
64 | https://www.reddit.com/r/WeAreTheMusicMakers/comments/3ajwe4/the_largest_midi_collection_on_the_internet/
65 |
66 | #### full-scale
67 | * [midiworld](http://www.midiworld.com)
68 | * [Lakh MIDI dataset](http://colinraffel.com/projects/lmd/)
69 |
70 |
71 | ---
72 |
73 | ## MISC
74 | ### Unchecked
75 | * http://www.musicstudents.com/jam.html (backing track and chord charts)
76 | * https://www.cs.hmc.edu/~keller/jazz/
77 | * http://www.ralphpatt.com/Song.html
78 | * http://www.saxuet.qc.ca/TheSaxyPage/midi.htm
79 | * http://www.thejazzpage.de/index1.html
80 | * http://cjam.lassecollin.se/
81 | * http://www.jazzpla.net/jazznote3000.htm
82 |
83 |
84 | [Theorytab]: https://www.hooktheory.com/theorytab
85 | [Hymnal]: https://www.hymnal.net/en/home
86 | [Wikifonia]: http://www.wikifonia.org/
87 | [Piano-e-Competition]: http://www.piano-e-competition.com
88 | [VGMdb]: https://www.vgmusic.com
89 | [Doug McKenzie Jazz]: http://bushgrafts.com/wp/
90 |
--------------------------------------------------------------------------------
/vgmdb/utils/VGMCrawler.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from bs4 import BeautifulSoup
3 | import os
4 | import sys
5 | import time
6 | import json
7 |
8 | class VGMCrawler():
9 | BASE_URL = 'https://www.vgmusic.com/music/console/'
10 | archive_dir = 'archive'
11 |
12 | def __init__(self, sleep_time=0.1, log=True):
13 | self.sleep_time = sleep_time
14 | self.log = log
15 | self.count = 0
16 |
17 | def _request_url(self, url, doctype='html'):
18 | # set header
19 | response = requests.get(url)
20 |
21 | # sleep
22 | time.sleep(self.sleep_time)
23 |
24 | # return
25 | if doctype =='html':
26 | soup = BeautifulSoup(response.text, 'html.parser')
27 | return soup
28 | elif doctype =='content':
29 | return response.content
30 | else:
31 | return response
32 |
33 | def _log_print(self, log, quite=False):
34 | if not quite:
35 | print(log)
36 |
37 | if self.log:
38 | with open("log.txt", "a") as f:
39 | print(log, file=f)
40 |
41 | def fetch_dirs(self, url):
42 | soup = self._request_url(url)
43 | tr_list = soup.find('table').find_all('tr')
44 | dir_list = []
45 |
46 | for i in range(3, len(tr_list)-1):
47 | dir_list.append(tr_list[i].find_all('td')[1].text)
48 | return dir_list
49 |
50 | def fetch_songs(self, url):
51 | soup = self._request_url(url)
52 | tr_list = soup.find('table').find_all('tr')
53 | song_list = []
54 |
55 | if len(tr_list) == 4:
56 | return None
57 | for i in range(2, len(tr_list)):
58 | now_tr = tr_list[i]
59 | if now_tr.get("class") == ['header']:
60 | album_name = now_tr.text.strip('\n')
61 | else:
62 | if not now_tr.td.get('colspan'):
63 |
64 | # get info
65 | info = now_tr.td.text.split('\n')
66 | song_midi = now_tr.a['href']
67 | song_name = info[0]
68 | song_size = info[1]
69 | song_author = info[3]
70 | song_list.append({
71 | 'filename':song_midi,
72 | 'song_name':song_name,
73 | 'song_size':song_size,
74 | 'song_author':song_author,
75 | 'album_name':album_name})
76 | print(' |%30s |%30s |%13s |%10s |%s '%(album_name, song_name, song_size, song_author, song_midi ))
77 |
78 |
79 | return song_list
80 |
81 | def crawl_songs(self, url, song_list, dir_path):
82 | if not os.path.exists(dir_path):
83 | os.makedirs(dir_path)
84 |
85 | for idx, s in enumerate(song_list):
86 | sys.stdout.write('%d/%d - total: %d\n' % (idx, len(song_list), self.count))
87 | sys.stdout.flush()
88 | midi_url = url + s['filename']
89 | content = self._request_url(midi_url, doctype='content')
90 | fn = os.path.join(dir_path, s['filename'])
91 | with open(fn, "wb") as f:
92 | f.write(content)
93 |
94 | self.count += 1
95 |
96 | def crawl_archive(self):
97 | dir_list = self.fetch_dirs(self.BASE_URL)
98 |
99 | if not os.path.exists(self.archive_dir,):
100 | os.makedirs(self.archive_dir,)
101 |
102 | info = dict()
103 | for d in dir_list:
104 | root_dir = d.strip('/')
105 | print('{{%s}}' % root_dir)
106 | subdir_url = self.BASE_URL + d
107 | subdir_list = self.fetch_dirs(subdir_url)
108 |
109 | tmp_dict = dict()
110 | for sd in subdir_list:
111 | root_subdir = sd.strip('/')
112 | print('[%s]'%root_subdir)
113 | page_url = subdir_url + sd
114 | sl = self.fetch_songs(page_url)
115 | if sl:
116 | self.crawl_songs(page_url, sl, os.path.join(self.archive_dir, root_dir, root_subdir))
117 | tmp_dict[sd] = sl
118 |
119 | info[d] = tmp_dict
120 |
121 | with open(os.path.join(self.archive_dir,'archive.json'), "w") as f:
122 | json.dump(info, f)
123 |
124 | def run(self):
125 | s = time.time()
126 | self.crawl_archive()
127 | e = time.time()
128 | self._log_print(time.strftime("\nElapsed time: %H:%M:%S", time.gmtime(s-e)))
129 | self._log_print('Total %d Songs'%self.count)
130 |
131 | if __name__ == '__main__':
132 | vc = VGMCrawler()
133 | vc.run()
134 |
--------------------------------------------------------------------------------
/piano-e-competition/utils/EPcompCrawler.py:
--------------------------------------------------------------------------------
1 | import random
2 | import requests
3 | from bs4 import BeautifulSoup
4 | import os
5 | import sys
6 | import time
7 | import json
8 | import re
9 | import random
10 |
11 | class EPcompCrawler():
12 | BASE_URL = 'http://www.piano-e-competition.com'
13 | ROOT = 'archive'
14 | YEARS = ['/midi_2002.asp', '/midi_2004.asp', '/midi_2006.asp',
15 | '/midi_2008.asp', '/midi_2009.asp', '/midi_2011.asp']
16 |
17 | def __init__(self, sleep_time=0.1, log=True):
18 | self.sleep_time = sleep_time
19 | self.log = log
20 | self.mid_cnt = 0
21 | self.zip_cnt = 0
22 |
23 | def _request_url(self, url, doctype='html'):
24 | response = requests.get(url)
25 |
26 | # sleep
27 | time.sleep(self.sleep_time)
28 |
29 | # return
30 | if doctype =='html':
31 | soup = BeautifulSoup(response.text, 'html.parser')
32 | return soup
33 | elif doctype =='content':
34 | return response.content
35 | else:
36 | return response
37 |
38 | def _log_print(self, log, quite=False):
39 | if not quite:
40 | print(log)
41 |
42 | if self.log:
43 | with open("log.txt", "a") as f:
44 | print(log, file=f)
45 |
46 | def fetch_year_songs(self, year_url):
47 | soup = self._request_url(year_url)
48 | a_list = soup.find_all('a')
49 | midi_list = []
50 | zip_list = []
51 | print(len(a_list))
52 |
53 | for idx in range(len(a_list)):
54 | a = a_list[idx]
55 |
56 | url = a.get('href')
57 | if url and (('.MID' in url) or ('.mid' in url)):
58 | now_performer = re.search('(\D+?)(\d+?).', url.split('/')[-1]).group(1)
59 | song_name = re.sub( '\s+', ' ', a.text.replace('\r', '').replace('\n', '')).strip()
60 | try:
61 | composer = a.parent.parent.td.text.strip()
62 | except:
63 | try:
64 | composer = a.parent.parent.parent.td.text.strip()
65 | except:
66 | composer = a.parent.parent.parent.parent.td.text.strip()
67 | print('%-10s |%-40s |[%s | %s]' % (now_performer, url, song_name, composer))
68 | midi_list.append((now_performer, url, song_name))
69 |
70 | if url and (('.ZIP' in url) or ('.zip' in url)):
71 | zip_fn = url.lstrip('../')
72 | print('%-10s %s'%(now_performer, zip_fn))
73 | zip_list.append((now_performer, zip_fn))
74 | return midi_list, zip_list
75 |
76 | def crawl_year_songs(self, midi_list, zip_list, dir_path):
77 | path_midi = os.path.join(dir_path, 'midi')
78 | path_zip = os.path.join(dir_path, 'zip')
79 |
80 | if not os.path.exists(path_midi):
81 | os.makedirs(path_midi)
82 | if not os.path.exists(path_zip):
83 | os.makedirs(path_zip)
84 |
85 | print('=================midi=================')
86 | for idx in range(len(midi_list)):
87 | m_url = midi_list[idx][1]
88 |
89 | if '/ecompetition' not in m_url:
90 | m_url = 'http://www.piano-e-competition.com/ecompetition/' + m_url
91 | else:
92 | m_url = self.BASE_URL + m_url
93 |
94 |
95 | print(idx, m_url)
96 | content = self._request_url(m_url, doctype='content')
97 | fn = m_url.split('/')[-1]
98 |
99 | with open(os.path.join(path_midi, fn), "wb") as f:
100 | f.write(content)
101 |
102 | print('=================zip=================')
103 | for idx in range(len(zip_list)):
104 | z_url = zip_list[idx][1]
105 | z_url = self.BASE_URL + '/' + z_url
106 | print(idx, z_url)
107 | content = self._request_url(z_url, doctype='content')
108 | fn = z_url.split('/')[-1]
109 |
110 | with open(os.path.join(path_zip, fn), "wb") as f:
111 | f.write(content)
112 |
113 | def crawl_archive(self):
114 |
115 | if not os.path.exists(self.ROOT):
116 | os.makedirs(self.ROOT)
117 |
118 | archive_dict = dict()
119 |
120 | for y in self.YEARS:
121 | ml, zl = self.fetch_year_songs(self.BASE_URL+y)
122 | year = re.search('(\d{4}).',y).group(1)
123 | path_year = os.path.join(self.ROOT, year)
124 | if not os.path.exists( path_year ):
125 | os.makedirs( path_year )
126 |
127 | print('{%s}' % year)
128 |
129 | self.mid_cnt += len(ml)
130 | self.zip_cnt += len(zl)
131 |
132 | tmp = {'mid':ml, 'zip':zl}
133 | archive_dict[year] = tmp
134 |
135 | self.crawl_year_songs(ml, zl, path_year)
136 |
137 | with open(os.path.join(self.ROOT, 'archive.json'), "w") as f:
138 | json.dump(archive_dict, f)
139 |
140 | def run(self):
141 | s = time.time()
142 | self.crawl_archive()
143 | e = time.time()
144 |
145 | self._log_print('Total midi files %d'%self.mid_cnt)
146 | self._log_print('Total zip files %d'%self.zip_cnt)
147 | self._log_print(time.strftime("\nElapsed time: %H:%M:%S", time.gmtime(s-e)))
148 |
149 |
150 | if __name__ == '__main__':
151 | ec = EPcompCrawler()
152 | ec.run()
--------------------------------------------------------------------------------
/theorytab/utils/theorytab_crawler.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from bs4 import BeautifulSoup
3 | import os
4 | import time
5 | import json
6 | import string
7 |
8 |
9 | website = 'https://www.hooktheory.com'
10 | base_url = website + '/theorytab/artists/'
11 | sleep_time = 0.11
12 | alphabet_list = string.ascii_lowercase
13 |
14 |
15 | def song_retrieval(artist, song, path_song):
16 |
17 | song_url = 'https://www.hooktheory.com/theorytab/view/' + artist + '/' + song
18 | response_song = requests.get(song_url)
19 |
20 | soup = BeautifulSoup(response_song.text, 'html.parser')
21 | li_list = soup.findAll("li", {"role": "presentation"})
22 |
23 | section_list = []
24 | pk_list = []
25 |
26 | # section
27 | for i in range(len(li_list)-1):
28 | sec = li_list[i].text.strip().lower().replace(" ", "-")
29 | section_list.append(sec)
30 | pk_list.append(soup.findAll("div", {"role": "tabpanel", "id": sec})[0].contents[0]['id'])
31 |
32 | # save xml
33 | for idx, pk in enumerate(pk_list):
34 | req_url = 'https://www.hooktheory.com/songs/getXmlByPk?pk=' + str(pk)
35 | response_info = requests.get(req_url)
36 | content = response_info.text
37 |
38 | with open(os.path.join(path_song, section_list[idx] + ".xml"), "w", encoding="utf-8") as f:
39 | f.write(content)
40 | time.sleep(0.08)
41 |
42 | # get genre
43 | wikiid = soup.findAll("multiselect", {"items": "genres"})[0]['wikiid']
44 | response_genre = requests.get('https://www.hooktheory.com/wiki/' + str(wikiid) + '/genres')
45 | genre_act_list = json.loads(response_genre.text)
46 | genres = []
47 | for g in genre_act_list:
48 | if g['active']:
49 | genres.append(g['name'])
50 |
51 | # saving
52 | info = {'section': section_list, 'pk': pk_list, 'song_url': song_url,
53 | 'genres': genres, 'wikiid': wikiid}
54 |
55 | with open(os.path.join(path_song, 'song_info.json'), "w") as f:
56 | json.dump(info, f)
57 |
58 |
59 | def get_song_list(url_artist, quite=False):
60 | response_tmp = requests.get(website + url_artist)
61 | soup = BeautifulSoup(response_tmp.text, 'html.parser')
62 | item_list = soup.find_all("li", {"class": "grid-item"})
63 |
64 | song_name_list = []
65 | for item in item_list:
66 | song_name = item.find_all("a", {"class": "a-tab-cover"})[0]['href'].split('/')[-1]
67 | song_name_list.append(song_name)
68 | if not quite:
69 | print(' > %s' % song_name)
70 | return song_name_list
71 |
72 |
73 | def traverse_website():
74 | '''
75 | Retrieve all urls of artists and songs from the website
76 | '''
77 |
78 | list_pages = []
79 | archive_artist = dict()
80 | artist_count = 0
81 | song_count = 0
82 |
83 | for ch in alphabet_list:
84 | time.sleep(sleep_time)
85 | url = base_url + ch
86 | response_tmp = requests.get(url)
87 | soup = BeautifulSoup(response_tmp.text, 'html.parser')
88 | page_count = 0
89 |
90 | print('==[%c]=================================================' % ch)
91 |
92 | # get artists list by pages
93 | url_artist_list = []
94 | for page in range(1, 9999):
95 | url = 'https://www.hooktheory.com/theorytab/artists/'+ch+'?page=' + str(page)
96 |
97 | time.sleep(sleep_time)
98 | response_tmp = requests.get(url)
99 | soup = BeautifulSoup(response_tmp.text, 'html.parser')
100 | item_list = soup.find_all("li", {"class": "overlay-trigger"})
101 |
102 | if item_list:
103 | print(url)
104 | page_count += 1
105 | else:
106 | break
107 |
108 | for item in item_list:
109 | url_artist_list.append(item.find_all("a", {"class": "a-no-decoration"})[0]['href'])
110 |
111 | print('Total:', len(url_artist_list))
112 |
113 | print('----')
114 |
115 | if not page_count:
116 | page_count = 1
117 |
118 | # get song of artists
119 | artist_song_dict = dict()
120 |
121 | for url_artist in url_artist_list:
122 | artist_count += 1
123 | time.sleep(sleep_time)
124 | artist_name = url_artist.split('/')[-1]
125 | print(artist_name)
126 | song_name_list = get_song_list(url_artist)
127 | song_count += len(song_name_list)
128 | artist_song_dict[artist_name] = song_name_list
129 |
130 | archive_artist[ch] = artist_song_dict
131 | list_pages.append(page_count)
132 |
133 | print('=======================================================')
134 | print(list_pages)
135 | print('Artists:', artist_count)
136 | print('Songs:', song_count)
137 |
138 | archive_artist['num_song'] = song_count
139 | archive_artist['num_artist'] = artist_count
140 |
141 | with open('archive_artist.json', "w") as f:
142 | json.dump(archive_artist, f)
143 |
144 |
145 | if __name__ == '__main__':
146 |
147 | traverse_website()
148 |
149 | # root for crawled dataset
150 | root_dir = 'archive'
151 | with open('archive_artist.json', "r") as f:
152 | archive_artist = json.load(f)
153 |
154 | count_ok = 0
155 | song_count = archive_artist['num_song']
156 |
157 | for ch in alphabet_list:
158 | path_ch = os.path.join(root_dir, ch)
159 | print('==[%c]=================================================' % ch)
160 |
161 | if not os.path.exists(path_ch):
162 | os.makedirs(path_ch)
163 |
164 | for a_name in archive_artist[ch].keys():
165 | for s_name in archive_artist[ch][a_name]:
166 |
167 | try:
168 | print('(%3d/%3d) %s %s' % (count_ok, song_count, a_name, s_name))
169 | path_song = os.path.join(path_ch, a_name, s_name)
170 |
171 | if not os.path.exists(path_song):
172 | os.makedirs(path_song)
173 |
174 | time.sleep(sleep_time)
175 | song_retrieval(a_name, s_name, path_song)
176 |
177 | count_ok += 1
178 |
179 | except Exception as e:
180 | print(e)
181 |
182 | print('total:', count_ok)
183 |
--------------------------------------------------------------------------------
/5-track-pianoroll/parser.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import os
3 | from pypianoroll import Multitrack, Track
4 | import json
5 | import pickle
6 |
7 | family_name=[
8 | 'drum',
9 | 'bass',
10 | 'guitar',
11 | 'string',
12 | 'piano',
13 | ]
14 |
15 | family_thres = [
16 | (2, 24), # drum
17 | (1, 96), # bass
18 | (2, 156), # guitar
19 | (2, 156), # string,
20 | (2, 156), # piano
21 | ]
22 |
23 |
24 | def findall_endswith(root):
25 | """Traverse `root` recursively and yield all files ending with `postfix`"""
26 | for dirpath, _, filenames in os.walk(root):
27 | for filename in filenames:
28 | if filename.endswith('.npz'):
29 | yield os.path.join(dirpath, filename)
30 |
31 | def check_which_family(track):
32 | is_piano = lambda program, is_drum: not is_drum and ((program >= 0 and program <= 7)
33 | or (program >= 16 and program <= 23))
34 | is_guitar = lambda program: program >= 24 and program <= 31
35 | is_bass = lambda program: program >= 32 and program <= 39
36 | is_string = lambda program: program >=40 and program <= 51
37 |
38 | # drum, bass, guitar, string, piano
39 | is_instr_act = lambda program, is_drum: np.array([is_drum, is_bass(program), is_guitar(program),
40 | is_string(program), is_piano(program, is_drum)])
41 |
42 | instr_act = is_instr_act(track.program, track.is_drum)
43 | return instr_act
44 |
45 | def check_instr_act(multitrack):
46 | instr_act_all = np.zeros(5)
47 | for track in multitrack.tracks:
48 | instr_act = check_which_family(track)
49 | instr_act_all += instr_act
50 | instr_act_cnt = sum(instr_act_all > 0)
51 | return instr_act_all, instr_act_cnt
52 |
53 | def segment_quality(pianoroll, thres_pitch, thres_beats):
54 | pitch_sum = sum(np.sum(pianoroll, axis=0) > 0)
55 | beat_sum = sum(np.sum(pianoroll, axis=1) > 0)
56 | score = pitch_sum + beat_sum
57 | return (pitch_sum >= thres_pitch) and (beat_sum >= thres_beats), (pitch_sum, beat_sum)
58 |
59 | def proc_instr_intersection_list(npz_list):
60 | cnt_ok = 0
61 | list_ok = []
62 | thres_instr_num = 5
63 | for nidx in range(len(npz_list)): #len(npz_list)
64 | if nidx % 500 is 0:
65 | print(nidx, '-', cnt_ok)
66 | npz_file = npz_list[nidx]
67 | multitrack = Multitrack(npz_file)
68 |
69 | if len(multitrack.tracks) < 5:
70 | continue
71 |
72 | instr_act_all, instr_act_cnt = check_instr_act(multitrack)
73 |
74 | if instr_act_cnt != 5:
75 | continue
76 |
77 | list_ok.append(npz_file)
78 | cnt_ok += 1
79 |
80 | print(cnt_ok)
81 | return list_ok
82 |
83 |
84 | if __name__ == '__main__':
85 | # root = 'lpd_cleansed'
86 | # npz_list = list(findall_endswith(root))
87 |
88 | # with open('npz_list.pickle', 'wb') as f:
89 | # pickle.dump(npz_list, f, protocol=pickle.HIGHEST_PROTOCOL)
90 | # with open('npz_list.pickle', 'rb') as f:
91 | # npz_list = pickle.load(f)
92 |
93 | # list_ok = proc_instr_intersection_list(npz_list)
94 | # with open('list_ok.pickle', 'wb') as f:
95 | # pickle.dump(list_ok, f, protocol=pickle.HIGHEST_PROTOCOL)
96 | with open('list_ok.pickle', 'rb') as f:
97 | list_ok = pickle.load(f)
98 |
99 |
100 |
101 | num_consecutive_bar = 8
102 | resol = 96
103 | down_sample = 2
104 | cnt_totall_segments = 0
105 | cnt_augmented = 0
106 | ok_segment_list = []
107 | hop_size = (num_consecutive_bar / 4)
108 |
109 | num_list_ok = len(list_ok)
110 | for oid in range(len(list_ok)):
111 | print('==', oid, '/', num_list_ok,'===============')
112 | npz_ok = list_ok[oid]
113 | multitrack = Multitrack(npz_ok)
114 | downbeat = multitrack.downbeat
115 |
116 | num_bar = len(downbeat) // resol
117 | hop_iter = 0
118 |
119 | song_ok_segments = []
120 | for bidx in range(num_bar-num_consecutive_bar):
121 | if hop_iter > 0:
122 | hop_iter -= 1
123 | continue
124 |
125 |
126 | st = bidx * resol
127 | ed = st + num_consecutive_bar * resol
128 |
129 | best_instr = [None] * 5
130 | best_score = [-1] * 5
131 | second_act = [False] * 5
132 | second_instr = [None] * 5
133 | is_all_ok = [False] * 5
134 | for tidx, track in enumerate(multitrack.tracks):
135 | # track[st:ed].plot()
136 | tmp_map = check_which_family(track)
137 | in_family = np.where(tmp_map)[0]
138 |
139 | if not len(in_family):
140 | continue
141 | family = in_family[0]
142 |
143 | tmp_pianoroll = track[st:ed:down_sample].pianoroll
144 | is_ok, score = segment_quality(tmp_pianoroll, family_thres[family][0], family_thres[family][1])
145 |
146 | if is_ok and sum(score) > best_score[family]:
147 | track.name = family_name[family]
148 | best_instr[family] = track[st:ed:down_sample]
149 | best_score[family] = sum(score)
150 | is_all_ok[family] = True
151 |
152 | if sum(is_all_ok) == 5:
153 | # print(bidx)
154 | hop_iter = np.random.randint(0, 1) + hop_size
155 | song_ok_segments.append(Multitrack(tracks=best_instr,
156 | downbeat=list(range(0, 383, 48)), beat_resolution=12))
157 |
158 | cnt_ok_segment = len(song_ok_segments)
159 | if cnt_ok_segment > 6:
160 | seed = (6, cnt_ok_segment//2)
161 | if cnt_ok_segment > 11:
162 | seed = (11, cnt_ok_segment//3)
163 | if cnt_ok_segment > 15:
164 | seed = (15, cnt_ok_segment//4)
165 |
166 | rand_idx = np.random.permutation(cnt_ok_segment)[:max(seed)]
167 | song_ok_segments = [song_ok_segments[ridx] for ridx in rand_idx]
168 | ok_segment_list.extend(song_ok_segments)
169 | cnt_ok_segment = len(rand_idx)
170 | else:
171 | ok_segment_list.extend(song_ok_segments)
172 |
173 | cnt_totall_segments += len(song_ok_segments)
174 | print('cur:%d | acc:%d'%(cnt_ok_segment, cnt_totall_segments))
175 |
176 | print('---')
177 | print(cnt_totall_segments)
178 | print(len(ok_segment_list))
179 | np.save('segments.npy', ok_segment_list)
--------------------------------------------------------------------------------
/hymnal/utils/hymnCrawler.py:
--------------------------------------------------------------------------------
1 | import requests
2 | from bs4 import BeautifulSoup
3 | import os
4 | import time
5 | import json
6 | import string
7 | import random
8 | from lxml import etree
9 |
10 | class HymnCrawler():
11 | BASE_URL = 'https://www.hymnal.net'
12 |
13 | def __init__(self, sleep_time = 0.1, log=True):
14 | self.sleep_time =sleep_time
15 | self.meta_category = {'classic': self.BASE_URL + '/en/song-index/h',
16 | 'new_tunes': self.BASE_URL + '/en/song-index/nt',
17 | 'new_songs': self.BASE_URL + '/en/song-index/ns',
18 | 'children': self.BASE_URL + '/en/song-index/c'}
19 |
20 | self.log = log
21 | self.metadata = None
22 |
23 | def _request_url(self, url, doctype='html'):
24 | response = requests.get(url)
25 | time.sleep(self.sleep_time)
26 | if doctype =='html':
27 | soup = BeautifulSoup(response.text, 'html.parser')
28 | return soup
29 | elif doctype =='content':
30 | return response.content
31 | else:
32 | pass
33 |
34 | def _log_print(self, log):
35 | print(log)
36 | if self.log:
37 | with open("log.txt", "a") as f:
38 | print(log, file=f)
39 |
40 | def fetch_page_list(self, url):
41 | soup = self._request_url(url)
42 | tag_list = soup.find_all('div', {'class':'list-group'})[0].find_all('a', {'class':'list-group-item'})
43 | return [t['href'] for t in tag_list]
44 |
45 | def fetch_category_list(self, url):
46 | song_list = []
47 | soup = self._request_url(url)
48 | try:
49 | tag_list = soup.find_all('div', {'class':'letters'})[0].findAll('a')
50 | alphabet_list = [t.text for t in tag_list]
51 | for ch in alphabet_list:
52 | page_url = url+ '/' + ch
53 | self._log_print(page_url)
54 | song_list.extend(self.fetch_page_list(page_url))
55 | except:
56 | self._log_print(url)
57 | song_list.extend(self.fetch_page_list(url))
58 | return song_list
59 |
60 | def fetch_song(self, url, song_dir):
61 |
62 | soup = self._request_url(url)
63 |
64 | # (url, extension, filename)
65 | data_list = [('/f=mid', '.mid', 'all'), ('/f=mp3', '.mp3', 'audio'), ('/f=tune', '.mid', 'melody'),
66 | ('/f=ppdf', '.pdf', 'ls_paino'), ('/f=pdf', '.pdf','ls_guitar'), ('/f=gtpdf', '.pdf', 'ls_text')]
67 |
68 | # save download files
69 | for d in data_list:
70 | r = requests.get(url+ d[0])
71 |
72 | if song_dir:
73 | with open(os.path.join(song_dir,d[2] + d[1]), 'wb') as f:
74 | f.write(r.content)
75 |
76 | # metadata
77 | content_list = []
78 | tag_list = soup.find_all('div', {'class':'row common-panel'})[0].find_all('div', {'class':'col-xs-7 col-sm-8 no-padding'})
79 |
80 | for t in tag_list:
81 | content_list.append(t.text.strip())
82 | label_list = []
83 | tag_list = soup.find_all('div', {'class':'row common-panel'})[0].find_all('label', {'class':'col-xs-5 col-sm-4'})
84 | for t in tag_list:
85 | label_list.append(t.text.replace(':',''))
86 | metadata = dict(zip(label_list, content_list))
87 |
88 | # title
89 | title = soup.find('h1', {'class':"text-center"}).text.strip()
90 | metadata['title'] = title
91 |
92 | # lyric table
93 | lyric_xml = soup.find_all('div', {'class':'col-xs-12 lyrics'})[0].find('table')
94 |
95 | if song_dir:
96 | with open( os.path.join(song_dir, 'song_metadata.json'), "w") as f:
97 | json.dump(metadata , f)
98 |
99 | with open( os.path.join(song_dir, 'lyric.xml'), "w", encoding='utf-8') as f:
100 | f.write(str(lyric_xml))
101 |
102 | return lyric_xml, metadata
103 |
104 | def craw_archive(self, archive_dir='archive'):
105 | metadata = dict()
106 | for k in self.meta_category.keys():
107 | category_url = self.meta_category[k]
108 | metadata[k] = self.fetch_category_list(category_url)
109 |
110 | # saving
111 | if archive_dir:
112 | if not os.path.exists(archive_dir):
113 | os.makedirs(archive_dir)
114 |
115 | with open(os.path.join(archive_dir, 'archive_metadata.json'), "w") as f:
116 | json.dump(metadata , f)
117 |
118 | return metadata
119 |
120 | def craw_songs(self, metadata, archive_dir='archive'):
121 | count = 0
122 | count_success = 0
123 | for k in list(metadata):
124 | self._log_print('> %s'%k)
125 | category_dir = os.path.join(archive_dir, k)
126 | if not os.path.exists(category_dir):
127 | os.makedirs(category_dir)
128 |
129 | song_list = metadata[k]
130 |
131 | numOfSongs = len(song_list)
132 | for i in range(numOfSongs):
133 | song_url = self.BASE_URL + song_list[i]
134 | song_id = song_url.split('/')[-1]
135 | self._log_print(' (%d/%d) %s'%(i+1, numOfSongs, song_url))
136 | song_dir = os.path.join(category_dir, song_id)
137 |
138 | if not os.path.exists(song_dir):
139 | os.makedirs(song_dir)
140 |
141 | try:
142 | self.fetch_song(song_url, song_dir)
143 | metadata['err'] = False
144 | count_success += 1
145 | except:
146 | self._log_print('error!!')
147 | metadata['err'] = True
148 |
149 | count += 1
150 | self._log_print('total: %d songs'%count)
151 |
152 | return metadata
153 |
154 | def reload(self, archive_dir='archive'):
155 | with open(os.path.join(archive_dir, 'archive_metadata.json'), "r") as f:
156 | self.metadata =json.load(f)
157 |
158 | def run(self, archive_dir='archive', reload=False):
159 |
160 | self._log_print("=================================================")
161 |
162 | if not reload:
163 | self.metadata = self.craw_archive(archive_dir=archive_dir)
164 | else:
165 | self.reload(archive_dir=archive_dir)
166 |
167 | self.metadata = self.craw_songs(self.metadata, archive_dir=archive_dir)
168 |
169 | with open(os.path.join(archive_dir, 'archive_metadata.json'), "w") as f:
170 | json.dump(self.metadata, f)
171 |
172 | if __name__ == '__main__':
173 |
174 | hc = HymnCrawler()
175 | s = time.time()
176 | hc.run()
177 | e = time.time()
178 | time.strftime("\nElapsed time: %H:%M:%S", time.gmtime(s-e))
179 |
180 |
--------------------------------------------------------------------------------
/vgmdb/utils/VGMCrawler.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {
7 | "collapsed": true
8 | },
9 | "outputs": [],
10 | "source": [
11 | "import requests\n",
12 | "from bs4 import BeautifulSoup\n",
13 | "import os\n",
14 | "import sys\n",
15 | "import time\n",
16 | "import json"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": 190,
22 | "metadata": {
23 | "collapsed": false
24 | },
25 | "outputs": [],
26 | "source": [
27 | "class VGMCrawler():\n",
28 | " BASE_URL = 'https://www.vgmusic.com/music/console/'\n",
29 | " archive_dir = 'archive'\n",
30 | " \n",
31 | " def __init__(self, sleep_time=0.1, log=True):\n",
32 | " self.sleep_time = sleep_time\n",
33 | " self.log = log\n",
34 | " self.count = 0\n",
35 | " \n",
36 | " def _request_url(self, url, doctype='html'):\n",
37 | " # set header\n",
38 | " response = requests.get(url)\n",
39 | "\n",
40 | " # sleep\n",
41 | " time.sleep(self.sleep_time)\n",
42 | "\n",
43 | " # return\n",
44 | " if doctype =='html':\n",
45 | " soup = BeautifulSoup(response.text, 'html.parser')\n",
46 | " return soup\n",
47 | " elif doctype =='content':\n",
48 | " return response.content\n",
49 | " else:\n",
50 | " return response\n",
51 | "\n",
52 | " def _log_print(self, log, quite=False):\n",
53 | " if not quite:\n",
54 | " print(log)\n",
55 | "\n",
56 | " if self.log:\n",
57 | " with open(\"log.txt\", \"a\") as f:\n",
58 | " print(log, file=f)\n",
59 | " \n",
60 | " def fetch_dirs(self, url):\n",
61 | " soup = self._request_url(url)\n",
62 | " tr_list = soup.find('table').find_all('tr')\n",
63 | " dir_list = []\n",
64 | "\n",
65 | " for i in range(3, len(tr_list)-1):\n",
66 | " dir_list.append(tr_list[i].find_all('td')[1].text)\n",
67 | " return dir_list\n",
68 | " \n",
69 | " def fetch_songs(self, url):\n",
70 | " soup = self._request_url(url)\n",
71 | " tr_list = soup.find('table').find_all('tr')\n",
72 | " song_list = []\n",
73 | " \n",
74 | " if len(tr_list) == 4:\n",
75 | " return None\n",
76 | " for i in range(2, len(tr_list)):\n",
77 | " now_tr = tr_list[i]\n",
78 | " if now_tr.get(\"class\") == ['header']:\n",
79 | " album_name = now_tr.text.strip('\\n')\n",
80 | " else:\n",
81 | " if not now_tr.td.get('colspan'):\n",
82 | "\n",
83 | " # get info\n",
84 | " info = now_tr.td.text.split('\\n')\n",
85 | " song_midi = now_tr.a['href']\n",
86 | " song_name = info[0]\n",
87 | " song_size = info[1]\n",
88 | " song_author = info[3]\n",
89 | " song_list.append({\n",
90 | " 'filename':song_midi, \n",
91 | " 'song_name':song_name,\n",
92 | " 'song_size':song_size,\n",
93 | " 'song_author':song_author,\n",
94 | " 'album_name':album_name})\n",
95 | " print(' |%30s |%30s |%13s |%10s |%s '%(album_name, song_name, song_size, song_author, song_midi ))\n",
96 | " \n",
97 | " \n",
98 | " return song_list\n",
99 | " \n",
100 | " def crawl_songs(self, url, song_list, dir_path):\n",
101 | " if not os.path.exists(dir_path):\n",
102 | " os.makedirs(dir_path)\n",
103 | " \n",
104 | " for idx, s in enumerate(song_list):\n",
105 | " sys.stdout.write('%d/%d - total: %d\\n' % (idx, len(song_list), self.count))\n",
106 | " sys.stdout.flush()\n",
107 | " midi_url = url + s['filename']\n",
108 | " content = self._request_url(midi_url, doctype='content')\n",
109 | " fn = os.path.join(dir_path, s['filename'])\n",
110 | " with open(fn, \"wb\") as f:\n",
111 | " f.write(content)\n",
112 | " \n",
113 | " self.count += 1\n",
114 | " \n",
115 | " def crawl_archive(self):\n",
116 | " dir_list = self.fetch_dirs(self.BASE_URL)\n",
117 | " \n",
118 | " if not os.path.exists(self.archive_dir,):\n",
119 | " os.makedirs(self.archive_dir,)\n",
120 | " \n",
121 | " info = dict()\n",
122 | " for d in dir_list:\n",
123 | " root_dir = d.strip('/')\n",
124 | " print('{{%s}}' % root_dir)\n",
125 | " subdir_url = self.BASE_URL + d\n",
126 | " subdir_list = self.fetch_dirs(subdir_url)\n",
127 | " \n",
128 | " tmp_dict = dict()\n",
129 | " for sd in subdir_list:\n",
130 | " root_subdir = sd.strip('/')\n",
131 | " print('[%s]'%root_subdir)\n",
132 | " page_url = subdir_url + sd\n",
133 | " sl = self.fetch_songs(page_url)\n",
134 | " if sl:\n",
135 | " self.crawl_songs(page_url, sl, os.path.join(self.archive_dir, root_dir, root_subdir))\n",
136 | " tmp_dict[sd] = sl\n",
137 | "\n",
138 | " info[d] = tmp_dict\n",
139 | " \n",
140 | " with open(os.path.join(self.archive_dir,'archive.json'), \"w\") as f:\n",
141 | " json.dump(info, f) \n",
142 | " \n",
143 | " def run(self): \n",
144 | " s = time.time()\n",
145 | " self.crawl_archive()\n",
146 | " e = time.time()\n",
147 | " self._log_print(time.strftime(\"\\nElapsed time: %H:%M:%S\", time.gmtime(s-e)))\n",
148 | " self._log_print('Total %d Songs'&self.count)\n",
149 | " \n"
150 | ]
151 | },
152 | {
153 | "cell_type": "code",
154 | "execution_count": null,
155 | "metadata": {
156 | "collapsed": true
157 | },
158 | "outputs": [],
159 | "source": [
160 | "if __name__ == '__main__':\n",
161 | " vc = VGMCrawler()\n",
162 | " vc.run()"
163 | ]
164 | },
165 | {
166 | "cell_type": "code",
167 | "execution_count": 191,
168 | "metadata": {
169 | "collapsed": false
170 | },
171 | "outputs": [],
172 | "source": [
173 | "vc = VGMCrawler()\n",
174 | "vc.fetch_songs('https://www.vgmusic.com/music/console/magnavox/odyssey/')"
175 | ]
176 | }
177 | ],
178 | "metadata": {
179 | "kernelspec": {
180 | "display_name": "Python [mir]",
181 | "language": "python",
182 | "name": "Python [mir]"
183 | },
184 | "language_info": {
185 | "codemirror_mode": {
186 | "name": "ipython",
187 | "version": 3
188 | },
189 | "file_extension": ".py",
190 | "mimetype": "text/x-python",
191 | "name": "python",
192 | "nbconvert_exporter": "python",
193 | "pygments_lexer": "ipython3",
194 | "version": "3.5.2"
195 | }
196 | },
197 | "nbformat": 4,
198 | "nbformat_minor": 0
199 | }
200 |
--------------------------------------------------------------------------------
/hymnal/utils/hymnCrawler.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "metadata": {
7 | "collapsed": true
8 | },
9 | "outputs": [],
10 | "source": [
11 | "import requests\n",
12 | "from bs4 import BeautifulSoup\n",
13 | "import os\n",
14 | "import time\n",
15 | "import json\n",
16 | "import string\n",
17 | "import random\n",
18 | "from lxml import etree"
19 | ]
20 | },
21 | {
22 | "cell_type": "code",
23 | "execution_count": null,
24 | "metadata": {
25 | "collapsed": false
26 | },
27 | "outputs": [],
28 | "source": [
29 | "class HymnCraler():\n",
30 | " BASE_URL = 'https://www.hymnal.net'\n",
31 | " \n",
32 | " def __init__(self, sleep_time = 0.1, log=True):\n",
33 | " self.sleep_time =sleep_time\n",
34 | " self.meta_category = {'classic': self.BASE_URL + '/en/song-index/h', \n",
35 | " 'new_tunes': self.BASE_URL + '/en/song-index/nt', \n",
36 | " 'new_songs': self.BASE_URL + '/en/song-index/ns', \n",
37 | " 'children': self.BASE_URL + '/en/song-index/c'}\n",
38 | " \n",
39 | " self.log = log\n",
40 | " self.metadata = None \n",
41 | "\n",
42 | " def _request_url(self, url, doctype='html'):\n",
43 | " response = requests.get(url)\n",
44 | " if doctype =='html':\n",
45 | " soup = BeautifulSoup(response.text, 'html.parser')\n",
46 | " return soup\n",
47 | " elif doctype =='content':\n",
48 | " return response.content\n",
49 | " else:\n",
50 | " pass\n",
51 | "\n",
52 | " def _log_print(self, log):\n",
53 | " print(log)\n",
54 | " if self.log:\n",
55 | " with open(\"log.txt\", \"a\") as f:\n",
56 | " print(log, file=f)\n",
57 | "\n",
58 | " def fetch_page_list(self, url):\n",
59 | " soup = self._request_url(url)\n",
60 | " tag_list = soup.find_all('div', {'class':'list-group'})[0].find_all('a', {'class':'list-group-item'})\n",
61 | " return [t['href'] for t in tag_list]\n",
62 | "\n",
63 | " def fetch_category_list(self, url):\n",
64 | " song_list = []\n",
65 | " soup = self._request_url(url)\n",
66 | " try:\n",
67 | " tag_list = soup.find_all('div', {'class':'letters'})[0].findAll('a')\n",
68 | " alphabet_list = [t.text for t in tag_list]\n",
69 | " for ch in alphabet_list:\n",
70 | " page_url = url+ '/' + ch\n",
71 | " self._log_print(page_url)\n",
72 | " song_list.extend(self.fetch_page_list(page_url))\n",
73 | " except:\n",
74 | " self._log_print(url)\n",
75 | " song_list.extend(self.fetch_page_list(url))\n",
76 | " return song_list\n",
77 | "\n",
78 | " def fetch_song(self, url, song_dir):\n",
79 | "\n",
80 | " soup = self._request_url(url)\n",
81 | " \n",
82 | " # (url, extension, filename)\n",
83 | " data_list = [('/f=mid', '.mid', 'all'), ('/f=mp3', '.mp3', 'audio'), ('/f=tune', '.mid', 'melody'),\n",
84 | " ('/f=ppdf', '.pdf', 'ls_paino'), ('/f=pdf', '.pdf','ls_guitar'), ('/f=gtpdf', '.pdf', 'ls_text')]\n",
85 | "\n",
86 | " # save download files\n",
87 | " for d in data_list:\n",
88 | " r = requests.get(url+ d[0])\n",
89 | "\n",
90 | " if song_dir:\n",
91 | " with open(os.path.join(song_dir,d[2] + d[1]), 'wb') as f:\n",
92 | " f.write(r.content)\n",
93 | "\n",
94 | " # metadata\n",
95 | " content_list = [] \n",
96 | " tag_list = soup.find_all('div', {'class':'row common-panel'})[0].find_all('div', {'class':'col-xs-7 col-sm-8 no-padding'})\n",
97 | "\n",
98 | " for t in tag_list:\n",
99 | " content_list.append(t.text.strip())\n",
100 | " label_list = []\n",
101 | " tag_list = soup.find_all('div', {'class':'row common-panel'})[0].find_all('label', {'class':'col-xs-5 col-sm-4'})\n",
102 | " for t in tag_list:\n",
103 | " label_list.append(t.text.replace(':',''))\n",
104 | " metadata = dict(zip(label_list, content_list))\n",
105 | " \n",
106 | " # title\n",
107 | " title = soup.find('h1', {'class':\"text-center\"}).text.strip()\n",
108 | " metadata['title'] = title\n",
109 | " \n",
110 | " # lyric table\n",
111 | " lyric_xml = soup.find_all('div', {'class':'col-xs-12 lyrics'})[0].find('table')\n",
112 | "\n",
113 | " if song_dir:\n",
114 | " with open( os.path.join(song_dir, 'song_metadata.json'), \"w\") as f:\n",
115 | " json.dump(metadata , f)\n",
116 | "\n",
117 | " with open( os.path.join(song_dir, 'lyric.xml'), \"w\", encoding='utf-8') as f:\n",
118 | " f.write(str(lyric_xml))\n",
119 | "\n",
120 | " return lyric_xml, metadata\n",
121 | "\n",
122 | " def craw_archive(self, archive_dir='archive'):\n",
123 | " metadata = dict()\n",
124 | " for k in self.meta_category.keys():\n",
125 | " category_url = self.meta_category[k]\n",
126 | " metadata[k] = self.fetch_category_list(category_url)\n",
127 | "\n",
128 | " # saving\n",
129 | " if archive_dir:\n",
130 | " if not os.path.exists(archive_dir):\n",
131 | " os.makedirs(archive_dir)\n",
132 | "\n",
133 | " with open(os.path.join(archive_dir, 'archive_metadata.json'), \"w\") as f:\n",
134 | " json.dump(metadata , f)\n",
135 | "\n",
136 | " return metadata\n",
137 | "\n",
138 | " def craw_songs(self, metadata, archive_dir='archive'):\n",
139 | " count = 0\n",
140 | " count_success = 0\n",
141 | " for k in list(metadata):\n",
142 | " self._log_print('> %s'%k)\n",
143 | " category_dir = os.path.join(archive_dir, k)\n",
144 | " if not os.path.exists(category_dir):\n",
145 | " os.makedirs(category_dir)\n",
146 | "\n",
147 | " song_list = metadata[k]\n",
148 | "\n",
149 | " numOfSongs = len(song_list)\n",
150 | " for i in range(numOfSongs):\n",
151 | " song_url = self.BASE_URL + song_list[i]\n",
152 | " song_id = song_url.split('/')[-1]\n",
153 | " self._log_print(' (%d/%d) %s'%(i+1, numOfSongs, song_url))\n",
154 | " song_dir = os.path.join(category_dir, song_id)\n",
155 | "\n",
156 | " if not os.path.exists(song_dir):\n",
157 | " os.makedirs(song_dir)\n",
158 | " \n",
159 | " try:\n",
160 | " self.fetch_song(song_url, song_dir)\n",
161 | " metadata['err'] = False\n",
162 | " count_success += 1\n",
163 | " except:\n",
164 | " self._log_print('error!!')\n",
165 | " metadata['err'] = True\n",
166 | " \n",
167 | " count += 1\n",
168 | " self._log_print('total: %d songs'%count)\n",
169 | " \n",
170 | " return metadata\n",
171 | " \n",
172 | " def reload(self, archive_dir='archive'):\n",
173 | " with open(os.path.join(archive_dir, 'archive_metadata.json'), \"r\") as f:\n",
174 | " self.metadata =json.load(f)\n",
175 | " \n",
176 | " def run(self, archive_dir='archive', reload=False): \n",
177 | " \n",
178 | " self._log_print(\"=================================================\")\n",
179 | " \n",
180 | " if not reload:\n",
181 | " self.metadata = self.craw_archive(archive_dir=archive_dir)\n",
182 | " else:\n",
183 | " self.reload(archive_dir=archive_dir)\n",
184 | "\n",
185 | " self.metadata = self.craw_songs(self.metadata, archive_dir=archive_dir)\n",
186 | " \n",
187 | " with open(os.path.join(archive_dir, 'archive_metadata.json'), \"w\") as f:\n",
188 | " json.dump(self.metadata, f)\n",
189 | " \n"
190 | ]
191 | },
192 | {
193 | "cell_type": "code",
194 | "execution_count": null,
195 | "metadata": {
196 | "collapsed": false
197 | },
198 | "outputs": [
199 | {
200 | "name": "stdout",
201 | "output_type": "stream",
202 | "text": [
203 | "> new_songs\n",
204 | " (1/1070) https://www.hymnal.net/en/hymn/ns/528\n",
205 | " (2/1070) https://www.hymnal.net/en/hymn/ns/595\n",
206 | " (3/1070) https://www.hymnal.net/en/hymn/ns/584\n",
207 | " (4/1070) https://www.hymnal.net/en/hymn/ns/524\n",
208 | " (5/1070) https://www.hymnal.net/en/hymn/ns/501\n",
209 | " (6/1070) https://www.hymnal.net/en/hymn/lb/27\n",
210 | " (7/1070) https://www.hymnal.net/en/hymn/ns/550\n",
211 | " (8/1070) https://www.hymnal.net/en/hymn/ns/378\n",
212 | " (9/1070) https://www.hymnal.net/en/hymn/ns/309\n",
213 | "error!!\n",
214 | " (10/1070) https://www.hymnal.net/en/hymn/ns/398\n"
215 | ]
216 | }
217 | ],
218 | "source": [
219 | "if __name__ == '__main__':\n",
220 | " \n",
221 | " hc = HymnCraler()\n",
222 | " \n",
223 | " s = time.time()\n",
224 | " hc.run(reload=True)\n",
225 | " e = time.time()\n",
226 | " time.strftime(\"\\nElapsed time: %H:%M:%S\", time.gmtime(s-e))\n",
227 | " \n"
228 | ]
229 | }
230 | ],
231 | "metadata": {
232 | "anaconda-cloud": {},
233 | "kernelspec": {
234 | "display_name": "Python [mir]",
235 | "language": "python",
236 | "name": "Python [mir]"
237 | },
238 | "language_info": {
239 | "codemirror_mode": {
240 | "name": "ipython",
241 | "version": 3
242 | },
243 | "file_extension": ".py",
244 | "mimetype": "text/x-python",
245 | "name": "python",
246 | "nbconvert_exporter": "python",
247 | "pygments_lexer": "ipython3",
248 | "version": "3.5.2"
249 | }
250 | },
251 | "nbformat": 4,
252 | "nbformat_minor": 0
253 | }
254 |
--------------------------------------------------------------------------------
/theorytab/archive/a/aage-aleksandersen/fire-pils-og-en-pizza/verse.xml:
--------------------------------------------------------------------------------
1 |