├── docs └── proje.pdf ├── .env-example ├── commands └── search │ ├── links │ ├── styles.js │ ├── artists.js │ └── instruments.js │ ├── getAttributes.js │ ├── pieces.js │ └── index.js ├── package.json ├── lib ├── _command.js └── _db.js ├── index.js ├── .gitignore └── README.md /docs/proje.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fatiiates/8notes/HEAD/docs/proje.pdf -------------------------------------------------------------------------------- /.env-example: -------------------------------------------------------------------------------- 1 | MONGO_URL= 2 | MONGO_USER= 3 | MONGO_PASS= 4 | MONGO_DB= 5 | 6 | DATABASE_URL= 7 | DATABASE_USER= 8 | DATABASE_PASS= 9 | DATABASE_DB= -------------------------------------------------------------------------------- /commands/search/links/styles.js: -------------------------------------------------------------------------------- 1 | const { By } = require('selenium-webdriver'); 2 | 3 | async function getStyleLink(param = 1) { 4 | try { 5 | 6 | let styleSelector = By.css('#ulfornav > li:nth-of-type(2) > .menuforward li:nth-of-type(' + param + ') > a'); 7 | let styleLink = await driver.findElement(styleSelector).getAttribute("href"); 8 | return styleLink; 9 | 10 | } catch (error){ 11 | throw error; 12 | } 13 | }; 14 | 15 | module.exports = getStyleLink; 16 | 17 | -------------------------------------------------------------------------------- /commands/search/links/artists.js: -------------------------------------------------------------------------------- 1 | const { By } = require('selenium-webdriver'); 2 | 3 | async function getArtistLink(param = 1) { 4 | try { 5 | 6 | let styleSelector = By.css('#ulfornav > li:nth-of-type(3) > .menuforward li:nth-of-type(' + param + ') > a'); 7 | let styleLink = await driver.findElement(styleSelector).getAttribute("href"); 8 | return styleLink; 9 | 10 | } catch (error){ 11 | throw error; 12 | } 13 | }; 14 | 15 | module.exports = getArtistLink; 16 | 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "notes8", 3 | "version": "1.0.0", 4 | "description": "notes8.com sitesi üzerinden web scrapping ile veri çekme", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm test", 8 | "start": "node index.js", 9 | "stage-1": "node index.js search --all", 10 | "stage-2": "node index.js search --all-instrument" 11 | }, 12 | "keywords": [ 13 | "Web", 14 | "Scrapping" 15 | ], 16 | "author": "Fatih ATES, Bayram Atakan KOSDURMA, Tayfur GULEN", 17 | "license": "ISC", 18 | "dependencies": { 19 | "chalk": "^4.1.0", 20 | "chromedriver": "^90.0.0", 21 | "dotenv": "^8.2.0", 22 | "mongodb": "^3.6.6", 23 | "selenium-webdriver": "^4.0.0-beta.2", 24 | "uuid": "^8.3.2", 25 | "yargs": "^16.2.0" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /commands/search/links/instruments.js: -------------------------------------------------------------------------------- 1 | const { By } = require('selenium-webdriver'); 2 | 3 | async function getClassicalStyleLink(param = 1) { 4 | try { 5 | 6 | let styleSelector = By.css('#ulfornav > li > .menuforward li:nth-of-type(' + param + ') > a'); 7 | let styleLink = await driver.findElement(styleSelector).getAttribute("href"); 8 | 9 | await driver.navigate().to(styleLink); 10 | 11 | let classicalSelectorByAnchorTags = await driver.findElements(By.css('#a2 > .full_width_display > .gs2')); 12 | let classicalStyleLink = await classicalSelectorByAnchorTags[2].getAttribute('href'); 13 | return classicalStyleLink; 14 | 15 | } catch (error){ 16 | throw error; 17 | } 18 | }; 19 | 20 | module.exports = getClassicalStyleLink; 21 | 22 | -------------------------------------------------------------------------------- /lib/_command.js: -------------------------------------------------------------------------------- 1 | const { Builder } = require('selenium-webdriver'); 2 | const { ServiceBuilder, setDefaultService } = require('selenium-webdriver/chrome'); 3 | const { path } = require('chromedriver'); 4 | setDefaultService(new ServiceBuilder(path).build()); 5 | 6 | class Command{ 7 | 8 | static init = async function(param = null){ 9 | try { 10 | var driver = new Builder().forBrowser('chrome').build(); 11 | 12 | await driver.get('https://www.8notes.com/'); 13 | 14 | if(param == null){ 15 | global.driver = driver; 16 | if([undefined, null].includes(global.driver)){ 17 | console.log("Web driver başlatılırken bir problem oluştu."); 18 | process.exit(1); 19 | } 20 | } 21 | else 22 | return driver; 23 | 24 | } catch (error) { 25 | throw error; 26 | } 27 | } 28 | 29 | static destroy = function(param = null){ 30 | try { 31 | if(param == null){ 32 | if(![undefined, null].includes(global.driver)) 33 | driver.quit(); 34 | } 35 | else 36 | param.quit(); 37 | } catch (error) { 38 | throw error; 39 | } 40 | 41 | } 42 | 43 | } 44 | 45 | module.exports = Command 46 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk') 2 | const yargs = require('yargs') 3 | const SearchCommand = require('./commands/search') 4 | 5 | yargs.command({ 6 | command: 'search', 7 | describe: 'searching', 8 | builder: { 9 | instrument: { 10 | describe: 'searching area', 11 | demandOption: false, 12 | type: 'string' 13 | }, 14 | style: { 15 | describe: 'searching area', 16 | demandOption: false, 17 | type: 'string' 18 | } 19 | }, 20 | handler: async function (argv) { 21 | try { 22 | if (![undefined, "", null].includes(argv.all)) 23 | await SearchCommand.searchingForAll("all") 24 | else if(![undefined, "", null].includes(argv.allİnstrument) || ![undefined, "", null].includes(argv.allInstrument)) 25 | await SearchCommand.searchingForAll('instrument') 26 | else if (![undefined, "", null].includes(argv.instrument)) 27 | await SearchCommand.searchingByInstrument(argv.instrument) 28 | else if (![undefined, "", null].includes(argv.style)) 29 | await SearchCommand.searchingByStyle(argv.style) 30 | else if (![undefined, "", null].includes(argv.artist)) 31 | await SearchCommand.searchingByArtist(argv.artist) 32 | else 33 | console.log(chalk.red.inverse('There are missing parameters')); 34 | 35 | } catch (error) { 36 | console.log(error); 37 | } 38 | finally { 39 | SearchCommand.destroy(); 40 | } 41 | } 42 | }) 43 | yargs.parse() -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | /outputs 107 | /proje.pdf 108 | /getlinks.js -------------------------------------------------------------------------------- /lib/_db.js: -------------------------------------------------------------------------------- 1 | const { MongoClient } = require('mongodb'); 2 | require('dotenv').config() 3 | 4 | class Database { 5 | 6 | 7 | static #CONNECTION_STRING = `mongodb+srv://root:nodejs123@8notes.nizgg.mongodb.net/8notes?retryWrites=true&w=majority`; 8 | 9 | //static #CONNECTION_STRING = `mongodb+srv://${process.env.DATABASE_USER}:${process.env.DATABASE_PASS}@${process.env.DATABASE_URL}/${process.env.DATABASE_DB}?retryWrites=true&w=majority`; 10 | static client = new MongoClient(this.#CONNECTION_STRING, {useNewUrlParser: true, useUnifiedTopology: true}); 11 | 12 | static connect = async () => { 13 | try { 14 | console.log(process.env); 15 | await this.client.connect() 16 | .then(() => console.log("Connection succesful!")) 17 | .catch((err) => {throw err}); 18 | 19 | return this.client; 20 | 21 | } catch (error) { 22 | throw error; 23 | } 24 | } 25 | 26 | static listDatabases = async (client) => { 27 | var databasesList = await client.db().admin().listDatabases(); 28 | 29 | console.log("Databases:"); 30 | databasesList.databases.forEach(db => console.log(` - ${db.name}`)); 31 | } 32 | 33 | static destroy = async () => { 34 | try { 35 | 36 | this.client.close() 37 | .then(() => console.log("Client closed!")) 38 | .catch((err) => { throw err }); 39 | 40 | } catch (error) { 41 | throw error; 42 | } 43 | } 44 | 45 | static createScrapingsCollection = async () => { 46 | try { 47 | 48 | const db = await this.selectDatabase(); 49 | 50 | await db.listCollections(null, {nameOnly: true}).toArray() 51 | .then(async (collections) => { 52 | var someCollection = collections.some(el => el.name == "scrapings"); 53 | if(!someCollection) 54 | await db.createCollection("scrapings") 55 | .then(() => console.log("Collection created!")) 56 | .catch((err) => { throw err }); 57 | }) 58 | .catch((err) => { throw err }); 59 | 60 | } catch (error) { 61 | throw error; 62 | } 63 | } 64 | 65 | static selectDatabase = async (db = process.env.DATABASE_DB) => { 66 | try { 67 | 68 | const selectedDB = this.client.db(db); 69 | 70 | return selectedDB; 71 | 72 | } catch (error) { 73 | throw error; 74 | } 75 | } 76 | 77 | } 78 | 79 | module.exports = Database; -------------------------------------------------------------------------------- /commands/search/getAttributes.js: -------------------------------------------------------------------------------- 1 | const { workerData, isMainThread, parentPort } = require('worker_threads'); 2 | 3 | const { By } = require('selenium-webdriver'); 4 | const fs = require('fs'); 5 | 6 | const Command = require('../../lib/_command'); 7 | 8 | let queue = []; 9 | let jsonData = { pieces: [] }; 10 | 11 | const main = async function() { 12 | 13 | try { 14 | const driver = await Command.init('new'); 15 | parentPort.on('message', (message) => { 16 | 17 | if (message == 'exit') 18 | parentPort.unref(); 19 | else if (message == 'start') 20 | getAttributes(driver); 21 | else 22 | queue.push(message); 23 | 24 | }); 25 | } catch (error) { 26 | throw error; 27 | } 28 | 29 | } 30 | 31 | const getAttributes = async function(driver){ 32 | var localDriver = driver; 33 | try { 34 | while(queue.length > 0) { 35 | 36 | const array = queue[0]; 37 | for (let j = 0; j < array.length; j++) { 38 | error = false; 39 | await localDriver.navigate().to('https://www.8notes.com' + array[j].link) 40 | .catch(async err => { 41 | await localDriver.quit(); 42 | localDriver = null; 43 | localDriver = await Command.init('new'); 44 | console.log('Error Item: ' + array[j].link); 45 | j -= 1; 46 | error = true; 47 | }); 48 | if (error) 49 | continue; 50 | const img_containers = await localDriver.findElements(By.className('img-container')); 51 | var img_src = []; 52 | for (const img_container of img_containers) { 53 | const img = await img_container.findElement(By.css('img')); 54 | const src = await img.getAttribute('src'); 55 | if(src) 56 | img_src.push(src); 57 | } 58 | 59 | const midi_a_tag = await localDriver.findElement(By.css('#midi_container > .ilistbox > .versionlist > ul > li:nth-of-type(3) > a')); 60 | const midi_href = await midi_a_tag.getAttribute('href'); 61 | 62 | const comp_table = await localDriver.findElement(By.css('#infobox > .comp_table')); 63 | const comp_table_content = await comp_table.getText(); 64 | 65 | array[j] = { 66 | ...array[j], 67 | img_src, 68 | midi_href, 69 | comp_table_content 70 | } 71 | 72 | } 73 | jsonData.pieces.push(...array); 74 | queue.shift(); 75 | const data = JSON.stringify(jsonData, null, '\t'); 76 | 77 | await fs.writeFile(workerData.filename, data, { flag: 'w+' }, (err) => { 78 | if (err) { 79 | throw err; 80 | } 81 | }); 82 | } 83 | 84 | } catch (error) { 85 | throw error; 86 | } 87 | finally { 88 | Command.destroy(driver); 89 | parentPort.postMessage('done'); 90 | } 91 | 92 | } 93 | 94 | if(!isMainThread) 95 | main(); -------------------------------------------------------------------------------- /commands/search/pieces.js: -------------------------------------------------------------------------------- 1 | const { By } = require('selenium-webdriver'); 2 | const fs = require('fs/promises'); 3 | const { v4: uuidv4 } = require('uuid'); 4 | const { Worker } = require('worker_threads') 5 | 6 | const Database = require('../../lib/_db'); 7 | 8 | function runService(workerData) { 9 | 10 | const worker = new Worker(__dirname + '/getAttributes.js', { workerData }); 11 | 12 | worker.on('message', (message) => { 13 | if (message == "done") 14 | worker.postMessage('exit'); 15 | }); 16 | 17 | worker.on("exit", async (message) => { 18 | console.log("İşlem tamamlandı."); 19 | try { 20 | const client = await Database.connect(); 21 | 22 | await Database.createScrapingsCollection(); 23 | 24 | const db = await Database.selectDatabase(); 25 | const filenameSplit = workerData.filename.split('_'); 26 | 27 | await fs.readFile(__dirname + '/../../' + workerData.filename, { encoding:'utf8',flag: 'r' }) 28 | .then(async data => { 29 | await db.collection("scrapings").insertOne({ 30 | [filenameSplit[2]]: JSON.parse(data) 31 | }); 32 | }) 33 | .catch(err => { throw err }); 34 | 35 | } catch (error) { 36 | console.log(error); 37 | } 38 | finally { 39 | await Database.destroy(); 40 | } 41 | }); 42 | return worker; 43 | } 44 | 45 | const getPiecesByLink = async function(links) { 46 | try { 47 | 48 | const filename = 'outputs/pieces_dump_' + (new Date()).getTime() + '_' + uuidv4() + '.json'; 49 | var jsonData = { pieces: [] }; 50 | var data = JSON.stringify(jsonData, null, '\t'); 51 | 52 | await fs.writeFile(__dirname + '/../../' + filename, data, { flag: 'w+' }) 53 | .catch(err => { throw err }); 54 | 55 | const worker = runService({ filename }); 56 | 57 | var startControl = false; 58 | for (const link of links) { 59 | await driver.navigate().to(link); 60 | 61 | let paginationMax = await driver.findElement(By.css('.pagination li:nth-last-child(2)')).getAttribute("textContent"); 62 | paginationMax = parseInt(paginationMax); 63 | 64 | for (let i = 2; i <= paginationMax + 1 ; i++) { 65 | 66 | let tbody = await driver.findElement(By.css('.table-responsive tbody')); 67 | let tr = await tbody.findElements(By.css('tr')); 68 | 69 | let difficultyLevels = { 70 | "Beginners Level": 0, 71 | "Easy Level": 1, 72 | "Intermediate Level": 2, 73 | "Advanced Level": 3 74 | } 75 | 76 | var pieces = []; 77 | for (const el of tr) { 78 | let td = await el.findElements(By.css('td')); 79 | 80 | let link = (await el.getAttribute('onClick')).split('document.location=')[1].replace(/'/g, ''); 81 | let artist = (await td[1].getAttribute('textContent')).trim(); 82 | let title = await td[2].getAttribute('textContent'); 83 | let difficulty = difficultyLevels[await (await td[3].findElement(By.css('img'))).getAttribute('title')]; 84 | 85 | pieces.push({ 86 | link, 87 | artist, 88 | title, 89 | difficulty 90 | }); 91 | 92 | } 93 | 94 | worker.postMessage(pieces) 95 | if(!startControl){ 96 | worker.postMessage('start'); 97 | startControl = true; 98 | } 99 | 100 | if(i != paginationMax + 1){ 101 | var url = new URL(link); 102 | url.searchParams.append('page', i); 103 | 104 | await driver.navigate().to(url.href); 105 | await driver.findElement(By.css('.pagination li:nth-last-child(2)')) 106 | .then(async (el) => { 107 | paginationMax = parseInt(await el.getAttribute("textContent")); 108 | }) 109 | .catch(() => 0); 110 | 111 | } 112 | } 113 | 114 | } 115 | 116 | } catch (error) { 117 | throw error; 118 | } 119 | } 120 | 121 | module.exports = { 122 | getPiecesByLink 123 | } -------------------------------------------------------------------------------- /commands/search/index.js: -------------------------------------------------------------------------------- 1 | const chalk = require('chalk') 2 | 3 | const getClassicalStyleLink = require('./links/instruments'); 4 | const getStyleLink = require('./links/styles'); 5 | const getArtistLink = require('./links/artists'); 6 | 7 | const piece = require('./pieces'); 8 | const Command = require('../../lib/_command'); 9 | 10 | class SearchCommand extends Command { 11 | 12 | static searchingByInstrument = async function(title) { 13 | try { 14 | const instruments = ["piano", 15 | "guitar", 16 | "violin", 17 | "flute", 18 | "saxophone", 19 | "voice", 20 | "clarinet", 21 | "trumpet" 22 | ]; 23 | if(title === "all"){ 24 | var links = []; 25 | for (let i = 0; i < instruments.length; i++) { 26 | const classicalStyleLink = await getClassicalStyleLink(i+1); 27 | links.push(classicalStyleLink); 28 | } 29 | return links; 30 | }else{ 31 | var i = instruments.indexOf(title.toLowerCase()); 32 | if(i == -1) 33 | console.log(chalk.red.inverse('No result for this instrument')); 34 | else{ 35 | await SearchCommand.init(); 36 | const classicalStyleLink = await getClassicalStyleLink(i+1); 37 | await piece.getPiecesByLink([classicalStyleLink]); 38 | } 39 | } 40 | 41 | } catch (error) { 42 | throw error; 43 | } 44 | 45 | } 46 | 47 | static searchingByStyle = async function(title) { 48 | try { 49 | const styles = { 50 | 1: "classical", 51 | 4: "rock and pop", 52 | 5: "christmas" 53 | }; 54 | var control = false; 55 | 56 | if(title === "all"){ 57 | var links = []; 58 | for (const [key, value] of Object.entries(styles)){ 59 | const styleLink = await getStyleLink(key); 60 | links.push(styleLink); 61 | } 62 | return links; 63 | }else 64 | for (const [key, value] of Object.entries(styles)){ 65 | if (title.toLowerCase() == value){ 66 | await SearchCommand.init(); 67 | const styleLink = await getStyleLink(key); 68 | await piece.getPiecesByLink([styleLink]); 69 | control = true; 70 | return 0; 71 | } 72 | } 73 | 74 | if(!control) 75 | console.log(chalk.red.inverse('No result for this style')); 76 | 77 | } catch (error) { 78 | throw error; 79 | } 80 | } 81 | 82 | static searchingByArtist = async function(title) { 83 | try { 84 | const artists = ["bach", 85 | "beethoven", 86 | "mozart", 87 | "tchaikovsky", 88 | "scott joplin", 89 | "chopin" 90 | ]; 91 | if(title === "all"){ 92 | var links = []; 93 | for (let i = 0; i < artists.length; i++) { 94 | const artistLink = await getArtistLink(i+1); 95 | links.push(artistLink); 96 | } 97 | return links; 98 | }else{ 99 | var i = artists.indexOf(title.toLowerCase()); 100 | if(i == -1) 101 | console.log(chalk.red.inverse('No result for this artist')); 102 | else{ 103 | await SearchCommand.init(); 104 | const artistLink = await getArtistLink(i+1); 105 | await piece.getPiecesByLink([artistLink]); 106 | } 107 | } 108 | 109 | } catch (error) { 110 | throw error; 111 | } 112 | } 113 | 114 | static searchingForAll = async function(command = "all") { 115 | try { 116 | 117 | await SearchCommand.init(); 118 | var allLinks = []; 119 | 120 | if(command == "all"){ 121 | const styleLinks = await SearchCommand.searchingByStyle("all"); 122 | const artistLinks = await SearchCommand.searchingByArtist("all"); 123 | const classicalStyleLinks = await SearchCommand.searchingByInstrument("all"); 124 | 125 | allLinks = artistLinks.concat(styleLinks, classicalStyleLinks); 126 | }else if(command == "instrument") 127 | allLinks = await SearchCommand.searchingByInstrument("all"); 128 | else if(command == "style") 129 | allLinks = await SearchCommand.searchingByStyle("all"); 130 | else if(command == "artist") 131 | allLinks = await SearchCommand.searchingByArtist("all"); 132 | else{ 133 | console.log(chalk.red.inverse('Wrong parameter.')); 134 | return 0; 135 | } 136 | 137 | await piece.getPiecesByLink(allLinks); 138 | 139 | } catch (error) { 140 | throw error; 141 | } 142 | 143 | } 144 | } 145 | 146 | module.exports = SearchCommand 147 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Scraping 8notes web site with NodeJS

2 |

3 | A scraping tool by instrument (classic), style or artist on 8notes website developed with NodeJS 4 | 5 |

6 | 7 | - [EN description](#en) 8 | - [TR açıklama](#tr) 9 | 10 | # DEPENDENCIES/GEREKSİNİMLER 11 | 12 | - NodeJS ^14.15.1 13 | - npm ^6.14.8 14 | 15 | # EN 16 | 17 | # DESCRIPTION 18 | 19 | A tool that allows you to capture music from the 8notes site. While the tool is shooting the music asynchronously, it can pull all the information of the music it pulls in a child thread. 20 | 21 |  According to classical instruments by following 22 | 23 | + classical piano 24 | + classical guitar 25 | + classical violin 26 | + classical flute 27 | + classical saxophone 28 | + classical voice 29 | + classical clarinet 30 | + classical trumpet 31 | 32 |  or according to the following music styles 33 | 34 | + classical style 35 | + Rock and pop styles 36 | + christmas style 37 | 38 |   or by the following artists 39 | 40 | + classical Bach 41 | + classical Beethoven 42 | + classical Mozart 43 | + classical Tchaikovsky 44 | + classical Scott Joplin 45 | + classical Chopin 46 | 47 | a scraping tool that searches on the [8notes](https://www.8notes.com/) web site. 48 | 49 | # INSTALL AND USAGE 50 | 51 | ## UBUNTU 20.04 için NodeJS 52 | 53 | ### NODEJS KURULUMU 54 | 55 | SIGN IN AS ROOT USER WHEN INSTALLING NODEJS 56 | 57 | Run the following command to install NVM(Node Version Manager). 58 | 59 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash 60 | 61 | After NVM is successfully installed, run the following command or restart your machine to use NVM. 62 | 63 | source ~/.bashrc 64 | 65 | or 66 | 67 | reboot 68 | 69 | You can run the command below to find the NodeJS version to install. (If you know your version, you don't need to run the command.) 70 | 71 | nvm ls-remote 72 | 73 | After you find your NodeJS version, you can install it on your server with the following command. (V14.15.1 is recommended) 74 | 75 | nvm install 76 | 77 | After NodeJS is installed, you can test it with the ```node``` command. 78 | 79 | When NodeJS is installed for root, run the script below to make it available to other users. 80 | 81 | n=$(which node);n=${n%/bin/node}; chmod -R 755 $n/bin/*; sudo cp -r $n/{bin,lib,share} /usr/local 82 | 83 | ## DEPENDENCY INSTALLING AND USAGE 84 | 85 | ## Chromedriver 86 | 87 | By default, version 90.0.0 comes ready in Package json. If you have a different version of Chrome, you need to update this value. 88 | 89 | 1. Open your computer's command line interface. 90 | 2. Change directory to the directory where the repository is located. 91 | 3. To install the packages, first run the following command. 92 | 93 | npm i 94 | 95 | 4. To browse the site; 96 | 97 | - Run the following command to browse by instruments. 98 | 99 | node index.js search --style="" 100 | 101 | - Run the following command to browse by style. 102 | 103 | node index.js search --style="" 104 | 105 | - Run the following command to browse by artist. 106 | 107 | node index.js search --style="" 108 | 109 | - Run the following command browse all music 110 | 111 | npm run-script stage-1 112 | 113 | - To pull in depth information of asynchronously scanned music while scanning all music 114 | 115 | npm run-script stage-2 116 | 117 | 5. Processes may take a little longer depending on your internet speed, your computer's characteristics and the speed of the site's response to requests. 118 | 6. The results are given as json type output under the outputs folder. 119 | 120 | 121 | # TR 122 | 123 | # AÇIKLAMA 124 | 125 | 8notes sitesinden müzikleri taramanızı sağlayan bir araç. Araç, müziği eşzamansız olarak çekerken, çektiği müziğin tüm bilgilerini bir child thread içinde çekebilir. 126 | 127 |  Araca Komut satırından verilen argümanlara bakarak classical instrumentlara göre: 128 | 129 | + classical piano 130 | + classical guitar 131 | + classical violin 132 | + classical flute 133 | + classical saxophone 134 | + classical voice 135 | + classical clarinet 136 | + classical trumpet 137 | 138 |  veya komut satırından verilen argümanlara bakarak music stylelerine göre: 139 | 140 | + classical style 141 | + Rock and pop styles 142 | + christmas style 143 | 144 |   veya komut satırından verilen argümanlara bakarak artistlere göre: 145 | 146 | + classical Bach 147 | + classical Beethoven 148 | + classical Mozart 149 | + classical Tchaikovsky 150 | + classical Scott Joplin 151 | + classical Chopin 152 | 153 | arama yapan bir scraping aracı. 154 | 155 | # KURULUM VE KULLANIM 156 | 157 | ## UBUNTU 20.04 için NodeJS 158 | 159 | ### NODEJS KURULUMU 160 | NODEJS KURARKEN ROOT KULLANICISI OLARAK OTURUM AÇIN 161 | 162 | NVM(Node Version Manager) kurulumu için çalıştırın. 163 | 164 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash 165 | 166 | NVM başarıyla kurulduktan sonra NVM'i kullanabilmek için aşağıdaki komutu çalıştırın ya da sunucunuzu yeniden başlatın. 167 | 168 | source ~/.bashrc 169 | 170 | veya 171 | 172 | reboot 173 | 174 | Daha sonra kuracağınız NodeJS sürümünü bulmak için aşağıdaki komutu çalıştırabilirsiniz.(Eğer sürümünüzü biliyorsanız komutu çalıştırmanıza gerek yok.) 175 | 176 | nvm ls-remote 177 | 178 | Nodejs sürümünüzü bulduktan sonra aşağıdaki komut ile sunucunuza kurabilirsiniz.(V14.15.1 tavsiye edilir) 179 | 180 | nvm install 181 | 182 | NodeJS kurulduktan sonra ```node``` komutu ile test edebilirsiniz. 183 | 184 | NodeJS root için kurulduğunda diğer kullanıcıların da kullanabilmesini sağlamak için aşağıdaki betiği çalıştırın. 185 | 186 | n=$(which node);n=${n%/bin/node}; chmod -R 755 $n/bin/*; sudo cp -r $n/{bin,lib,share} /usr/local 187 | 188 | ## ARAÇ KURULUMU ve KULLANIMI 189 | 190 | ## Chromedriver 191 | 192 | Package json içerisinde varsayılan olarak 90.0.0 versiyonu hazır olarak gelmektedir. Eğer ki farklı bir Chrome sürümünüz mevcutsa bu değeri güncellemeniz gerekmektedir. 193 | 194 | 1. Bilgisayarınızın komut satırı arayüzünü açın. 195 | 2. Dizin değiştirerek deponun bulunduğu dizinin içine gelin. 196 | 3. Paketleri kurmak için öncelikle aşağıdaki komutu çalıştırın. 197 | 198 | npm i 199 | 200 | 4. Site üzerinde tarama yapmak için; 201 | 202 | - Enstrümanlara göre tarama yapmak için aşağıdaki komutu çalıştırın. 203 | 204 | node index.js search --style="" 205 | 206 | - Stile göre tarama yapmak için aşağıdaki komutu çalıştırın. 207 | 208 | node index.js search --style="" 209 | 210 | - Sanatçıya göre tarama yapmak için aşağıdaki komutu çalıştırın. 211 | 212 | node index.js search --style="" 213 | 214 | - Tüm müzikleri taramak için aşağıdaki komutu çalıştırın. 215 | 216 | npm run-script stage-1 217 | 218 | - Müzikler bir yandan taranırken, taranmış müziklerin bilgilerini asenkron bir şekilde derinlemesine çekmek için aşağıdaki komutu çalıştırın. 219 | 220 | npm run-script stage-2 221 | 222 | 223 | 5. İşlemler internet hızınıza, bilgisayarınızın özelliklerine ve sitenin isteklere yanıt verme hızına göre biraz uzun sürebilir. 224 | 225 | 6. Sonuçlar outputs klasörü altında json türünde çıktı olarak verilmektedir. --------------------------------------------------------------------------------