├── static └── .gitkeep ├── anyaudio ├── helpers │ ├── __init__.py │ ├── redis_utils.py │ ├── encryption.py │ ├── pafymodule.py │ ├── networking.py │ ├── trending.py │ ├── data.py │ ├── database.py │ ├── helpers.py │ └── search.py ├── templates │ ├── robots.txt │ ├── unauthorized.html │ ├── app.html │ ├── home.html │ ├── lite_song.html │ ├── lite_search.html │ ├── log_page.html │ ├── explore.html │ ├── terms-of-use.html │ └── index.html ├── static │ ├── img │ │ ├── bg_main.jpg │ │ └── favicon.ico │ ├── style.css │ ├── main.js │ ├── js │ │ └── main_new.js │ └── css │ │ └── style.css ├── views │ ├── __init__.py │ ├── api_v2.py │ ├── generic.py │ └── api_v1.py ├── schedulers │ ├── youtube_dl_upgrade.py │ ├── __init__.py │ └── trending.py └── __init__.py ├── .env ├── .dockerignore ├── .gitignore ├── requirements.txt ├── Makefile ├── AUTHORS.md ├── scripts ├── set_ffmpeg.sh ├── run_ec2.sh └── ec2_hook.sh ├── run.sh ├── Pipfile ├── .editorconfig ├── docs ├── OPENSHIFT.md ├── DOCKER.md ├── EC2.md └── api │ └── v1 │ └── API-v1.md ├── Dockerfile ├── docker-compose.yml ├── tests ├── test_trending.py ├── test_search.py ├── test_get_url.py ├── __init__.py ├── test_stream.py └── test_download.py ├── app.py ├── README.md ├── .openshift └── action_hooks │ └── build ├── .travis.yml └── Pipfile.lock /static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /anyaudio/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=anyaudio 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .dockerignore 3 | .openshift 4 | -------------------------------------------------------------------------------- /anyaudio/templates/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Crawl-delay: 10 3 | Disallow: / 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-workspace 2 | *.pyc 3 | *.m4a 4 | *.mp3 5 | *.db 6 | .idea/ 7 | .DS_Store 8 | -------------------------------------------------------------------------------- /anyaudio/static/img/bg_main.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anyaudio/anyaudio-server/HEAD/anyaudio/static/img/bg_main.jpg -------------------------------------------------------------------------------- /anyaudio/static/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/anyaudio/anyaudio-server/HEAD/anyaudio/static/img/favicon.ico -------------------------------------------------------------------------------- /anyaudio/views/__init__.py: -------------------------------------------------------------------------------- 1 | import anyaudio.views.generic 2 | import anyaudio.views.api_v1 3 | import anyaudio.views.api_v2 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | youtube-dl 3 | requests 4 | gunicorn 5 | eventlet 6 | greenlet 7 | psycopg2 8 | mutagen 9 | pafy 10 | flask-cors 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | run: 2 | bash run.sh 3 | 4 | install: 5 | pip install -r requirements.txt 6 | 7 | test: 8 | # assumes you have FFMPEG installed (in PATH) 9 | $(eval export FFMPEG_PATH=ffmpeg) 10 | $(eval export OPENSHIFT_PYTHON_IP=127.0.0.1) 11 | python -m unittest discover tests 12 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | ## List of Authors for AnyAudio 2 | 3 | | Name | Github | EMail | 4 | |:----:|:------:|:-----:| 5 | | AVI ARYAN | [@aviaryan](https://github.com/aviaryan) | aviaryan123@gmail.com | 6 | | PRATYUSH SINGH | [@singhpratyush](https://github.com/singhpratyush) | singh.pratyush96@gmail.com| 7 | -------------------------------------------------------------------------------- /scripts/set_ffmpeg.sh: -------------------------------------------------------------------------------- 1 | # This script downloads and sets up ffmpeg in directory ffmpeg 2 | # http://johnvansickle.com/ffmpeg/ 3 | wget -O ffmpeg.tar.xz http://johnvansickle.com/ffmpeg/builds/ffmpeg-git-64bit-static.tar.xz 4 | mkdir ffmpeg 5 | tar -xf ffmpeg.tar.xz -C ffmpeg --strip-components 1 6 | rm ffmpeg.tar.xz 7 | cd ffmpeg 8 | find . ! -iname ffmpeg -delete 9 | -------------------------------------------------------------------------------- /scripts/run_ec2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export OPENSHIFT_PYTHON_IP=0.0.0.0 4 | export OPENSHIFT_PYTHON_PORT=5000 5 | export FFMPEG_PATH=ffmpeg/ffmpeg 6 | export OPENSHIFT_POSTGRESQL_DB_HOST=0.0.0.0 7 | export OPENSHIFT_POSTGRESQL_DB_PORT=5432 8 | export OPENSHIFT_POSTGRESQL_DB_USERNAME=aa 9 | export OPENSHIFT_POSTGRESQL_DB_PASSWORD=aa 10 | export POSTGRESQL_DB_NAME=anyaudio 11 | 12 | /home/ubuntu/miniconda2/bin/python app.py 13 | -------------------------------------------------------------------------------- /anyaudio/templates/unauthorized.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Unauthorized 7 | 8 | 9 |
10 |
11 |

You are not authorized to take this action.

12 |
13 |
14 | 15 | 16 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | export FFMPEG_PATH=ffmpeg; 2 | export OPENSHIFT_PYTHON_IP=0.0.0.0; 3 | # export OPENSHIFT_PYTHON_PORT=80; 4 | 5 | # PSQL env vars 6 | export OPENSHIFT_POSTGRESQL_DB_HOST=0.0.0.0; 7 | export OPENSHIFT_POSTGRESQL_DB_PORT=5432; 8 | export OPENSHIFT_POSTGRESQL_DB_USERNAME=ymp3; 9 | export OPENSHIFT_POSTGRESQL_DB_PASSWORD=ymp3; 10 | export POSTGRESQL_DB_NAME=ymp3; 11 | 12 | # Dev configs 13 | export PLAYLIST_VIDEOS_LIMIT=40; 14 | # export PLAYLIST_LIST_LIMIT=5; 15 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | requests = "*" 10 | gunicorn = "*" 11 | eventlet = "*" 12 | greenlet = "*" 13 | "psycopg2" = "*" 14 | mutagen = "*" 15 | pafy = "*" 16 | Flask = "*" 17 | Flask-Cors = "*" 18 | request = "*" 19 | youtube_dl = "*" 20 | htmlparser = "*" 21 | redis = "*" 22 | 23 | [requires] 24 | python_version = "3.5" 25 | 26 | [scripts] 27 | app = "python app.py" 28 | -------------------------------------------------------------------------------- /anyaudio/schedulers/youtube_dl_upgrade.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from anyaudio import logger 4 | from . import Scheduler 5 | 6 | 7 | class YoutubeDLUpgrader(Scheduler): 8 | 9 | def __init__(self, name=' python-2.7 10 | ``` 11 | 12 | * Add Postgresql cartridge and set the database name environment variable. 13 | 14 | ```sh 15 | rhc cartridge add postgresql-9.2 -a 16 | rhc env set POSTGRESQL_DB_NAME= -a 17 | ``` 18 | 19 | * Push this repo to openshift server. 20 | 21 | * Done 22 | -------------------------------------------------------------------------------- /anyaudio/helpers/redis_utils.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | from anyaudio import logger 4 | 5 | redis_client = redis.StrictRedis() 6 | 7 | def get_or_create_video_download_link(video_id, format, callback): 8 | key = 'video:download:%s:%s' % (video_id, format) 9 | download_url = redis_client.get(key) 10 | if not download_url: 11 | logger.info('[Redis] cache miss for %s' % key) 12 | download_url = callback(video_id, format) 13 | redis_client.set(key, download_url, ex=60 * 60 * 6) # Expires in 6 hours 14 | else: 15 | logger.info('[Redis] cache hit for %s' % key) 16 | download_url = download_url.decode('utf-8') 17 | return download_url 18 | -------------------------------------------------------------------------------- /anyaudio/schedulers/__init__.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | from time import sleep 3 | from traceback import print_exc 4 | 5 | 6 | class Scheduler(): 7 | 8 | def __init__(self, name, period): 9 | self.name = name 10 | self.period = period 11 | 12 | def __str__(self): 13 | return self.name 14 | 15 | def start(self): 16 | worker = Thread(target=self.run_repeater) 17 | worker.daemon = True 18 | worker.start() 19 | return worker 20 | 21 | def run_repeater(self): 22 | while True: 23 | try: 24 | self.run() 25 | sleep(self.period) 26 | except Exception: 27 | print_exc() 28 | 29 | def run(self): 30 | raise NotImplementedError() 31 | -------------------------------------------------------------------------------- /anyaudio/templates/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | AnyAudio - Android App Download 6 | 7 | 8 | Please wait while we process your download... 9 | 10 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /anyaudio/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import Flask 3 | from os import environ 4 | from flask_cors import CORS 5 | 6 | 7 | app = Flask(__name__) 8 | CORS(app, resources={r"/api/*": {"origins": "*"}}) 9 | app.config['DEBUG'] = True 10 | 11 | DATABASE_PATH = 'SQLite.db' 12 | LOCAL = True 13 | if environ.get('OPENSHIFT_PYTHON_IP'): 14 | LOCAL = False 15 | 16 | # Logger 17 | logger = logging.getLogger('anyaudio-server') 18 | handler = logging.StreamHandler() 19 | handler.setFormatter(logging.Formatter( 20 | '%(relativeCreated)6d %(threadName)s %(message)s' 21 | )) 22 | logger.addHandler(handler) 23 | logger.setLevel(logging.INFO) 24 | 25 | 26 | @app.after_request 27 | def after_request(response): 28 | response.headers.add('Accept-Ranges', 'bytes') 29 | return response 30 | 31 | 32 | import anyaudio.views 33 | -------------------------------------------------------------------------------- /scripts/ec2_hook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PATH="/home/ubuntu/miniconda2/bin:$PATH" 3 | 4 | while read oldrev newrev ref 5 | do 6 | branch=`echo $ref | cut -d/ -f3` 7 | if [ "ec2" == "$branch" -o "master" == "$branch" ]; then 8 | git --work-tree=/home/ubuntu/app/ checkout -f $branch 9 | echo 'Changes pushed to Amazon EC2 PROD.' 10 | cd /home/ubuntu/app 11 | pip install --upgrade -r requirements.txt --no-cache-dir 12 | echo 'Python requirements upgraded' 13 | pkill -f gunicorn || true 14 | echo 'Killed old instance' 15 | # install ffmpeg (download at deploy because always use latest version) 16 | # bash scripts/set_ffmpeg.sh 17 | echo 'upgraded ffmpeg' 18 | # http://stackoverflow.com/questions/7917324/git-post-commit-hook-as-a-background-task 19 | nohup ./scripts/run_ec2.sh &>/dev/null & 20 | echo 'Server is live' 21 | fi 22 | done 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2-alpine 2 | MAINTAINER Avi Aryan 3 | 4 | ENV INSTALL_PATH /anyaudio 5 | RUN mkdir -p $INSTALL_PATH 6 | 7 | WORKDIR $INSTALL_PATH 8 | 9 | # update needed for wget 10 | # update tar .. --strip-componenets not available in current version 11 | # install permanent deps 12 | RUN apk --update add --no-cache ca-certificates wget tar xz postgresql-dev 13 | RUN update-ca-certificates 14 | 15 | COPY requirements.txt requirements.txt 16 | 17 | # install deps 18 | RUN apk update 19 | RUN apk add --no-cache --virtual build-dependencies gcc python-dev libevent-dev linux-headers musl-dev \ 20 | && pip install -r requirements.txt \ 21 | && apk del build-dependencies 22 | 23 | # install ffmpeg 24 | COPY scripts/set_ffmpeg.sh scripts/set_ffmpeg.sh 25 | RUN ash scripts/set_ffmpeg.sh 26 | 27 | # copy remaining files 28 | COPY . . 29 | 30 | CMD python app.py 31 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | data: 5 | image: postgres:9-alpine 6 | environment: 7 | POSTGRES_PASSWORD: 'test' 8 | volumes: 9 | - /var/lib/postgresql 10 | command: echo true 11 | 12 | postgres: 13 | image: postgres:9-alpine 14 | environment: 15 | POSTGRES_PASSWORD: 'test' 16 | volumes_from: 17 | - data 18 | ports: 19 | - '5432:5432' 20 | 21 | web: 22 | build: . 23 | environment: 24 | OPENSHIFT_PYTHON_IP: '0.0.0.0' 25 | OPENSHIFT_PYTHON_PORT: '5000' 26 | FFMPEG_PATH: 'ffmpeg/ffmpeg' 27 | OPENSHIFT_POSTGRESQL_DB_HOST: postgres 28 | OPENSHIFT_POSTGRESQL_DB_PORT: '5432' 29 | OPENSHIFT_POSTGRESQL_DB_USERNAME: postgres 30 | OPENSHIFT_POSTGRESQL_DB_PASSWORD: test 31 | POSTGRESQL_DB_NAME: anyaudio 32 | links: 33 | - postgres:postgres 34 | ports: 35 | - '80:5000' 36 | -------------------------------------------------------------------------------- /docs/DOCKER.md: -------------------------------------------------------------------------------- 1 | ## Docker Deployment 2 | 3 | Follow these steps to have YoutubeMP3 running inside a Docker container. 4 | This tutorial assumes you have Docker and docker-compose installed. 5 | 6 | * Clone the repo and cd into it 7 | 8 | ```sh 9 | $ git clone https://github.com/aviaryan/youtube-mp3-server.git && cd youtube-mp3-server 10 | ``` 11 | 12 | * Build the image 13 | 14 | ```sh 15 | $ docker-compose build 16 | ``` 17 | 18 | * Run the app 19 | 20 | ```sh 21 | $ docker-compose up 22 | ``` 23 | 24 | * Open a new shell and run the following command. 25 | 26 | ```sh 27 | docker-compose run postgres psql -h postgres -p 5432 -U postgres --password 28 | # enter password as test 29 | ``` 30 | 31 | * When in psql shell, create the database and then exit using `\q`. 32 | 33 | ```sql 34 | create database anyaudio; 35 | ``` 36 | 37 | * Close the server and then start it again. Then navigate to `http://localhost` to view the app. 38 | -------------------------------------------------------------------------------- /anyaudio/helpers/encryption.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import json 4 | 5 | 6 | def encode(key, clear): 7 | st = '' 8 | incr = get_key_hash(key) 9 | for _ in clear: 10 | st += chr(incr + ord(_)) 11 | return base64.urlsafe_b64encode(st.encode('utf-8')) 12 | 13 | 14 | def decode(key, enc): 15 | st = '' 16 | enc = enc.replace('-', '+').replace('_', '/') 17 | enc = base64.b64decode(enc) # dont know why urlsafe decode doesn't work 18 | incr = get_key_hash(key) 19 | for _ in enc: 20 | st += chr(_ - incr) 21 | return st 22 | 23 | 24 | def get_key(): 25 | return os.environ.get('SECRET_KEY', 'default key') 26 | 27 | 28 | def get_key_hash(key): 29 | c = 0 30 | for _ in key: 31 | c += ord(_) 32 | return c % 20 33 | 34 | 35 | def encode_data(key, **kwargs): 36 | data = json.dumps(kwargs) 37 | return encode(key, data) 38 | 39 | 40 | def decode_data(key, data): 41 | dec = decode(key, data) 42 | return json.loads(dec) 43 | -------------------------------------------------------------------------------- /tests/test_trending.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import json 3 | from tests import YMP3TestCase 4 | from anyaudio.schedulers import trending 5 | from anyaudio.helpers.data import trending_playlist 6 | 7 | 8 | class TestTrending(YMP3TestCase): 9 | """ 10 | Test Trending 11 | """ 12 | def test_trending_worker_run(self): 13 | """run trending worker and see if it goes well""" 14 | scheduler = trending.TrendingScheduler() 15 | scheduler.run_repeater = scheduler.run 16 | worker = scheduler.start() 17 | worker.join() 18 | # ^^ wait for above to finish 19 | for playlist in trending_playlist: 20 | resp = self.app.get('/api/v1/trending?type=%s' % playlist[0]) 21 | self.assertEqual(resp.status_code, 200, playlist[0]) 22 | self.assertIn('stream_url', resp.data) 23 | data = json.loads(resp.data) 24 | self.assertNotEqual(data['metadata']['count'], 0, playlist[0]) 25 | 26 | 27 | if __name__ == '__main__': 28 | unittest.main() 29 | -------------------------------------------------------------------------------- /tests/test_search.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import json 3 | from tests import YMP3TestCase 4 | 5 | 6 | class TestSearch(YMP3TestCase): 7 | """ 8 | Test Search feature 9 | """ 10 | def successful_search_test(self, version): 11 | """test a successful search of a music video""" 12 | result = self._search('Numb', version=version) 13 | self.assertIn('Numb', result) 14 | self.assertIn('Linkin Park', result) 15 | data = json.loads(result) 16 | self.assertEqual(len(data['results']), data['metadata']['count']) 17 | self.assertTrue(len(data['results']) >= 10, result) 18 | 19 | def test_successful_search_v1(self): 20 | self.successful_search_test('v1') 21 | 22 | def test_successful_search_v2(self): 23 | self.successful_search_test('v2') 24 | 25 | def test_long_video_not_returned(self): 26 | """test search of a term which usually will have long videos as results""" 27 | resp = self.app.get('/api/v1/search?q=mashup nonstop') 28 | data = json.loads(resp.data) 29 | self.assertTrue(len(data['results']) < 13) 30 | 31 | 32 | if __name__ == '__main__': 33 | unittest.main() 34 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | from subprocess import call 3 | 4 | from anyaudio.helpers.database import init_databases 5 | from anyaudio.schedulers import trending, youtube_dl_upgrade 6 | 7 | 8 | if __name__ == '__main__': 9 | 10 | # Create SQLite tables 11 | init_databases() 12 | 13 | # Start schedulers 14 | trending_scheduler = trending.TrendingScheduler() 15 | trending_scheduler.start() 16 | youtube_dl_upgrade_scheduler = youtube_dl_upgrade.YoutubeDLUpgrader() 17 | youtube_dl_upgrade_scheduler.start() 18 | 19 | call(['bash', 'run.sh']) 20 | 21 | # http://docs.gunicorn.org/en/stable/settings.html 22 | cmd = 'gunicorn anyaudio:app -w 4' 23 | # Comment following line on CentOS due to bug in eventlet 24 | cmd += ' --worker-class eventlet' 25 | cmd += ' --reload' 26 | cmd += ' --log-level info' 27 | cmd += ' -b %s:%s' % ( 28 | environ.get('OPENSHIFT_PYTHON_IP', '127.0.0.1'), 29 | environ.get('OPENSHIFT_PYTHON_PORT', '5000') 30 | ) 31 | cmd += ' --worker-connections 1000 -t 150' # 4 mins = 10 secs 32 | call(cmd.split()) 33 | # app.run( 34 | # host=environ.get('OPENSHIFT_PYTHON_IP', '127.0.0.1'), 35 | # port=int(environ.get('OPENSHIFT_PYTHON_PORT', 5000)) 36 | # ) 37 | -------------------------------------------------------------------------------- /tests/test_get_url.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from tests import YMP3TestCase 3 | 4 | 5 | class TestGetUrl(YMP3TestCase): 6 | """ 7 | Test Get download url 8 | """ 9 | def test_good_get_url(self): 10 | """test a successful search of a music video""" 11 | result = self._search('Love Story', just_results=True) 12 | get_url = result[0]['get_url'] 13 | resp = self.app.get(get_url) 14 | self.assertEqual(resp.status_code, 200) 15 | self.assertIn('url', resp.data) 16 | self.assertIn('/d?', resp.data) 17 | 18 | def test_fake_get_url(self): 19 | """test the 5xx response in case of fake get url""" 20 | resp = self.app.get('/api/v1/g?url=somefalseurl') 21 | self.assertEqual(resp.status_code, 500) 22 | 23 | def test_get_url_b64_padding_issue(self): 24 | """ 25 | test b64 decoding of url which used to throw IncorrectPadding error 26 | PS - Didn't had another sample with the same behavior 27 | """ 28 | result = self._search('Hot Desi Naukrani Ke Sath Romance', just_results=True) 29 | get_url = result[0]['get_url'] 30 | resp = self.app.get(get_url) 31 | self.assertEqual(resp.status_code, 200) 32 | self.assertIn('url', resp.data) 33 | 34 | 35 | if __name__ == '__main__': 36 | unittest.main() 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AnyAudio 2 | 3 | Download any song that this world ever heard, and that too in your favorite format MP3. 4 | 5 | A rich public API is also included. 6 | 7 | [![Join the chat at https://gitter.im/Any-Audio/anyaudio-server](https://badges.gitter.im/iiitv/algos.svg)](https://gitter.im/Any-Audio/anyaudio-server?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 8 | [![Build Status](https://travis-ci.org/anyaudio/anyaudio-server.svg?branch=master)](https://travis-ci.org/anyaudio/anyaudio-server) 9 | 10 | [Android App](https://github.com/bxute/musicgenie) 11 | 12 | [![Launch on OpenShift](http://launch-shifter.rhcloud.com/button.svg)](https://openshift.redhat.com/app/console/application_type/custom?cartridges%5B%5D=python-2.7&initial_git_url=https%3A%2F%2Fgithub.com%2Faviaryan%2Fyoutube%2Dmp3%2Dserver.git&name=youtube%2Dmp3%2Dserver) 13 | 14 | ## Running 15 | 16 | ``` 17 | pipenv run app 18 | ``` 19 | 20 | ## API 21 | 22 | See [API v1 documentation](docs/api/v1/API-v1.md) 23 | 24 | 25 | ## Deployment on Openshift instructions 26 | 27 | See [docs/OPENSHIFT.md](docs/OPENSHIFT.md) 28 | 29 | 30 | ## Deployment using Docker instructions 31 | 32 | See [docs/DOCKER.md](docs/DOCKER.md) 33 | 34 | ## External Dependencies 35 | * `ffmpeg` 36 | 37 | Make sure that you have `ffmpeg` and its path is set properly in `run.sh` 38 | 39 | ## Running tests 40 | 41 | ```bash 42 | make test 43 | # or 44 | # python -m unittest discover tests 45 | ``` 46 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import json 3 | # import os 4 | from anyaudio import app 5 | from anyaudio.helpers.database import init_databases 6 | 7 | init_databases() 8 | 9 | 10 | class YMP3TestCase(unittest.TestCase): 11 | def setUp(self): 12 | app.config['TESTING'] = True 13 | # DATABASE_PATH = 'test.db' 14 | self.app = app.test_client() 15 | 16 | def _search(self, term, just_results=False, version='v1'): 17 | """ 18 | searches and returns the result 19 | :just_results - If true, return results dict 20 | """ 21 | resp = self.app.get('/api/' + version + '/search?q=%s' % term) 22 | self.assertEqual(resp.status_code, 200) 23 | if just_results: 24 | return json.loads(resp.data)['results'] 25 | else: 26 | return resp.data 27 | 28 | def _search_v2(self, term, just_results=False): 29 | return self._search(term, just_results=just_results, version='v2') 30 | 31 | def _get_dl_link(self, url, just_url=False): 32 | """ 33 | from get download url, get the download url 34 | """ 35 | resp = self.app.get(url) 36 | self.assertEqual(resp.status_code, 200) 37 | if just_url: 38 | return json.loads(resp.data)['url'] 39 | else: 40 | return resp.data 41 | 42 | def tearDown(self): 43 | pass 44 | # if os.path.isfile('test.db'): 45 | # os.unlink('test.db') 46 | -------------------------------------------------------------------------------- /.openshift/action_hooks/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Written by Priyend Somaroo, 06 Jun 2016, Vardaan Enterpises, www.vardaan.com 4 | # 5 | # This will execute pip install but using no caching in order to fix broken 6 | # cache problems with python-2.7 cartridge as at 06 Jun 2016 7 | # 8 | # Ref: http://stackoverflow.com/questions/29913677/openshift-app-with-flask-sqlalchemy-and-sqlite-problems-with-database-reverti 9 | # Ref: http://stackoverflow.com/questions/21691202/how-to-create-file-execute-mode-permissions-in-git-on-windows 10 | # 11 | # 12 | # *** Very important *** : this file must be marked executable using 'chmod +x'. 13 | # In Windows this is a problem so we simply mark it in the git repo as executable as follows. 14 | # - In Windows open command prompt and change to the folder with this build file 15 | # - Then run 'git update-index --chmod=+x build' 16 | # - Ten check the permissions aer 0755 using 'git ls-files --stage' . 17 | # 18 | # Normal commits and push's occur after that. 19 | 20 | # This build hook gets executed at the end of the build cycle before delpoy 21 | 22 | # Change to repo directory and run pip install with no caching 23 | cd ${OPENSHIFT_REPO_DIR} 24 | pip install --upgrade -r requirements.txt --no-cache-dir 25 | 26 | # install ffmpeg (download at deploy because always use latest version) 27 | bash scripts/set_ffmpeg.sh 28 | 29 | # kill old processes (ps aux). For some reason, they are still hanging and don't 30 | # allow new deployment to take place 31 | # pkill fails if nothing killed 32 | pkill -f gunicorn || true 33 | -------------------------------------------------------------------------------- /anyaudio/static/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | background-color: #fdfdfd; 4 | } 5 | 6 | .header { 7 | padding: 1em 0 1em 2em; 8 | background-color: #dddddd; 9 | margin-bottom: 1em; 10 | } 11 | 12 | #container { 13 | max-width: 800px; 14 | margin: 2em auto 0 auto; 15 | } 16 | 17 | #search_group { 18 | display: flex; 19 | flex-direction: row; 20 | margin-bottom: .5em; 21 | } 22 | 23 | #search_group #search { 24 | width: 90%; 25 | max-width: 650px; 26 | font-size: 1.4em; 27 | } 28 | 29 | #search_group #searchBtn { 30 | font-size: 1.4em; 31 | min-width: 70px; 32 | margin-left: auto; /** pull right **/ 33 | } 34 | 35 | #terms_text { 36 | margin-bottom: 2.5em; 37 | } 38 | 39 | #loading_text { 40 | font-size: 2em; 41 | opacity: 0.7; 42 | margin-bottom: 1.5em; 43 | } 44 | 45 | .search_result { 46 | display: flex; 47 | flex-direction: row; 48 | padding: 1em; 49 | background: #eeeeee; 50 | margin-bottom: 1.5em; 51 | border-right-style: outset; 52 | } 53 | 54 | .result_image { 55 | } 56 | 57 | .result_text { 58 | margin-left: 2em; 59 | margin-top: 1em; 60 | } 61 | 62 | .result_text span { 63 | opacity: 0.8; 64 | } 65 | 66 | .search_result .thumb { 67 | max-width: 200px; 68 | max-height: 200px; 69 | } 70 | 71 | .title { 72 | font-weight: bold; 73 | font-size: 1.2em; 74 | } 75 | 76 | .uploader { 77 | font-size: 1.1em; 78 | } 79 | 80 | a.btn { 81 | text-decoration: none; 82 | color: black; 83 | font-size: 1.1em; 84 | border: 1px solid; 85 | 86 | min-width: 100px; 87 | background: #F5F5F5; 88 | padding: 0.3em; 89 | } 90 | 91 | 92 | /** 93 | * HELPERS 94 | */ 95 | 96 | .flex_force { 97 | display: flex !important; 98 | } 99 | -------------------------------------------------------------------------------- /anyaudio/templates/home.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | AnyAudio | Lite 15 | 16 | 17 | 18 |
19 |

AnyAudio Lite

20 |
21 | 22 |
23 | 26 | 27 |
28 | 29 | 30 |
31 |
32 | By using this website, you agree to our Terms of Use 33 |
34 | 35 |

Popular Searches -

36 | {% for item in searches %} 37 | {{ item.0 }} 38 | {% endfor %} 39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /tests/test_stream.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import json 3 | import requests 4 | from tests import YMP3TestCase 5 | 6 | 7 | class TestStream(YMP3TestCase): 8 | """ 9 | Test Streaming feature 10 | """ 11 | def test_good_stream(self): 12 | """test a successful stream of a music video""" 13 | result = self._search('Love Story', just_results=True) 14 | stream_url = result[0]['stream_url'] 15 | resp = self.app.get(stream_url) 16 | self.assertEqual(resp.status_code, 200) 17 | self.assertIn('/stream_handler?', resp.data) 18 | # stream and download audio 19 | final_url = json.loads(resp.data)['url'] 20 | resp = self.app.get(final_url) 21 | self.assertTrue(len(resp.data) > 1000 * 1000) 22 | 23 | def test_fake_stream_url(self): 24 | """test the 5xx response in case of fake stream url""" 25 | resp = self.app.get('/api/v1/stream?url=somefalseurl') 26 | self.assertEqual(resp.status_code, 500) 27 | 28 | 29 | class TestStreamV2(YMP3TestCase): 30 | def test_good_stream(self): 31 | result = self._search_v2('Love Story', just_results=True) 32 | stream_url = result[0]['stream_url'] 33 | self.assertIn('/v2/stream', stream_url, stream_url) 34 | resp = self.app.get(stream_url) 35 | self.assertNotIn('stream_handler', resp.data) 36 | # stream and download audio 37 | final_url = json.loads(resp.data)['url'] 38 | self.assertIn('googlevideo', final_url, final_url) 39 | resp = requests.get(final_url) 40 | self.assertTrue(len(resp.content) > 500 * 1000, resp) 41 | 42 | 43 | if __name__ == '__main__': 44 | unittest.main() 45 | -------------------------------------------------------------------------------- /anyaudio/helpers/pafymodule.py: -------------------------------------------------------------------------------- 1 | import pafy 2 | from anyaudio import logger 3 | 4 | 5 | def get_download(url): 6 | """ 7 | gets download link of a audio from url 8 | url - can be full youtube url or just video id 9 | """ 10 | vid = pafy.new(url) 11 | audio_streams = vid.audiostreams 12 | return find_stream( 13 | audio_streams, 14 | [['m4a', 128], ['m4a', 192], ['ogg', 128], ['ogg', 192], ['m4a', 300], ['*', 0]] 15 | ) 16 | 17 | 18 | def get_stream(url): 19 | """ 20 | gets stream link of a audio from url 21 | """ 22 | vid = pafy.new(url) 23 | audio_streams = vid.audiostreams 24 | return find_stream( 25 | audio_streams, 26 | [ 27 | ['webm', 64], ['webm', 80], ['m4a', 64], ['webm', 128], ['webm', 192], ['m4a', 128], 28 | ['webm', 300], ['m4a', 300], ['*', 0] 29 | # ^^ 300 used as bitrate sometimes varies slightly over 256, 300 has no side effects 30 | ] 31 | ) 32 | 33 | 34 | def find_stream(streams, prefs): 35 | """ 36 | finds stream by priority 37 | streams = streams in descending order of bitrate 38 | prefs = [[format, bitrate]] 39 | bitrate - assumed to be less than equal to 40 | """ 41 | final = '' 42 | for item in prefs: 43 | # fallback case 44 | if item[0] == '*': 45 | final = streams[0] 46 | break 47 | # general preferences 48 | for stream in streams: 49 | if stream.extension == item[0] and int(stream.bitrate.replace('k', '')) <= item[1]: 50 | final = stream 51 | break 52 | if final: 53 | break 54 | logger.info(final) 55 | return final.url 56 | -------------------------------------------------------------------------------- /anyaudio/schedulers/trending.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | import threading 3 | 4 | from anyaudio import logger 5 | from . import Scheduler 6 | from ..helpers.data import trending_playlist 7 | from ..helpers.trending import get_trending_videos 8 | from ..helpers.networking import open_page 9 | from ..helpers.database import save_trending_songs, clear_trending 10 | 11 | 12 | class TrendingScheduler(Scheduler): 13 | 14 | def __init__(self, name='Trending Scheduler', period=21600, playlist=trending_playlist, 15 | connection_delay=0): 16 | Scheduler.__init__(self, name, period) 17 | self.playlist = playlist[:int(environ.get('PLAYLIST_LIST_LIMIT', 1000))] 18 | self.connection_delay = connection_delay 19 | 20 | def _worker(self, pl): 21 | logger.info('Crawling playlist "%s"' % pl[0]) 22 | 23 | playlist_name = pl[0] 24 | playlist_url = pl[1] 25 | 26 | html = open_page( 27 | url=playlist_url, 28 | sleep_upper_limit=self.connection_delay, 29 | ) 30 | 31 | song_data = get_trending_videos(html) 32 | print('Fetched song data') 33 | 34 | clear_trending(playlist_name) 35 | print('Cleared playlist') 36 | save_trending_songs(playlist_name, song_data) 37 | print('Saved trending') 38 | logger.info('Saved playlist "%s"' % pl[0]) 39 | 40 | def run(self): 41 | """ 42 | Run the trending crawler 43 | """ 44 | threads = [] 45 | for pl in self.playlist: 46 | thread = threading.Thread(target=self._worker, args=(pl, )) 47 | thread.start() 48 | threads.append(thread) 49 | 50 | for thread in threads: 51 | thread.join() 52 | -------------------------------------------------------------------------------- /anyaudio/helpers/networking.py: -------------------------------------------------------------------------------- 1 | from anyaudio.helpers.data import user_agents 2 | 3 | import requests 4 | 5 | from random import choice, uniform 6 | from time import sleep 7 | from traceback import print_exc 8 | 9 | 10 | def get_user_agent(): 11 | return choice(user_agents) 12 | 13 | 14 | def get_request_content(url, user_agent, allow_redirects, params): 15 | 16 | req = requests.get( 17 | url, 18 | headers={ 19 | 'User-Agent': user_agent 20 | }, 21 | allow_redirects=allow_redirects, 22 | params=params 23 | ) 24 | 25 | return req.content 26 | 27 | 28 | def post_request_content(url, allow_redirects, data): 29 | 30 | req = requests.post( 31 | url, 32 | data=data, 33 | allow_redirects=allow_redirects, 34 | ) 35 | 36 | return req.content 37 | 38 | 39 | def open_page(url, user_agent=get_user_agent(), sleep_lower_limit=0, sleep_upper_limit=0, 40 | allow_redirects=True, type='GET', params=None, data=None): 41 | """ 42 | Load a page and return its content 43 | """ 44 | try: 45 | sleep( 46 | uniform( 47 | sleep_lower_limit, 48 | sleep_upper_limit 49 | ) 50 | ) 51 | 52 | if type == 'GET': 53 | if not params: 54 | params = {} 55 | ret = get_request_content( 56 | url, 57 | user_agent, 58 | allow_redirects, 59 | params 60 | ) 61 | else: 62 | if not data: 63 | data = {} 64 | ret = post_request_content( 65 | url, 66 | allow_redirects, 67 | data 68 | ) 69 | return ret.decode('utf-8') 70 | except Exception: 71 | print_exc() 72 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | before_install: 5 | - sudo apt-get update -qq 6 | - sudo apt-get install -qq libpq-dev python2.7-dev postgresql-contrib-9.4 7 | env: 8 | APP_CONFIG: "config.TestingConfig" 9 | DOCKER_COMPOSE_VERSION: 1.8.0 10 | PATH: $PATH:$(pwd)/ffmpeg 11 | OPENSHIFT_POSTGRESQL_DB_HOST: localhost 12 | OPENSHIFT_POSTGRESQL_DB_USERNAME: ymp3 13 | OPENSHIFT_POSTGRESQL_DB_PORT: 5432 14 | OPENSHIFT_POSTGRESQL_DB_PASSWORD: ymp3 15 | POSTGRESQL_DB_NAME: ymp3 16 | PLAYLIST_VIDEOS_LIMIT: 6 17 | 18 | services: 19 | - docker 20 | - postgresql 21 | 22 | install: 23 | # - docker-compose build 24 | - pip install -r requirements.txt 25 | - docker build -t ymp3 . 26 | - docker images | grep -i ymp3 27 | - docker run -d -p 127.0.0.1:80:5000 --name ymp3 ymp3 28 | 29 | before_script: 30 | - psql -c "CREATE DATABASE ymp3;" -U postgres 31 | - psql -c "CREATE USER ymp3 WITH PASSWORD 'ymp3';" -U postgres 32 | - psql -c "GRANT ALL ON DATABASE ymp3 TO ymp3;" -U postgres 33 | 34 | script: 35 | # install ffmpeg 36 | - bash scripts/set_ffmpeg.sh 37 | # test 38 | - make test 39 | 40 | addons: 41 | postgresql: "9.4" 42 | 43 | notifications: 44 | webhooks: 45 | urls: 46 | - https://webhooks.gitter.im/e/087f782f596ba1c88c73 47 | on_success: change 48 | on_failure: always 49 | on_start: never 50 | 51 | # https://docs.travis-ci.com/user/docker/#Using-Docker-Compose 52 | # before_install: 53 | # - sudo apt-get update 54 | # - sudo apt-get remove -y docker-engine 55 | # - sudo apt-get install -y docker-engine 56 | # - sudo rm /usr/local/bin/docker-compose 57 | # - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose 58 | # - chmod +x docker-compose 59 | # - sudo mv docker-compose /usr/local/bin 60 | -------------------------------------------------------------------------------- /anyaudio/views/api_v2.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | 3 | from flask import jsonify, request 4 | from anyaudio import app 5 | 6 | from anyaudio.helpers.search import get_videos, get_video_attrs, \ 7 | get_search_results_html, make_search_api_response 8 | from anyaudio.helpers.helpers import record_request, make_error_response 9 | from anyaudio.helpers.encryption import get_key, decode_data 10 | 11 | from anyaudio.helpers.pafymodule import get_stream, get_download 12 | 13 | 14 | @app.route('/api/v2/stream') 15 | @record_request 16 | def stream_v2(): 17 | url = request.args.get('url') 18 | try: 19 | vid_id = decode_data(get_key(), url)['id'] 20 | url = get_stream(vid_id) 21 | return jsonify(status=200, url=url) 22 | except Exception as e: 23 | return make_error_response(msg=str(e), endpoint='api/v2/stream') 24 | 25 | 26 | @app.route('/api/v2/g') 27 | @record_request 28 | def get_link_v2(): 29 | url = request.args.get('url') 30 | try: 31 | data = decode_data(get_key(), url) 32 | vid_id = data['id'] 33 | retval = get_download(vid_id) 34 | return jsonify(status=200, url=retval) 35 | except Exception as e: 36 | return make_error_response(msg=str(e), endpoint='/api/v2/g') 37 | 38 | 39 | @app.route('/api/v2/search') 40 | @record_request 41 | def search_v2(): 42 | try: 43 | search_term = request.args.get('q') 44 | raw_html = get_search_results_html(search_term) 45 | vids = get_videos(raw_html) 46 | ret_vids = [] 47 | for _ in vids: 48 | temp = get_video_attrs(_, removeLongResult=False) 49 | if temp: 50 | temp['get_url'] = '/api/v2' + temp['get_url'] 51 | temp['stream_url'] = '/api/v2' + temp['stream_url'] 52 | temp['suggest_url'] = temp['get_url'].replace('v2/g?', 'v1/suggest?', 1) 53 | ret_vids.append(temp) 54 | ret_dict = make_search_api_response(search_term, ret_vids, '/api/v2/search') 55 | except Exception as e: 56 | return make_error_response(msg=str(e), endpoint='/api/v2/search') 57 | 58 | return jsonify(ret_dict) 59 | -------------------------------------------------------------------------------- /anyaudio/helpers/trending.py: -------------------------------------------------------------------------------- 1 | import re 2 | from os import environ 3 | from anyaudio import logger 4 | from anyaudio.helpers.networking import open_page 5 | from anyaudio.helpers.encryption import get_key, encode_data 6 | from anyaudio.helpers.helpers import html_unescape 7 | 8 | 9 | def get_trending_videos(html): 10 | """ 11 | Get trending youtube videos from html 12 | """ 13 | regex = '(.*?).*?by.*?>(.*?).*?(.*?)(.*?) ', 53 | html 54 | )[0] 55 | 56 | 57 | def get_description(html): 58 | desc = re.findall( 59 | '

(.*?)

0: 64 | return desc[0] 65 | return '' 66 | -------------------------------------------------------------------------------- /tests/test_download.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import requests 3 | from tests import YMP3TestCase 4 | 5 | 6 | class TestDownload(YMP3TestCase): 7 | """ 8 | Test download 9 | """ 10 | def test_successful_download(self): 11 | """test successful download of a music video""" 12 | result = self._search('Payphone Maroon 5', just_results=True) 13 | get_url = result[0]['get_url'] 14 | title = result[0]['title'] 15 | dl_url = self._get_dl_link(get_url, just_url=True) + '&format=mp3' 16 | resp = self.app.get(dl_url) 17 | self.assertTrue(len(resp.data) > 100000, resp.data) 18 | # test filename 19 | self.assertIn(title[:10], resp.headers['Content-Disposition'], resp.headers) 20 | # test file length 21 | self.assertEqual(int(resp.headers['Content-Length']), len(resp.data)) 22 | 23 | def test_failed_download(self): 24 | """test fail""" 25 | resp = self.app.get('/api/v1/d?url=askfasfk') 26 | self.assertEqual(resp.status_code, 500) 27 | self.assertTrue(len(resp.data) < 1000) 28 | 29 | def test_successful_download_m4a(self): 30 | """test successful download of a music in m4a""" 31 | # search and get link 32 | result = self._search('Payphone Maroon 5', just_results=True) 33 | get_url = result[0]['get_url'] 34 | dl_url = self._get_dl_link(get_url, just_url=True) + '&format=m4a' 35 | resp = self.app.get(dl_url) 36 | # test 37 | self.assertTrue(len(resp.data) > 100000, resp.data) 38 | self.assertEqual(int(resp.headers['Content-Length']), len(resp.data)) 39 | self.assertIn( 40 | '.m4a', resp.headers['Content-Disposition'], resp.headers['Content-Disposition'] 41 | ) 42 | 43 | 44 | class TestDownloadV2(YMP3TestCase): 45 | def test_successful_download(self): 46 | result = self._search_v2('Payphone Maroon 5', just_results=True) 47 | get_url = result[0]['get_url'] 48 | # title = result[0]['title'] 49 | dl_url = self._get_dl_link(get_url, just_url=True) 50 | resp = requests.get(dl_url) 51 | self.assertTrue(len(resp.content) > 100000, resp) 52 | 53 | 54 | if __name__ == '__main__': 55 | unittest.main() 56 | -------------------------------------------------------------------------------- /docs/EC2.md: -------------------------------------------------------------------------------- 1 | ### Deploy on EC2 2 | 3 | * Create an EC2 instance (ubuntu). 4 | * ssh into it. 5 | * apt-get update and upgrade 6 | 7 | * Then run the following commands. 8 | 9 | ```sh 10 | mkdir anyaudio.git 11 | mkdir app 12 | cd anyaudio.git 13 | git init --bare 14 | ``` 15 | 16 | * Now set the post-receive git hook. 17 | 18 | ```sh 19 | nano hooks/post-receive 20 | ``` 21 | 22 | * In nano, paste the contents of scripts/ec2_hook.sh 23 | 24 | * Then chmod the script 25 | 26 | ```sh 27 | chmod 775 hooks/post-receive 28 | ``` 29 | 30 | * Done. 31 | 32 | ```sh 33 | git remote add ec2 ssh://ec2-user@/home/ubuntu/anyaudio.git 34 | git push ec2 master 35 | ``` 36 | 37 | 38 | ### Running on EC2 39 | 40 | * Install Python. 41 | 42 | ```sh 43 | wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh 44 | bash Miniconda2-latest-Linux-x86_64.sh 45 | ``` 46 | 47 | * Install Postgres and build deps 48 | 49 | ```sh 50 | sudo apt install build-essential postgresql libpq-dev 51 | ``` 52 | 53 | * Install ffmpeg 54 | 55 | ```sh 56 | bash scripts/set_ffmpeg.sh 57 | ``` 58 | 59 | * Create postgres database 60 | 61 | ```sh 62 | sudo -u postgres psql 63 | ``` 64 | 65 | ```psql 66 | create user aa with password 'aa'; 67 | create database anyaudio with owner aa; 68 | ``` 69 | 70 | * To access server on port 80, run the following command. Also add it to `/etc/rc.local` without the sudo. ([Credits](http://stackoverflow.com/questions/16573668/)) 71 | 72 | ``` 73 | sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 5000 74 | ``` 75 | 76 | * Now run the server manually. You can also `git push` to have server run trigerred. 77 | 78 | ```sh 79 | python scripts/run_ec2.sh 80 | ``` 81 | 82 | 83 | ### Setting custom domain 84 | 85 | * Create an elastic IP and associate it with EC2 instance. http://andnovar.tech/2014/05/03/pointing-godaddy-domain-aws-ec2-instance/ 86 | * Now the EC2 url has changed so be sure to make the changes where necessary. 87 | * Change A record in your domain DNS tool to the elastic IP. 88 | * If you are using Cloudflare, change www to point to the EC2 domain and change A record to IP. 89 | 90 | 91 | #### Credits 92 | 93 | * https://gist.github.com/aviaryan/393fbb7d96b133d6dfbd430a21c5e73b 94 | * http://stackoverflow.com/questions/4632749/how-to-push-to-git-on-ec2 95 | -------------------------------------------------------------------------------- /anyaudio/templates/lite_song.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | AnyAudio | Lite 16 | 17 | 18 | 19 |
20 |

AnyAudio Lite

21 |
22 | 23 |
24 | 27 | 28 |
29 | By using this website, you agree to our Terms 30 | of Use 31 |
32 | 33 | 36 |
37 | Download 38 |
39 |

Related Videos -

40 | {% for result in video['suggestions'] %} 41 |
42 |
43 | 46 |
47 |
48 | 49 | {{ result['title'] }}
50 | {{ result['uploader'] }}
51 | {{ result['time'] }} · {{ result['views'] }} views
53 | {{ result['length'] }}

54 |
55 |
56 | {% endfor %} 57 | 58 |
59 | 60 | 61 | 62 | {##} 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /anyaudio/templates/lite_search.html: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | AnyAudio | Lite 16 | 17 | 18 | 19 |
20 |

AnyAudio Lite

21 |
22 | 23 |
24 | 27 | 28 |
29 | 31 | 32 |
33 |
34 | By using this website, you agree to our Terms 35 | of Use 36 |
37 | 38 |

Search results for {{ term }}

39 | {% for result in results %} 40 |
41 |
42 | 45 |
46 |
47 | 48 | {{ result['title'] }}
49 | {{ result['uploader'].decode('utf-8') }}
50 | {{ result['time'] }} · {{ result['views'] }} views
52 | {{ result['length'] }}

53 |
54 |
55 | {% endfor %} 56 | 57 |
58 | 59 | 60 | 61 | {##} 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /anyaudio/helpers/data.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | trending_playlist = [ 4 | ('Popular', 'https://www.youtube.com/playlist?list=PLFgquLnL59alCl_2TQvOiD5Vgm1hCaGSI'), 5 | ('Latest', 'https://www.youtube.com/playlist?list=PLFgquLnL59akA2PflFpeQG9L01VFg90wS'), 6 | ('India', 'https://www.youtube.com/playlist?list=PLFgquLnL59alF0GjxEs0V_XFCe7LM3ReH'), 7 | ('Weekly', 'https://www.youtube.com/playlist?list=PLFgquLnL59alW3xmYiWRaoz0oM3H17Lth'), 8 | ('Electronic', 'https://www.youtube.com/playlist?list=PLFPg_IUxqnZNnACUGsfn50DySIOVSkiKI'), 9 | ('Popular Music Videos', 'https://www.youtube.com/playlist?list=PLFgquLnL59alCl_2TQvOiD5Vgm1hCaGSI'), 10 | ('New Music This Week', 'https://www.youtube.com/playlist?list=PLFgquLnL59alW3xmYiWRaoz0oM3H17Lth'), 11 | ('Top Tracks', 'https://www.youtube.com/playlist?list=PLFgquLnL59alcyTM2lkWJU34KtfPXQDaX'), 12 | ('Hip Hop and R&B Hotlist', 'https://www.youtube.com/playlist?list=PLFgquLnL59amBBTCULGWSotJu2CkioYkj'), 13 | ('Pop Hotlist', 'https://www.youtube.com/playlist?list=PLFgquLnL59altZg1f_Kr1kGUYE6j-NE0M'), 14 | ('Most Viewed', 'https://www.youtube.com/playlist?list=PL8A83124F1D79BD4F') 15 | ] 16 | 17 | user_agents = [ 18 | 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 19 | 'Googlebot/2.1 (+http://www.google.com/bot.html)', 20 | 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)' 21 | ' Ubuntu Chromium/49.0.2623.108 Chrome/49.0.2623.108 Safari/537.36', 22 | 'Gigabot/3.0 (http://www.gigablast.com/spider.html)', 23 | 'Mozilla/5.0 (Windows; U; Windows NT 5.1; pt-BR) AppleWebKit/533.3 ' 24 | '(KHTML, like Gecko) QtWeb Internet Browser/3.7 http://www.QtWeb.net', 25 | 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) ' 26 | 'Chrome/41.0.2228.0 Safari/537.36', 27 | 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, ' 28 | 'like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2', 29 | 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.4pre) ' 30 | 'Gecko/20070404 K-Ninja/2.1.3', 31 | 'Mozilla/5.0 (Future Star Technologies Corp.; Star-Blade OS; x86_64; U; ' 32 | 'en-US) iNet Browser 4.7', 33 | 'Mozilla/5.0 (Windows; U; Windows NT 6.1; rv:2.2) Gecko/20110201', 34 | 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13) ' 35 | 'Gecko/20080414 Firefox/2.0.0.13 Pogo/2.0.0.13.6866', 36 | 'WorldWideweb (NEXT)' 37 | ] 38 | 39 | table_creation_sqlite_statements = [ 40 | '''create table if not exists trending_songs(id_ text, title_ text, thumb_ text, uploader_ text, length_ text, views_ text, get_url_ text, playlist_ text, description text)''', 41 | ] 42 | -------------------------------------------------------------------------------- /anyaudio/helpers/database.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sqlite3 3 | from anyaudio import DATABASE_PATH 4 | 5 | from ..helpers.data import table_creation_sqlite_statements 6 | 7 | 8 | def init_databases(): 9 | init_sqlite_database() 10 | 11 | 12 | def get_sqlite_connection(): 13 | conn = sqlite3.connect(DATABASE_PATH) 14 | return conn, conn.cursor() 15 | 16 | 17 | def init_sqlite_database(): 18 | conn, cursor = get_sqlite_connection() 19 | 20 | for statement in table_creation_sqlite_statements: 21 | cursor.execute(statement) 22 | 23 | conn.commit() 24 | conn.close() 25 | 26 | 27 | def save_trending_songs(playlist_name, songs): 28 | conn, cursor = get_sqlite_connection() 29 | 30 | try: 31 | sql = 'insert into trending_songs values(?,?,?,?,?,?,?,?,?)' 32 | 33 | data = [ 34 | ( 35 | song['id'], 36 | song['title'], 37 | song['thumb'], 38 | song['uploader'], 39 | song['length'], 40 | song['views'], 41 | song['get_url'], 42 | playlist_name, 43 | song['description'] 44 | ) for song in songs 45 | ] 46 | 47 | cursor.executemany(sql, data) 48 | conn.commit() 49 | 50 | except Exception: 51 | import traceback 52 | traceback.print_exc() 53 | pass 54 | conn.close() 55 | 56 | 57 | def get_trending(type='popular', count=25, offset=0, get_url_prefix=''): 58 | conn, cursor = get_sqlite_connection() 59 | 60 | sql = 'select * from trending_songs where playlist_ = ? limit ? offset ?' 61 | 62 | rows = cursor.execute(sql, (type, count, offset)) 63 | 64 | vids = [] 65 | for row in rows: 66 | row = list(row) 67 | row[6] = row[6].decode('utf-8') 68 | vids.append( 69 | { 70 | 'id': row[0], 71 | 'title': row[1], 72 | 'thumb': row[2], 73 | 'uploader': row[3], 74 | 'length': row[4], 75 | 'views': row[5], 76 | 'get_url': get_url_prefix + row[6], 77 | 'stream_url': (get_url_prefix + row[6]).replace('/g?', '/stream?', 1), 78 | 'description': row[8], 79 | 'suggest_url': (get_url_prefix + row[6]).replace('/g?', '/suggest?', 1), 80 | } 81 | ) 82 | 83 | conn.close() 84 | 85 | return vids 86 | 87 | 88 | def clear_trending(pl_name): 89 | conn, cur = get_sqlite_connection() 90 | 91 | sql = 'delete from trending_songs where playlist_ = ?' 92 | 93 | cur.execute(sql, (pl_name,)) 94 | 95 | conn.commit() 96 | conn.close() 97 | 98 | -------------------------------------------------------------------------------- /anyaudio/helpers/helpers.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import wraps 3 | from subprocess import check_output 4 | from anyaudio import LOCAL, logger 5 | from flask import request, jsonify 6 | from youtube_dl import YoutubeDL 7 | from html.parser import HTMLParser 8 | from mutagen.mp4 import MP4, MP4Cover 9 | from anyaudio.helpers.networking import open_page 10 | 11 | 12 | FILENAME_EXCLUDE = '<>:"/\|?*;' 13 | # semi-colon is terminator in header 14 | 15 | 16 | def delete_file(path): 17 | """ 18 | safely delete file. Needed in case of Asynchronous threads 19 | """ 20 | try: 21 | os.remove(path) 22 | except Exception: 23 | pass 24 | 25 | 26 | def get_ffmpeg_path(): 27 | if os.environ.get('FFMPEG_PATH'): 28 | return os.environ.get('FFMPEG_PATH') 29 | elif not LOCAL: # openshift 30 | return 'ffmpeg/ffmpeg' 31 | else: 32 | return 'ffmpeg' # hoping that it is set in PATH 33 | 34 | 35 | def get_video_info_ydl(vid_id): 36 | """ 37 | Gets video info using YoutubeDL 38 | """ 39 | ydl = YoutubeDL() 40 | try: 41 | info_dict = ydl.extract_info(vid_id, download=False) 42 | return info_dict 43 | except: 44 | return {} 45 | 46 | 47 | def get_filename_from_title(title, ext='.m4a'): 48 | """ 49 | Creates a filename from title 50 | """ 51 | if not title: 52 | return 'music' + ext 53 | title = HTMLParser().unescape(title) 54 | for _ in FILENAME_EXCLUDE: 55 | title = title.replace(_, ' ') # provide readability with space 56 | return title + ext # TODO - smart hunt 57 | 58 | 59 | def html_unescape(text): 60 | """ 61 | Remove &409D; type unicode symbols and convert them to real unicode 62 | """ 63 | try: 64 | title = HTMLParser().unescape(text) 65 | except Exception: 66 | title = text 67 | return title 68 | 69 | 70 | def record_request(func): 71 | """ 72 | Wrapper to log a request 73 | """ 74 | @wraps(func) 75 | def wrapper(*args, **kwargs): 76 | # TODO: Implement logging to some source 77 | return func(*args, **kwargs) 78 | return wrapper 79 | 80 | 81 | def add_cover(filename, video_id): 82 | raw_image = open_page('https://img.youtube.com/vi/%s/0.jpg' % video_id) 83 | 84 | audio = MP4(filename) 85 | cover = MP4Cover(raw_image) 86 | 87 | audio['covr'] = [cover] 88 | audio.save() 89 | 90 | 91 | def get_download_link_youtube(vid_id, frmat): 92 | """ 93 | gets the download link of a youtube video 94 | """ 95 | command = 'youtube-dl https://www.youtube.com/watch?v=%s -f %s -g' % (vid_id, frmat) 96 | logger.info(command) 97 | retval = check_output(command.split()) 98 | return retval.strip().decode('utf-8') 99 | 100 | 101 | def make_error_response(msg, endpoint, code=500): 102 | """ 103 | returns the error Response 104 | """ 105 | return jsonify({ 106 | 'status': code, 107 | 'requestLocation': endpoint, 108 | 'developerMessage': msg, 109 | 'userMessage': 'Some error occurred', 110 | 'errorCode': '500-001' 111 | }), code 112 | 113 | 114 | def generate_data(resp, chunk=2048): 115 | for data_chunk in resp.iter_content(chunk_size=chunk): 116 | yield data_chunk 117 | -------------------------------------------------------------------------------- /anyaudio/views/generic.py: -------------------------------------------------------------------------------- 1 | # from flask import redirect 2 | from flask import render_template, Markup, request 3 | # from flask import url_for 4 | 5 | from anyaudio import app 6 | # from anyaudio.helpers.encryption import encode_data, get_key 7 | from anyaudio.helpers.helpers import record_request, get_download_link_youtube 8 | # from anyaudio.helpers import database 9 | # from anyaudio.helpers.search import get_search_results_html, get_videos, \ 10 | # get_video_attrs, get_suggestions 11 | # 12 | # 13 | # @app.route('/lite', strict_slashes=False) 14 | # @record_request 15 | # def home(): 16 | # popular_searches = database.get_popular_searches(number=50) 17 | # return render_template('/home.html', searches=popular_searches) 18 | # 19 | # 20 | # @app.route('/lite/search') 21 | # @record_request 22 | # def lite_search(): 23 | # search_term = request.args.get('q', None) 24 | # if search_term is None: 25 | # return redirect('/lite') 26 | # raw_html = get_search_results_html(search_term) 27 | # vids = get_videos(raw_html) 28 | # ret_vids = [] 29 | # for _ in vids: 30 | # temp = get_video_attrs(_, removeLongResult=True) 31 | # if temp: 32 | # temp['get_url'] = '/api/v1' + temp['get_url'] 33 | # temp['stream_url'] = '/api/v1' + temp['stream_url'] 34 | # temp['suggest_url'] = temp['get_url'].replace('/g?', '/suggest?', 35 | # 1) 36 | # ret_vids.append(temp) 37 | # return render_template('/lite_search.html', results=ret_vids, 38 | # term=search_term) 39 | # 40 | # 41 | # @app.route('/lite/music/') 42 | # @record_request 43 | # def serve_music_lite(id): 44 | # video = {} 45 | # if 'bot' in request.user_agent.string.lower(): 46 | # url = "" 47 | # stream_url = "" 48 | # download_url = "" 49 | # else: 50 | # url = get_download_link_youtube( 51 | # id, 'webm[abr<=64]/webm[abr<=80]/m4a[abr<=64]/[abr<=96]/m4a') 52 | # stream_url = url_for('stream_handler', url=encode_data(get_key(), 53 | # url=url)) 54 | # url = get_download_link_youtube(id, 'm4a/bestaudio') 55 | # download_url = url_for( 56 | # 'download_file', url=encode_data(get_key(), url=url)) 57 | # 58 | # video['stream_url'] = stream_url 59 | # video['download_url'] = download_url 60 | # video['suggestions'] = get_suggestions(id) 61 | # return render_template('/lite_song.html', video=video) 62 | 63 | 64 | @app.route('/beta') 65 | @app.route('/') 66 | @record_request 67 | def home_beta(): 68 | return render_template('/index.html') 69 | 70 | 71 | @app.route('/terms-of-use') 72 | @record_request 73 | def terms_of_use(): 74 | return render_template('/terms-of-use.html') 75 | 76 | 77 | @app.route('/explore') 78 | @record_request 79 | def explore(): 80 | search_query = request.args.get('q') 81 | if search_query: 82 | search_query = '"{0}"'.format( 83 | search_query.replace('\"', '\\\"').strip()) 84 | else: 85 | search_query = '""' 86 | 87 | playlist = request.args.get('p') 88 | if playlist: 89 | playlist = '"{0}"'.format(playlist.replace('\"', '\\\"').strip()) 90 | else: 91 | playlist = '""' 92 | return render_template('/explore.html', query=Markup(search_query), playlist=Markup(playlist)) 93 | 94 | 95 | @app.route('/app') 96 | @record_request 97 | def download_app(): 98 | return render_template('/app.html') 99 | 100 | 101 | @app.route('/robots.txt') 102 | def get_robots(): 103 | return render_template("robots.txt"), 200, {'Content-Type': 'text/text; charset=utf-8'} 104 | -------------------------------------------------------------------------------- /anyaudio/static/main.js: -------------------------------------------------------------------------------- 1 | // search button on click 2 | // gets search results 3 | $('#searchBtn').click(function(){ 4 | search_text = $('#search').val(); 5 | // console.log(search_text); 6 | $('#loading_text').show(); 7 | // remove old data 8 | search_temp = $('.search_result').first().clone(); 9 | // reset 10 | search_temp.show(); 11 | // stop loading previous audio 12 | // http://stackoverflow.com/questions/4071872/html5-video-force-abort-of-buffering 13 | search_temp.find('.webm-audio').attr('src', ''); 14 | search_temp.find('.m4a-audio').attr('src', ''); 15 | search_temp.find('audio').load(); 16 | // set other defaults 17 | search_temp.find('.download').text('Get Link'); 18 | search_temp.find('.download').attr('href', '#'); 19 | search_temp.find('.download_mp3').text('Download MP3'); 20 | search_temp.find('.download_mp3').attr('href', '#'); 21 | search_temp.find('audio').hide(); 22 | search_temp.find('.stream').show(); 23 | search_temp.find('.stream').text('Stream'); 24 | search_temp.find('.thumb').attr('src', 'http://placehold.it/480x360.png?text=AnyAudio'); 25 | search_temp.unbind('click'); 26 | // delete 27 | $('.search_result').remove(); 28 | // get new results 29 | $.getJSON('/api/v2/search?q=' + search_text, success=function(data, textStatus, jqXHR){ 30 | // create new 31 | data = data['results']; 32 | for (i=0; i Save As)'); 76 | elem.click(download_start); 77 | elem.attr('href', data['url']); 78 | elem.attr('target', '_blank'); 79 | return false; 80 | }); 81 | } 82 | 83 | // starts the streaming 84 | function start_streaming(event){ 85 | event.preventDefault(); 86 | elem = $(event.target); 87 | elem.text('Connecting...'); 88 | elem.unbind('click'); 89 | $.getJSON(elem.attr('data-stream-url'), success=function(data, textStatus, jqXHR){ 90 | if (data['status'] != 200){ 91 | elem.text('Failed'); 92 | return false; 93 | } 94 | elem.hide(); 95 | var audio = elem.siblings('audio'); 96 | if (data['url'].search('audio%2Fwebm') > -1 || data['url'].search('audio/webm') > -1){ 97 | audio.find('.webm-audio').attr('src', data['url']); 98 | } else { 99 | audio.find('.m4a-audio').attr('src', data['url']); 100 | } 101 | audio.show(); 102 | audio.load(); 103 | audio[0].play(); // audio comes as array 104 | }); 105 | return false; 106 | } 107 | 108 | // start mp3 download 109 | function start_mp3_download(event){ 110 | event.preventDefault(); 111 | elem = $(event.target); 112 | elem.text('Please wait...'); 113 | elem.unbind('click'); 114 | $.getJSON(elem.attr('data-api-link'), success=function(data, textStatus, jqXHR){ 115 | elem.attr('href', data['link']); 116 | elem.text('Click to download'); 117 | }); 118 | return false; 119 | } 120 | 121 | // after download button is clicked 122 | function download_start(event){ 123 | $(event.target).text('Please wait'); 124 | elem = $(event.target); 125 | setTimeout(function(){ // let the link activate 126 | elem.attr('href', '#'); 127 | elem.removeAttr('download'); 128 | elem.removeAttr('target'); // don't open new tab 129 | elem.unbind('click'); 130 | }, 500); 131 | } 132 | 133 | $(document).ready(function(){ 134 | $('.search_result').hide(); 135 | $('#search').keyup(function(e){ 136 | if(e.keyCode == 13){ 137 | $(this).trigger("enterKey"); 138 | } 139 | }); 140 | 141 | $('#search').bind('enterKey', function(e){ 142 | $('#searchBtn').click(); 143 | }); 144 | }); 145 | -------------------------------------------------------------------------------- /anyaudio/templates/log_page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | API calls made to server 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 | 15 |

Insights of API calls to MusicGenie

16 |
17 |
18 |

API calls

19 |
20 |
21 | {% if prev_link %} 22 | Previous  23 | {% else %} 24 | Previous  25 | {% endif %} 26 | Next 27 |
28 |
29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | {% for row in logs %} 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | {% endfor %} 48 | 49 |
PathArgumentsRouteUser AgentRequest Time (IST)
{{ row['path'] }}{{ row['args'][:50] }}{{ row['access_route'] }}{{ row['user_agent'] }}{{ row['request_time'] }}
50 | 51 |
52 |
53 |

Insights of Day

54 | 55 | 56 | 57 | 58 | 59 | 60 | {% for elem in day_path %} 61 | 62 | 63 | 64 | 65 | {% endfor %} 66 | 67 | 68 | 69 | 70 | 71 |
PathNumber of Calls
{{ elem[0] }}{{ elem[1] }}
Total{{ day_sum }}
72 |
73 |
74 |

Insights of Month

75 | 76 | 77 | 78 | 79 | 80 | 81 | {% for elem in month_path %} 82 | 83 | 84 | 85 | 86 | {% endfor %} 87 | 88 | 89 | 90 | 91 | 92 |
PathNumber of Calls
{{ elem[0] }}{{ elem[1] }}
Total{{ month_sum }}
93 |
94 |
95 |

All Time Insights

96 | 97 | 98 | 99 | 100 | 101 | 102 | {% for elem in all_path %} 103 | 104 | 105 | 106 | 107 | {% endfor %} 108 | 109 | 110 | 111 | 112 | 113 |
PathNumber of Calls
{{ elem[0] }}{{ elem[1] }}
Total{{ all_sum }}
114 |
115 |
116 |

Popular Searches (Week)

117 | 118 | 119 | 120 | 121 | 122 | 123 | {% for elem in popular_queries %} 124 | 125 | 126 | 127 | 128 | {% endfor %} 129 | 130 |
QueryNumber of searches
{{ elem[0] }}{{ elem[1] }}
131 |
132 |
133 |
134 | 135 | 136 | -------------------------------------------------------------------------------- /anyaudio/helpers/search.py: -------------------------------------------------------------------------------- 1 | import re 2 | import traceback 3 | 4 | from anyaudio.helpers.encryption import get_key, encode_data 5 | 6 | from anyaudio.helpers.helpers import html_unescape 7 | from .networking import open_page 8 | 9 | INF = float("inf") 10 | SEARCH_SUFFIX = ' (song|full song|remix|karaoke|instrumental)' 11 | 12 | area_of_concern_regex = re.compile(r'