├── docs ├── api │ └── design.md ├── about │ └── contributing.md ├── usage │ ├── using-docker.md │ ├── configuration.md │ ├── production-nginx.md │ └── installation.md └── index.md ├── .dockerignore ├── static ├── favicon.ico └── robots.txt ├── requirements-mysql.txt ├── requirements.txt ├── models ├── __init__.py ├── message.py ├── subscription.py ├── service.py ├── mqtt.py └── gcm.py ├── controllers ├── __init__.py ├── gcm.py ├── mqtt.py ├── subscription.py ├── message.py └── service.py ├── Dockerfile ├── mkdocs.yml ├── shared.py ├── database.py ├── .gitignore ├── LICENSE ├── .gitlab-ci.yml ├── README.md ├── application.py ├── utils.py ├── config.py └── tests.py /docs/api/design.md: -------------------------------------------------------------------------------- 1 | # Design 2 | 3 | TBD -------------------------------------------------------------------------------- /docs/about/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | TBD -------------------------------------------------------------------------------- /docs/usage/using-docker.md: -------------------------------------------------------------------------------- 1 | # Using Docker 2 | 3 | TBD -------------------------------------------------------------------------------- /docs/usage/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | TBD -------------------------------------------------------------------------------- /docs/usage/production-nginx.md: -------------------------------------------------------------------------------- 1 | # Usage in Production (Nginx) 2 | 3 | TBD -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/*.pyc 2 | __pycache__ 3 | pushfish-api.cfg 4 | pushfish-api.db -------------------------------------------------------------------------------- /static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PushFish/pushfish-api/HEAD/static/favicon.ico -------------------------------------------------------------------------------- /requirements-mysql.txt: -------------------------------------------------------------------------------- 1 | pymysql 2 | sqlalchemy 3 | flask 4 | flask-sqlalchemy 5 | pyzmq 6 | requests 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | sqlalchemy 2 | flask 3 | flask-sqlalchemy 4 | pyzmq 5 | requests 6 | appdirs 7 | paho-mqtt -------------------------------------------------------------------------------- /static/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /message 3 | Disallow: /gcm 4 | Disallow: /listen 5 | Disallow: /service 6 | Disallow: /version 7 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | from .service import Service 2 | from .message import Message 3 | from .subscription import Subscription 4 | from .gcm import Gcm 5 | from .mqtt import MQTT 6 | -------------------------------------------------------------------------------- /controllers/__init__.py: -------------------------------------------------------------------------------- 1 | from .subscription import subscription 2 | from .message import message 3 | from .service import service 4 | from .gcm import gcm 5 | from .mqtt import mqtt 6 | -------------------------------------------------------------------------------- /docs/usage/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## From Source 4 | 5 | TBD 6 | ## Using Docker 7 | 8 | See [Using Docker](using-docker.md) 9 | ## Nginx for Production Use 10 | 11 | see [Usage in Production (Nginx)](production-nginx.md) -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-slim 2 | 3 | ENV HOME_DIR=/usr/src/app 4 | WORKDIR $HOME_DIR 5 | 6 | COPY requirements.txt ./ 7 | RUN pip install --no-cache-dir -r requirements.txt 8 | 9 | COPY . . 10 | ENV FLASK_APP=$HOME_DIR/application.py 11 | ENV PUSHFISH_CONFIG=$HOME_DIR/pushfish-api.cfg 12 | ENV PUSHFISH_DB=sqlite:////$HOME_DIR/pushfish-api.db 13 | CMD ["flask", "run", "--host", "0.0.0.0"] 14 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | ### PushFish 3 | PushFish is a FOSS alternative to PushBullet 4 | PushFish uses MQTT as a transport layer to remote devices, 5 | and aims to achieve full end-to-end encryption across all devices. 6 | 7 | PushFish is based on [PushJet](https://github.com/Pushjet/Pushjet-Server-Api) 8 | which in now abadoned. 9 | 10 | ### Goals 11 | * Always be fully free and open-source. 12 | * End-to-end encryption where it's possible. 13 | * TBD 14 | 15 | --- 16 | ## Design 17 | TBD 18 | 19 | ## Security 20 | TBD -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: PushFish-API docs 2 | site_description: 'Documentation for the PushFish-API server' 3 | 4 | theme: 5 | name: rtd-dropdown 6 | 7 | repo_url: https://gitlab.com/PushFish/PushFish-API 8 | nav: 9 | - Introduction: index.md 10 | - API: 11 | - Design: api/design.md 12 | - Usage: 13 | - Installation: usage/installation.md 14 | - Configuration: usage/configuration.md 15 | - Usage in Production (Nginx): usage/production-nginx.md 16 | - Using Docker: usage/using-docker.md 17 | - About: 18 | - Contributing: about/contributing.md -------------------------------------------------------------------------------- /shared.py: -------------------------------------------------------------------------------- 1 | from flask_sqlalchemy import SQLAlchemy 2 | import zmq 3 | # TODO: a better way of doing this 4 | from config import Config, fatal_error_exit_or_backtrace 5 | 6 | db = SQLAlchemy() 7 | 8 | zmq_relay_socket = None 9 | zeromq_context = None 10 | 11 | cfg = Config.get_global_instance() 12 | 13 | if cfg.zeromq_relay_uri: 14 | zeromq_context = zmq.Context() 15 | zmq_relay_socket = zeromq_context.socket(zmq.PUSH) 16 | try: 17 | zmq_relay_socket.connect(cfg.zeromq_relay_uri) 18 | except zmq.error.ZMQError as err: 19 | errstr = "coudn't connect to ZMQ relay, perhaps your option %s is wrong. current value:%s" 20 | fatal_error_exit_or_backtrace(err, errstr, None, "zeromq_relay_uri", cfg.zeromq_relay_uri) 21 | -------------------------------------------------------------------------------- /database.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import create_engine 2 | from sqlalchemy.orm import scoped_session, sessionmaker 3 | from sqlalchemy.ext.declarative import declarative_base 4 | from shared import db 5 | from config import Config 6 | 7 | cfg = Config.get_global_instance() 8 | 9 | engine = create_engine(cfg.database_uri, convert_unicode=True) 10 | db_session = scoped_session(sessionmaker(autocommit=False, 11 | autoflush=False, 12 | bind=engine)) 13 | Base = declarative_base() 14 | Base.query = db_session.query_property() 15 | 16 | 17 | def init_db(): 18 | # import all modules here that might define models so that 19 | # they will be registered properly on the metadata. Otherwise 20 | # you will have to import them first before calling init_db() 21 | import models 22 | db.create_all() 23 | Base.metadata.create_all(bind=engine) 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | venv/ 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | 56 | 57 | #editor 58 | .idea 59 | *.cfg 60 | 61 | #gedit 62 | *~ 63 | 64 | wsgi.py 65 | 66 | start_server 67 | uwsgi.ini 68 | 69 | # static website 70 | site/ 71 | public/ -------------------------------------------------------------------------------- /controllers/gcm.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, request, jsonify 2 | from utils import has_uuid, Error 3 | from models import Gcm 4 | from shared import db 5 | from config import Config 6 | 7 | cfg = Config.get_global_instance() 8 | 9 | gcm = Blueprint('gcm', __name__) 10 | 11 | 12 | @gcm.route("/gcm", methods=["POST"]) 13 | @has_uuid 14 | def gcm_register(client): 15 | registration = request.form.get('regid', False) or request.form.get('regId', False) 16 | 17 | if not registration: 18 | return Error.ARGUMENT_MISSING('regid') 19 | regs = Gcm.query.filter_by(uuid=client).all() 20 | for u in regs: 21 | db.session.delete(u) 22 | reg = Gcm(client, registration) 23 | db.session.add(reg) 24 | db.session.commit() 25 | return Error.NONE 26 | 27 | 28 | @gcm.route("/gcm", methods=["DELETE"]) 29 | @has_uuid 30 | def gcm_unregister(client): 31 | regs = Gcm.query.filter_by(uuid=client).all() 32 | for u in regs: 33 | db.session.delete(u) 34 | db.session.commit() 35 | return Error.NONE 36 | 37 | 38 | @gcm.route("/gcm", methods=["GET"]) 39 | def gcm_sender_id(): 40 | data = dict(sender_id=cfg.google_gcm_sender_id) 41 | return jsonify(data) 42 | -------------------------------------------------------------------------------- /controllers/mqtt.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify 2 | from utils import has_uuid, Error 3 | from models import MQTT 4 | from shared import db 5 | from config import Config 6 | 7 | cfg = Config.get_global_instance() 8 | 9 | mqtt = Blueprint('mqtt', __name__) 10 | 11 | 12 | @mqtt.route("/mqtt", methods=["POST"]) 13 | @has_uuid 14 | def mqtt_register(client): 15 | """ 16 | register by uuid to a service 17 | :param client: client uuid 18 | """ 19 | regs = MQTT.query.filter_by(uuid=client).all() 20 | for u in regs: 21 | db.session.delete(u) 22 | reg = MQTT(client) 23 | db.session.add(reg) 24 | db.session.commit() 25 | return Error.NONE 26 | 27 | 28 | @mqtt.route("/mqtt", methods=["DELETE"]) 29 | @has_uuid 30 | def mqtt_unregister(client): 31 | """ 32 | unregister by uuid to a service 33 | :param client: client uuid 34 | """ 35 | regs = MQTT.query.filter_by(uuid=client).all() 36 | for u in regs: 37 | db.session.delete(u) 38 | db.session.commit() 39 | return Error.NONE 40 | 41 | 42 | @mqtt.route("/mqtt", methods=["GET"]) 43 | def mqtt_broker_address(): 44 | data = dict(broker_address=cfg.mqtt_broker_address) 45 | return jsonify(data) 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, IOException 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY 17 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /models/message.py: -------------------------------------------------------------------------------- 1 | from shared import db 2 | from datetime import datetime 3 | from sqlalchemy import Integer, Unicode 4 | 5 | 6 | class Message(db.Model): 7 | id = db.Column(Integer, primary_key=True) 8 | service_id = db.Column(Integer, db.ForeignKey('service.id'), 9 | nullable=False) 10 | service = db.relationship('Service', backref=db.backref('message', 11 | lazy='dynamic', 12 | cascade="delete")) 13 | text = db.Column(db.TEXT, nullable=False) 14 | title = db.Column(Unicode(length=255)) 15 | level = db.Column(Integer, nullable=False, default=0) 16 | link = db.Column(db.TEXT, nullable=False, default='') 17 | timestamp_created = db.Column(db.TIMESTAMP, default=datetime.utcnow) 18 | 19 | def __init__(self, service, text, title=None, level=0, link=''): 20 | self.service = service 21 | self.text = text 22 | self.title = title 23 | self.level = level 24 | self.link = link 25 | 26 | def __repr__(self): 27 | return ''.format(self.id) 28 | 29 | def as_dict(self): 30 | return { 31 | "service": self.service.as_dict(), 32 | "message": self.text, 33 | "title": self.title, 34 | "link": self.link, 35 | "level": self.level, 36 | "timestamp": int((self.timestamp_created - datetime.utcfromtimestamp(0)).total_seconds()) 37 | } 38 | -------------------------------------------------------------------------------- /controllers/subscription.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify 2 | from utils import Error, has_service, has_uuid, queue_zmq_message 3 | from shared import db 4 | from models import Subscription 5 | from json import dumps as json_encode 6 | from config import Config 7 | 8 | cfg = Config.get_global_instance() 9 | 10 | subscription = Blueprint('subscription', __name__) 11 | 12 | 13 | @subscription.route('/subscription', methods=['POST']) 14 | @has_uuid 15 | @has_service 16 | def subscription_post(client, service): 17 | exists = Subscription.query.filter_by(device=client).filter_by(service=service).first() is not None 18 | if exists: 19 | return Error.DUPLICATE_LISTEN 20 | 21 | subscription_new = Subscription(client, service) 22 | db.session.add(subscription_new) 23 | db.session.commit() 24 | 25 | if cfg.zeromq_relay_uri: 26 | queue_zmq_message(json_encode({'subscription': subscription_new.as_dict()})) 27 | 28 | return jsonify({'service': service.as_dict()}) 29 | 30 | 31 | @subscription.route('/subscription', methods=['GET']) 32 | @has_uuid 33 | def subscription_get(client): 34 | subscriptions = Subscription.query.filter_by(device=client).all() 35 | return jsonify({'subscriptions': [_.as_dict() for _ in subscriptions]}) 36 | 37 | 38 | @subscription.route('/subscription', methods=['DELETE']) 39 | @has_uuid 40 | @has_service 41 | def subscription_delete(client, service): 42 | l = Subscription.query.filter_by(device=client).filter_by(service=service).first() 43 | if l is not None: 44 | db.session.delete(l) 45 | db.session.commit() 46 | return Error.NONE 47 | return Error.NOT_SUBSCRIBED 48 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | # Official docker image. 2 | image: docker:latest 3 | 4 | services: 5 | - docker:dind 6 | 7 | stages: 8 | - build 9 | - test 10 | - release 11 | 12 | variables: 13 | CONTAINER_TEST_IMAGE: registry.gitlab.com/pushfish/pushfish-api:$CI_COMMIT_REF_SLUG 14 | CONTAINER_RELEASE_IMAGE: registry.gitlab.com/pushfish/pushfish-api:latest 15 | 16 | before_script: 17 | - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN registry.gitlab.com 18 | 19 | build: 20 | stage: build 21 | script: 22 | - docker build --pull -t $CONTAINER_TEST_IMAGE . 23 | - docker push $CONTAINER_TEST_IMAGE 24 | 25 | pages: 26 | image: python:alpine 27 | before_script: 28 | - pip install mkdocs mkdocs-rtd-dropdown 29 | stage: build 30 | script: 31 | - mkdocs build 32 | - mv site public 33 | artifacts: 34 | paths: 35 | - public 36 | only: 37 | - master 38 | 39 | unitests: 40 | stage: test 41 | script: 42 | - docker pull $CONTAINER_TEST_IMAGE 43 | - docker network create pushfish 44 | - docker run -d --name mqtt --network pushfish toke/mosquitto:latest 45 | - docker run -e MQTT_ADDRESS='mqtt' --network pushfish $CONTAINER_TEST_IMAGE python tests.py 46 | - docker stop mqtt 47 | - docker rm mqtt 48 | 49 | pylint: 50 | image: python:3-slim 51 | allow_failure: true 52 | before_script: 53 | - echo 54 | stage: test 55 | script: 56 | - pip install --no-cache-dir -r requirements.txt 57 | - pip install --no-cache-dir pylint 58 | - FILES=$(find . -name "*.py") 59 | - pylint ${FILES} 60 | 61 | release-image: 62 | stage: release 63 | script: 64 | - docker pull $CONTAINER_TEST_IMAGE 65 | - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE 66 | - docker push $CONTAINER_RELEASE_IMAGE 67 | only: 68 | - master 69 | -------------------------------------------------------------------------------- /models/subscription.py: -------------------------------------------------------------------------------- 1 | from shared import db 2 | from sqlalchemy import Integer 3 | from datetime import datetime 4 | from .message import Message 5 | 6 | 7 | class Subscription(db.Model): 8 | id = db.Column(Integer, primary_key=True) 9 | device = db.Column(db.VARCHAR(40), nullable=False) 10 | service_id = db.Column(Integer, db.ForeignKey('service.id'), nullable=False) 11 | service = db.relationship('Service', backref=db.backref('subscription', 12 | lazy='dynamic', 13 | cascade="delete")) 14 | last_read = db.Column(Integer, db.ForeignKey('message.id'), nullable=True) 15 | timestamp_created = db.Column(db.TIMESTAMP, default=datetime.utcnow) 16 | timestamp_checked = db.Column(db.TIMESTAMP) 17 | 18 | def __init__(self, device, service): 19 | last_message = Message.query.order_by(Message.id.desc()).first() 20 | 21 | self.device = device 22 | self.service = service 23 | self.timestamp_checked = datetime.utcnow() 24 | self.last_read = last_message.id if last_message else None 25 | 26 | def __repr__(self): 27 | return ''.format(self.id) 28 | 29 | def messages(self): 30 | return Message.query \ 31 | .filter_by(service_id=self.service_id) \ 32 | .filter(Message.id > self.last_read) 33 | 34 | def as_dict(self): 35 | data = { 36 | "uuid": self.device, 37 | "service": self.service.as_dict(), 38 | "timestamp": int((self.timestamp_created - datetime.utcfromtimestamp(0)).total_seconds()), 39 | "timestamp_checked": int((self.timestamp_checked - datetime.utcfromtimestamp(0)).total_seconds()) 40 | } 41 | return data 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PushFish API [![License](http://img.shields.io/badge/license-BSD-blue.svg?style=flat)](/LICENSE) 2 | ================== 3 | This is the core for PushFish. It manages the whole shebang. 4 | 5 | 6 | ## Configuration 7 | The pushfish API server reads various options from a configuration file. This configuration file can be specified by setting the environment variable `PUSHFISH_CONFIG`. If this variable is not set, then the file is searched for in a default path. 8 | 9 | ``` 10 | ~/.config/pushfish-api/pushfish-api.cfg # on Linux 11 | %APPDATA%\pushfish-api\pushfish-api.cfg # on Windows 12 | ~/Library/Application Support/pushfish-api/pushfish-api.cfg # on OSX 13 | ``` 14 | 15 | where the value for "user" will be changed to your current username. If this file does not exist, then the API server will generate a default configuration, which looks like this: 16 | 17 | ``` 18 | [database] 19 | #for mysql, use something like: 20 | #uri = 'mysql+pymysql://pushfish@localhost/pushfish_api?charset=utf8mb4' 21 | 22 | #for sqlite (the default), use something like: 23 | uri = sqlite:////home/pushfish/.local/share/pushfish-api/pushfish-api.db 24 | 25 | [dispatch] 26 | google_api_key = 27 | google_gcm_sender_id = 509878466986 28 | #point this at the pushfish-connectors zeroMQ pubsub socket 29 | zeromq_relay_uri = 30 | 31 | [server] 32 | #set to 0 for production mode 33 | debug = 1 34 | 35 | ``` 36 | 37 | the format of the database URI is an SQLAlchemy URL as [described here](http://docs.sqlalchemy.org/en/latest/core/engines.html) 38 | 39 | Docker 40 | ------------------ 41 | Build the image: 42 | 43 | ``` 44 | docker build -t pushfish-api:latest . 45 | ``` 46 | 47 | Run: 48 | 49 | ``` 50 | docker run pushfish-api:latest 51 | ``` 52 | 53 | Run tests.py: 54 | 55 | ``` 56 | docker run pushfish-api:latest python tests.py 57 | ``` 58 | -------------------------------------------------------------------------------- /models/service.py: -------------------------------------------------------------------------------- 1 | from shared import db 2 | from datetime import datetime 3 | from sqlalchemy import Integer 4 | import hashlib 5 | from os import urandom 6 | from .subscription import Subscription 7 | from .message import Message 8 | 9 | 10 | class Service(db.Model): 11 | id = db.Column(Integer, primary_key=True) 12 | secret = db.Column(db.VARCHAR(32), nullable=False) 13 | public = db.Column(db.VARCHAR(40), nullable=False) 14 | name = db.Column(db.Unicode(length=255), nullable=False) 15 | icon = db.Column(db.TEXT, nullable=False, default='') 16 | timestamp_created = db.Column(db.TIMESTAMP, default=datetime.utcnow) 17 | 18 | def __init__(self, name, icon=''): 19 | self.secret = hashlib.sha1(urandom(100)).hexdigest()[:32] 20 | self.name = name 21 | self.icon = icon 22 | pub = list(hashlib.new('ripemd160', self.secret.encode("UTF-8")).hexdigest())[:40] 23 | sep = [4, 11, 24, 30] 24 | for s in sep: 25 | pub[s] = '-' 26 | self.public = ''.join(pub) 27 | 28 | def __repr__(self): 29 | return ''.format(self.id, self.name) 30 | 31 | def cleanup(self): 32 | threshold = self.subscribed().order_by(Subscription.last_read.asc()).first().last_read 33 | 34 | # Nothing to do 35 | if not threshold: 36 | return 37 | 38 | messages = Message.query \ 39 | .filter_by(service=self) \ 40 | .filter(threshold > Message.id) \ 41 | .all() 42 | 43 | map(db.session.delete, messages) 44 | 45 | def subscribed(self): 46 | return Subscription.query.filter_by(service=self) 47 | 48 | def as_dict(self, secret=False): 49 | data = { 50 | "public": self.public, 51 | "name": self.name, 52 | "created": int((self.timestamp_created - datetime.utcfromtimestamp(0)).total_seconds()), 53 | "icon": self.icon, 54 | } 55 | if secret: 56 | data["secret"] = self.secret 57 | return data 58 | -------------------------------------------------------------------------------- /application.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # coding=utf-8 3 | from __future__ import unicode_literals 4 | from flask import Flask, redirect, send_from_directory, request 5 | import logging 6 | from sqlalchemy.exc import OperationalError 7 | import sys 8 | 9 | from config import Config 10 | 11 | _LOGGER = logging.getLogger(name="pushfish_API") 12 | 13 | _LOGGER.info("creating Config object") 14 | cfg = Config(create=True) 15 | 16 | import database 17 | 18 | from shared import db 19 | from controllers import subscription, message, service, gcm, mqtt 20 | from utils import Error 21 | 22 | gcm_enabled = True 23 | if cfg.google_api_key == '': 24 | _LOGGER.warning("WARNING: GCM disabled, please enter the google api key for gcm") 25 | gcm_enabled = False 26 | if cfg.google_gcm_sender_id == 0: 27 | _LOGGER.warning('WARNING: GCM disabled, invalid sender id found') 28 | gcm_enabled = False 29 | 30 | mqtt_enabled = True 31 | if cfg.mqtt_broker_address == '': 32 | _LOGGER.warning("WARNING: MQTT disabled, please enter the address for mqtt broker") 33 | mqtt_enabled = False 34 | if cfg.mqtt_broker_address == 0: 35 | _LOGGER.warning('WARNING: MQTT disabled, invalid address for mqtt broker') 36 | mqtt_enabled = False 37 | 38 | app = Flask(__name__) 39 | app.debug = cfg.debug 40 | app.config['SQLALCHEMY_DATABASE_URI'] = cfg.database_uri 41 | app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False 42 | db.init_app(app) 43 | db.app = app 44 | 45 | try: 46 | database.init_db() 47 | except Exception as err: 48 | _LOGGER.error("couldn't initialize database with URI: %s", cfg.database_uri) 49 | if cfg.GLOBAL_BACKTRACE_ENABLE: 50 | raise err 51 | else: 52 | sys.exit(1) 53 | 54 | 55 | @app.route('/') 56 | def index(): 57 | return redirect('https://www.push.fish') 58 | 59 | 60 | @app.route('/robots.txt') 61 | @app.route('/favicon.ico') 62 | def robots_txt(): 63 | return send_from_directory(app.static_folder, request.path[1:]) 64 | 65 | 66 | @app.route('/version') 67 | def version(): 68 | with open('.git/refs/heads/master', 'r') as f: 69 | return f.read(7) 70 | 71 | 72 | @app.errorhandler(429) 73 | def limit_rate(e): 74 | return Error.RATE_TOOFAST 75 | 76 | 77 | app.register_blueprint(subscription) 78 | app.register_blueprint(message) 79 | app.register_blueprint(service) 80 | if gcm_enabled: 81 | app.register_blueprint(gcm) 82 | if mqtt_enabled: 83 | app.register_blueprint(mqtt) 84 | 85 | if __name__ == '__main__': 86 | app.run() 87 | -------------------------------------------------------------------------------- /models/mqtt.py: -------------------------------------------------------------------------------- 1 | from shared import db 2 | from sqlalchemy import Integer 3 | from datetime import datetime 4 | from config import Config 5 | from models import Subscription, Message 6 | import paho.mqtt.client as mqtt_api 7 | 8 | 9 | cfg = Config.get_global_instance() 10 | 11 | 12 | class MQTT(db.Model): 13 | id = db.Column(Integer, primary_key=True) 14 | uuid = db.Column(db.VARCHAR(40), nullable=False) 15 | timestamp_created = db.Column(db.TIMESTAMP, default=datetime.utcnow) 16 | 17 | def __init__(self, device): 18 | self.uuid = device 19 | 20 | def __repr__(self): 21 | return ''.format(self.uuid) 22 | 23 | def as_dict(self): 24 | data = { 25 | "uuid": self.service.as_dict(), 26 | "timestamp": int((self.timestamp_created - datetime.utcfromtimestamp(0)).total_seconds()), 27 | } 28 | return data 29 | 30 | @staticmethod 31 | def send_message(message): 32 | """ 33 | 34 | :type message: Message to send to mqtt subscribers 35 | """ 36 | subscriptions = Subscription.query.filter_by(service=message.service).all() 37 | if len(subscriptions) == 0: 38 | return 0 39 | mqtt_devices = MQTT.query.filter(MQTT.uuid.in_([l.device for l in subscriptions])).all() 40 | 41 | if len(mqtt_devices) > 0: 42 | data = dict(message=message.as_dict(), encrypted=False) 43 | MQTT.mqtt_send([r.uuid for r in mqtt_devices], data) 44 | 45 | if len(mqtt_devices) > 0: 46 | uuids = [g.uuid for g in mqtt_devices] 47 | mqtt_subscriptions = Subscription.query.filter_by(service=message.service).filter( 48 | Subscription.device.in_(uuids)).all() 49 | last_message = Message.query.order_by(Message.id.desc()).first() 50 | for l in mqtt_subscriptions: 51 | l.timestamp_checked = datetime.utcnow() 52 | l.last_read = last_message.id if last_message else 0 53 | db.session.commit() 54 | return len(mqtt_devices) 55 | 56 | @staticmethod 57 | def mqtt_send(uuids, data): 58 | url = cfg.mqtt_broker_address 59 | if ":" in url: 60 | port = url.split(":")[1] 61 | url = url .split(":")[0] 62 | else: 63 | # default port 64 | port = 1883 65 | 66 | client = mqtt_api.Client() 67 | client.connect(url, port, 60) 68 | 69 | for uuid in uuids: 70 | client.publish(uuid, str(data)) 71 | client.disconnect() 72 | -------------------------------------------------------------------------------- /models/gcm.py: -------------------------------------------------------------------------------- 1 | from flask import current_app 2 | from shared import db 3 | from sqlalchemy import Integer 4 | from datetime import datetime 5 | from config import Config 6 | from models import Subscription, Message 7 | import requests 8 | 9 | cfg = Config.get_global_instance() 10 | 11 | gcm_url = 'https://android.googleapis.com/gcm/send' 12 | 13 | 14 | class Gcm(db.Model): 15 | id = db.Column(Integer, primary_key=True) 16 | uuid = db.Column(db.VARCHAR(40), nullable=False) 17 | gcmid = db.Column(db.TEXT, nullable=False) 18 | timestamp_created = db.Column(db.TIMESTAMP, default=datetime.utcnow) 19 | 20 | def __init__(self, device, gcmid): 21 | self.uuid = device 22 | self.gcmid = gcmid 23 | 24 | def __repr__(self): 25 | return ''.format(self.uuid) 26 | 27 | def as_dict(self): 28 | data = { 29 | "uuid": self.service.as_dict(), 30 | "gcm_registration_id": self.gcmId, 31 | "timestamp": int((self.timestamp_created - datetime.utcfromtimestamp(0)).total_seconds()), 32 | } 33 | return data 34 | 35 | @staticmethod 36 | def send_message(message): 37 | """ 38 | 39 | :type message: Message 40 | """ 41 | subscriptions = Subscription.query.filter_by(service=message.service).all() 42 | if len(subscriptions) == 0: 43 | return 0 44 | gcm_devices = Gcm.query.filter(Gcm.uuid.in_([l.device for l in subscriptions])).all() 45 | 46 | if len(gcm_devices) > 0: 47 | data = dict(message=message.as_dict(), encrypted=False) 48 | Gcm.gcm_send([r.gcmid for r in gcm_devices], data) 49 | 50 | if len(gcm_devices) > 0: 51 | uuids = [g.uuid for g in gcm_devices] 52 | gcm_subscriptions = Subscription.query.filter_by(service=message.service).filter( 53 | Subscription.device.in_(uuids)).all() 54 | last_message = Message.query.order_by(Message.id.desc()).first() 55 | for l in gcm_subscriptions: 56 | l.timestamp_checked = datetime.utcnow() 57 | l.last_read = last_message.id if last_message else 0 58 | db.session.commit() 59 | return len(gcm_devices) 60 | 61 | @staticmethod 62 | def gcm_send(ids, data): 63 | url = 'https://android.googleapis.com/gcm/send' 64 | headers = dict(Authorization='key={}'.format(cfg.google_api_key)) 65 | data = dict(registration_ids=ids, data=data) 66 | 67 | if current_app.config['TESTING'] is True: 68 | current_app.config['TESTING_GCM'].append(data) 69 | else: 70 | requests.post(url, json=data, headers=headers) 71 | -------------------------------------------------------------------------------- /controllers/message.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from json import dumps as json_encode 3 | 4 | from flask import Blueprint, jsonify, request 5 | from flask import current_app 6 | 7 | from utils import Error, has_uuid, has_secret, queue_zmq_message 8 | from shared import db 9 | from models import Subscription, Message, Gcm, MQTT 10 | from config import Config 11 | 12 | cfg = Config.get_global_instance() 13 | 14 | message = Blueprint('message', __name__) 15 | 16 | 17 | @message.route('/message', methods=['POST']) 18 | @has_secret 19 | def message_send(service): 20 | text = request.form.get('message') 21 | if not text: 22 | return Error.ARGUMENT_MISSING('message') 23 | 24 | subscribers = Subscription.query.filter_by(service=service).count() 25 | if subscribers == 0: 26 | # Pretend we did something even though we didn't 27 | # Nobody is listening so it doesn't really matter 28 | return Error.NONE 29 | 30 | level = (request.form.get('level') or '3')[0] 31 | level = int(level) if level in "12345" else 3 32 | title = request.form.get('title', '').strip()[:255] 33 | link = request.form.get('link', '').strip() 34 | msg = Message(service, text, title, level, link) 35 | db.session.add(msg) 36 | db.session.commit() 37 | 38 | if cfg.google_api_key or current_app.config['TESTING']: 39 | Gcm.send_message(msg) 40 | 41 | if cfg.mqtt_broker_address: 42 | MQTT.send_message(msg) 43 | 44 | if cfg.zeromq_relay_uri: 45 | queue_zmq_message(json_encode({"message": msg.as_dict()})) 46 | 47 | service.cleanup() 48 | db.session.commit() 49 | return Error.NONE 50 | 51 | 52 | @message.route('/message', methods=['GET']) 53 | @has_uuid 54 | def message_recv(client): 55 | subscriptions = Subscription.query.filter_by(device=client).all() 56 | if not subscriptions: 57 | return jsonify({'messages': []}) 58 | 59 | msg = [] 60 | for l in subscriptions: 61 | msg += l.messages().all() 62 | 63 | last_read = max([0] + [m.id for m in msg]) 64 | for l in subscriptions: 65 | l.timestamp_checked = datetime.utcnow() 66 | l.last_read = max(l.last_read, last_read) 67 | l.service.cleanup() 68 | 69 | ret = jsonify({'messages': [m.as_dict() for m in msg]}) 70 | db.session.commit() 71 | return ret 72 | 73 | 74 | @message.route('/message', methods=['DELETE']) 75 | @has_uuid 76 | def message_read(client): 77 | subscriptions = Subscription.query.filter_by(device=client).all() 78 | if subscriptions: 79 | last_message = Message.query.order_by(Message.id.desc()).first() 80 | for l in subscriptions: 81 | l.timestamp_checked = datetime.utcnow() 82 | l.last_read = last_message.id if last_message else 0 83 | 84 | for l in subscriptions: 85 | l.service.cleanup() 86 | db.session.commit() 87 | 88 | return Error.NONE 89 | -------------------------------------------------------------------------------- /controllers/service.py: -------------------------------------------------------------------------------- 1 | from json import dumps as json_encode 2 | 3 | from flask import Blueprint, jsonify, request 4 | from utils import Error, is_service, is_secret, has_secret, queue_zmq_message 5 | 6 | from models import Service, Message 7 | from shared import db 8 | from config import Config 9 | 10 | cfg = Config.get_global_instance() 11 | 12 | service = Blueprint('service', __name__) 13 | 14 | 15 | @service.route('/service', methods=['POST']) 16 | def service_create(): 17 | name = request.form.get('name', '').strip() 18 | icon = request.form.get('icon', '').strip() 19 | if not name: 20 | return Error.ARGUMENT_MISSING('name') 21 | srv = Service(name, icon) 22 | db.session.add(srv) 23 | db.session.commit() 24 | return jsonify({"service": srv.as_dict(True)}) 25 | 26 | 27 | @service.route('/service', methods=['GET']) 28 | def service_info(): 29 | secret = request.form.get('secret', '') or request.args.get('secret', '') 30 | service_ = request.form.get('service', '') or request.args.get('service', '') 31 | 32 | if service_: 33 | if not is_service(service_): 34 | return Error.INVALID_SERVICE 35 | 36 | srv = Service.query.filter_by(public=service_).first() 37 | if not srv: 38 | return Error.SERVICE_NOTFOUND 39 | return jsonify({"service": srv.as_dict()}) 40 | 41 | if secret: 42 | if not is_secret(secret): 43 | return Error.INVALID_SECRET 44 | 45 | srv = Service.query.filter_by(secret=secret).first() 46 | if not srv: 47 | return Error.SERVICE_NOTFOUND 48 | return jsonify({"service": srv.as_dict()}) 49 | 50 | return Error.ARGUMENT_MISSING('service') 51 | 52 | 53 | @service.route('/service', methods=['DELETE']) 54 | @has_secret 55 | def service_delete(service): 56 | subscriptions = service.subscribed().all() 57 | messages = Message.query.filter_by(service=service).all() 58 | 59 | # In case we need to send this at a later point 60 | # when the subscriptions have been deleted. 61 | send_later = [] 62 | if cfg.zeromq_relay_uri: 63 | for l in subscriptions: 64 | send_later.append(json_encode({'subscription': l.as_dict()})) 65 | 66 | map(db.session.delete, subscriptions) # Delete all subscriptions 67 | map(db.session.delete, messages) # Delete all messages 68 | db.session.delete(service) 69 | 70 | db.session.commit() 71 | 72 | # Notify that the subscriptions have been deleted 73 | if cfg.zeromq_relay_uri: 74 | map(queue_zmq_message, send_later) 75 | 76 | return Error.NONE 77 | 78 | 79 | @service.route('/service', methods=['PATCH']) 80 | @has_secret 81 | def service_patch(service): 82 | fields = ['name', 'icon'] 83 | updated = False 84 | 85 | for field in fields: 86 | data = request.form.get(field, '').strip() 87 | if data != '': 88 | setattr(service, field, data) 89 | updated = True 90 | 91 | if updated: 92 | db.session.commit() 93 | return Error.NONE 94 | 95 | return Error.NO_CHANGES 96 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | from re import compile 2 | from json import dumps 3 | from functools import wraps 4 | 5 | from flask import request, jsonify 6 | 7 | from models import Service 8 | from shared import zmq_relay_socket 9 | 10 | uuid = compile(r'^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$') 11 | service = compile(r'^[a-zA-Z0-9]{4}-[a-zA-Z0-9]{6}-[a-zA-Z0-9]{12}-[a-zA-Z0-9]{5}-[a-zA-Z0-9]{9}$') 12 | is_uuid = lambda s: uuid.match(s) is not None 13 | is_service = lambda s: service.match(s) is not None 14 | is_secret = lambda s: compile(r'^[a-zA-Z0-9]{32}$').match(s) is not None 15 | 16 | QUERY_ACTION_NEW_MESSAGE = 0 17 | QUERY_UPDATE_LISTEN = 1 18 | 19 | 20 | class Error: 21 | @staticmethod 22 | def _e(message, error_code, http_status): 23 | return (dumps({'error': {'message': message, 'id': error_code}}), http_status) 24 | 25 | NONE = (dumps({'status': 'ok'}), 200) # OK 26 | INVALID_CLIENT = _e.__func__('Invalid client uuid', 1, 400) # Bad request 27 | INVALID_SERVICE = _e.__func__('Invalid service', 2, 400) # - || - 28 | INVALID_SECRET = _e.__func__('Invalid secret', 3, 400) # - || - 29 | DUPLICATE_LISTEN = _e.__func__('Already subscribed to that service', 4, 409) # Conflict 30 | RATE_TOOFAST = _e.__func__('Whoaw there cowboy, slow down!', 5, 429) # Too many requests 31 | SERVICE_NOTFOUND = _e.__func__('Service not found', 6, 404) 32 | INVALID_PUBKEY = _e.__func__('Invalid public key supplied. Please send a DER formatted base64 encoded key.', 8, 33 | 400) # Bad request 34 | CONNECTION_CLOSING = _e.__func__('Connection closing', 9, 499) # Client closed request 35 | NO_CHANGES = _e.__func__('No changes were made', 10, 400) # Bad request 36 | NOT_SUBSCRIBED = _e.__func__('Not subscribed to that service', 11, 409) # Conflict 37 | 38 | @staticmethod 39 | def ARGUMENT_MISSING(arg): 40 | return Error._e('Missing argument {}'.format(arg), 7, 400) # Bad request 41 | 42 | 43 | def has_uuid(f): 44 | @wraps(f) 45 | def df(*args, **kwargs): 46 | client = request.form.get('uuid', '') or request.args.get('uuid', '') 47 | if not client: 48 | return Error.ARGUMENT_MISSING('uuid') 49 | if not is_uuid(client): 50 | return Error.INVALID_CLIENT 51 | return f(*args, client=client, **kwargs) 52 | 53 | return df 54 | 55 | 56 | def has_service(f): 57 | @wraps(f) 58 | def df(*args, **kwargs): 59 | service = request.form.get('service', '') or request.args.get('service', '') 60 | if not service: 61 | return Error.ARGUMENT_MISSING('service') 62 | if not is_service(service): 63 | return Error.INVALID_SERVICE 64 | 65 | srv = Service.query.filter_by(public=service).first() 66 | if not srv: 67 | return Error.SERVICE_NOTFOUND 68 | return f(*args, service=srv, **kwargs) 69 | 70 | return df 71 | 72 | 73 | def has_secret(f): 74 | @wraps(f) 75 | def df(*args, **kwargs): 76 | secret = request.form.get('secret', '') or request.args.get('secret', '') 77 | if not secret: 78 | return Error.ARGUMENT_MISSING('secret') 79 | if not is_secret(secret): 80 | return Error.INVALID_SECRET 81 | 82 | srv = Service.query.filter_by(secret=secret).first() 83 | if not srv: 84 | return Error.SERVICE_NOTFOUND 85 | return f(*args, service=srv, **kwargs) 86 | 87 | return df 88 | 89 | 90 | def queue_zmq_message(message): 91 | zmq_relay_socket.send_string(message) 92 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | """ setting and getting the persistent configuration for pushfish-api server""" 2 | import sys 3 | import configparser 4 | import os 5 | import logging 6 | from typing import Type, TypeVar 7 | from collections import namedtuple 8 | import warnings 9 | 10 | import appdirs 11 | 12 | ConfigOption = namedtuple("ConfigOption", ["default", "type", "required", 13 | "envvar", "comment"]) 14 | 15 | APPNAME = "pushfish-api" 16 | _LOGGER = logging.getLogger(APPNAME) 17 | 18 | T = TypeVar("T", bound="Config") 19 | 20 | 21 | def construct_default_db_uri() -> str: 22 | dbpath = os.path.join(appdirs.user_data_dir(APPNAME), "pushfish-api.db") 23 | return "sqlite:///" + dbpath 24 | 25 | 26 | db_uri_comment = """#for mysql, use something like: 27 | #uri = 'mysql+pymysql://pushfish@localhost/pushfish_api?charset=utf8mb4'""" 28 | dispatch_zmq_comment = """#point zeromq_relay_uri at the zeromq pubsub socket for 29 | #the pushfish connectors """ 30 | server_debug_comment = """#set debug to 0 for production mode """ 31 | 32 | DEFAULT_VALUES = { 33 | "database": {"uri": ConfigOption(construct_default_db_uri, str, True, "PUSHFISH_DB", db_uri_comment)}, 34 | "dispatch": {"mqtt_broker_address": ConfigOption("", str, False, "MQTT_ADDRESS", None), 35 | "google_api_key": ConfigOption("", str, False, "PUSHFISH_GOOGLE_API_KEY", None), 36 | "google_gcm_sender_id": ConfigOption(123456789012, bool, True, "PUSHFISH_GCM_SENDER_ID", None), 37 | "zeromq_relay_uri": ConfigOption("", str, False, "PUSHFISH_ZMQ_RELAY_URI", dispatch_zmq_comment)}, 38 | "server": {"debug": ConfigOption(0, bool, False, "PUSHFISH_DEBUG", server_debug_comment)}} 39 | 40 | 41 | def call_if_callable(v, *args, **kwargs): 42 | """ if v is callable, call it with args and kwargs. If not, return v itself """ 43 | return v(*args, **kwargs) if callable(v) else v 44 | 45 | 46 | def get_config_file_path() -> str: 47 | """ 48 | gets a configuration file path for pushfish-api. 49 | 50 | First, the environment variable PUSHFISH_CONFIG will be checked. 51 | If that variable contains an invalid path, an exception is raised. 52 | 53 | If the variable is not set, the config file will be loaded from the 54 | platform specific standard config directory, e.g. 55 | 56 | on linux: ~/.config/pushfish-api/pushfish-api.cfg 57 | on Windows: C:\\Users\\user\\AppData\\Local\\pushfish-api\\pushfish-api.cfg 58 | on OSX: /Users/user/Library/Application Support/pushfish-api/pushfish-api.cfg 59 | 60 | The file is not created if it does not exist. 61 | 62 | """ 63 | 64 | # check environment variable first 65 | cfile = os.getenv("PUSHFISH_CONFIG") 66 | if not cfile: 67 | _LOGGER.info("PUSHFISH_CONFIG is not set, using default config file location") 68 | elif not os.path.exists(cfile): 69 | _LOGGER.warning("PUSHFISH_CONFIG file path does not exist, it will be created: %s", cfile) 70 | return cfile 71 | else: 72 | return cfile 73 | 74 | configdir = appdirs.user_config_dir(appname=APPNAME) 75 | return os.path.join(configdir, "pushfish-api.cfg") 76 | 77 | 78 | def write_default_config(path: str = None, overwrite: bool = False): 79 | """ writes out a config file with default options pre-loaded 80 | 81 | Arguments: 82 | path: the path for the config file to write. If not specified, 83 | calls get_config_file_path() to obtain a location 84 | 85 | overwrite: whether to overwrite an existing file. If False, and the 86 | path already exists, raises a RuntimeError 87 | 88 | """ 89 | if path is None: 90 | path = get_config_file_path() 91 | 92 | if os.path.exists(path): 93 | if not overwrite: 94 | errstr = "config file {} already exists. Not overwriting".format(path) 95 | _LOGGER.error(errstr) 96 | raise RuntimeError(errstr) 97 | else: 98 | _LOGGER.warning("overwriting existing config file %s with default", path) 99 | 100 | cfg = configparser.ConfigParser(allow_no_value=True) 101 | 102 | for section, settings in DEFAULT_VALUES.items(): 103 | cfg.add_section(section) 104 | for setting, value in settings.items(): 105 | v = call_if_callable(value.default) 106 | if value.comment is not None: 107 | cfg.set(section, value.comment) 108 | cfg[section][setting] = str(v) 109 | 110 | cfgdir = os.path.dirname(path) 111 | if not os.path.exists(cfgdir): 112 | if cfgdir: 113 | os.mkdir(cfgdir) 114 | with open(path, "x") as f: 115 | cfg.write(f) 116 | else: 117 | with open(path, "w") as f: 118 | cfg.write(f) 119 | 120 | 121 | class Config: 122 | """ reader for pushfish config file """ 123 | GLOBAL_INSTANCE = None 124 | GLOBAL_BACKTRACE_ENABLE = False 125 | 126 | @classmethod 127 | def get_global_instance(cls: Type[T]) -> T: 128 | """ returns the a global instance of the Config object. 129 | If one has not yet been defined, raises a RuntimeError""" 130 | if cls.GLOBAL_INSTANCE is None: 131 | raise RuntimeError("no global config instance exists. Construct a \ 132 | Config instance somewhere in the application") 133 | 134 | return cls.GLOBAL_INSTANCE 135 | 136 | def __init__(self, path: str = None, create: bool = False, 137 | overwrite: bool = False) -> None: 138 | """ 139 | arguments: 140 | path: path for config file. If not specified, calls get_default_config_path() 141 | create: create a default config file if it doesn't exist 142 | overwrite: overwrite the config file with the default even if it 143 | does already exist 144 | """ 145 | if not path: 146 | path = get_config_file_path() 147 | 148 | if not os.path.exists(path): 149 | if not create: 150 | errstr = "config file doesn't exist, and didn't pass create=True" 151 | _LOGGER.error(errstr) 152 | raise RuntimeError(errstr) 153 | 154 | _LOGGER.info("config file doesn't exist, creating it...") 155 | write_default_config(path=path, overwrite=False) 156 | elif overwrite: 157 | _LOGGER.warning("config file already exists, overwriting...") 158 | write_default_config(path=path, overwrite=True) 159 | 160 | self._cfg = configparser.ConfigParser() 161 | with open(path, "r") as f: 162 | self._cfg.read_file(f) 163 | 164 | # HACK: this is purely here so that the tests can override the global app 165 | # config 166 | if hasattr(self, "INJECT_CONFIG"): 167 | warnings.warn("running with injected config. If you see this \ 168 | whilst not running tests it IS AN ERROR") 169 | self = Config.GLOBAL_INSTANCE 170 | else: 171 | Config.GLOBAL_INSTANCE = self 172 | 173 | if self.debug: 174 | Config.GLOBAL_BACKTRACE_ENABLE = True 175 | 176 | self._check_spurious_keys() 177 | self._load_from_env_vars() 178 | 179 | def _load_from_env_vars(self): 180 | for section, optdict in DEFAULT_VALUES.items(): 181 | for name, opt in optdict.items(): 182 | envval = os.getenv(opt.envvar) 183 | if envval: 184 | _LOGGER.info("overriding config setting %s from environment variable %s", name, opt.envvar) 185 | try: 186 | self._cfg[section][name] = envval 187 | except ValueError as err: 188 | errstr = "couldn't get value of type %s for setting %s" 189 | fatal_error_exit_or_backtrace(err, errstr, _LOGGER, opt.type, name) 190 | except Exception as err: 191 | errstr = "failed to set value of setting %s from environment" 192 | fatal_error_exit_or_backtrace(err, errstr, _LOGGER, name) 193 | 194 | def _check_spurious_keys(self): 195 | for section in self._cfg.sections(): 196 | if section not in DEFAULT_VALUES: 197 | _LOGGER.critical("spurious section [%s] found in config file. ", section) 198 | _LOGGER.critical("don't know how to handle this, exiting...") 199 | sys.exit(1) 200 | 201 | for key in self._cfg[section].keys(): 202 | if key not in DEFAULT_VALUES[section]: 203 | _LOGGER.critical("spurious key %s in section [%s] found in config file. ", key, section) 204 | _LOGGER.critical("don't know how to handle this, exiting...") 205 | sys.exit(1) 206 | 207 | def _safe_get_cfg_value(self, section: str, key: str): 208 | opt = DEFAULT_VALUES[section][key] 209 | try: 210 | return opt.type(self._cfg[section][key]) 211 | except KeyError as err: 212 | reportstr = "no value for REQUIRED configuration option: %s in section [%s] defined" % (key, section) 213 | if opt.required: 214 | fatal_error_exit_or_backtrace(err, reportstr, _LOGGER) 215 | else: 216 | _LOGGER.warning(reportstr) 217 | defvalue = call_if_callable(opt.default) 218 | _LOGGER.warning("using default value of %s", str(defvalue)) 219 | return opt.type(defvalue) 220 | 221 | @property 222 | def database_uri(self) -> str: 223 | """ returns the database connection URI""" 224 | # HACK: create directory to run db IF AND ONLY IF it's identical to 225 | # default and doesn't exist. Please get rid of this with something 226 | # better soon 227 | val = self._safe_get_cfg_value("database", "uri") 228 | if val == construct_default_db_uri(): 229 | datadb = os.path.dirname(val).split("sqlite:///")[1] 230 | if not os.path.exists(datadb): 231 | try: 232 | os.mkdir(datadb) 233 | except PermissionError as err: 234 | errstr = "can't create default database directory. Exiting..." 235 | fatal_error_exit_or_backtrace(err, errstr, _LOGGER) 236 | return val 237 | 238 | @property 239 | def mqtt_broker_address(self) -> str: 240 | """ returns MQTT server address""" 241 | return self._safe_get_cfg_value("dispatch", "mqtt_broker_address") 242 | 243 | @property 244 | def google_api_key(self) -> str: 245 | """ returns google API key for gcm""" 246 | return self._safe_get_cfg_value("dispatch", "google_api_key") 247 | 248 | @property 249 | def google_gcm_sender_id(self) -> int: 250 | """ returns sender id for gcm""" 251 | return self._safe_get_cfg_value("dispatch", "google_gcm_sender_id") 252 | 253 | @property 254 | def zeromq_relay_uri(self) -> str: 255 | """ returns relay URI for zeromq dispatcher""" 256 | return self._safe_get_cfg_value("dispatch", "zeromq_relay_uri") 257 | 258 | @property 259 | def debug(self) -> bool: 260 | """ returns desired debug state of application. 261 | Overridden by the value of environment variable FLASK_DEBUG """ 262 | if int(os.getenv("FLASK_DEBUG", "0")): 263 | return True 264 | return self._safe_get_cfg_value("server", "debug") 265 | 266 | 267 | def fatal_error_exit_or_backtrace(err: Exception, 268 | msg: str, 269 | logger: logging.Logger, 270 | *logargs, **logkwargs): 271 | """ standard handling of fatal errors. Logs a critical error, then, if 272 | debug mode is enabled, rethrows the error (to get a backtrace or debug), 273 | and if not, exits the program with return code 1 274 | 275 | arguments: 276 | err: the exception that caused this situation. Can be None, in which case 277 | will not be re-raised 278 | 279 | msg: the message you want to log 280 | logger: the logger to log to. Can be None, in which case a default logger 281 | will be obtained 282 | 283 | logargs, logkwargs: arguments to pass on to the logging function 284 | 285 | """ 286 | if logger is None: 287 | logger = logging.getLogger("pushfish-api") 288 | 289 | logger.critical(msg, *logargs, **logkwargs) 290 | logger.critical("exiting...") 291 | if Config.GLOBAL_BACKTRACE_ENABLE: 292 | if err is not None: 293 | raise err 294 | sys.exit(1) 295 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from __future__ import unicode_literals 3 | 4 | import os 5 | from uuid import uuid4 6 | import unittest 7 | import string 8 | import random 9 | import json 10 | import logging 11 | from time import sleep 12 | from ast import literal_eval 13 | import paho.mqtt.client as mqtt_api 14 | 15 | from config import Config 16 | 17 | _LOGGER = logging.getLogger("pushfish-api-TESTS") 18 | 19 | 20 | def _random_str(length=10, unicode=True): 21 | # A random string with the "cupcake" in Japanese appended to it 22 | # Always make sure that there is some unicode in there 23 | random_str = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) 24 | 25 | if unicode: 26 | random_str = random_str[:-7] + 'カップケーキ' 27 | # It's important that the following is a 4+-byte Unicode character. 28 | random_str = '😉' + random_str 29 | 30 | return random_str 31 | 32 | 33 | def _failing_loader(s): 34 | data = json.loads(s) 35 | if 'error' in data: 36 | err = data['error'] 37 | raise AssertionError("Got an unexpected error, [{}] {}".format(err['id'], err['message'])) 38 | 39 | return data 40 | 41 | 42 | _messages_received = [] 43 | 44 | 45 | def _message_callback(client, userdata, message): 46 | """ 47 | mqtt subscribe callback function 48 | puts received messages in _messages_received 49 | """ 50 | message = {"data": literal_eval(message.payload.decode("utf-8")), "topic": message.topic, "qos": message.qos, 51 | "retain": message.retain} 52 | _messages_received.append(message) 53 | 54 | 55 | # NOTE: don't inherit these from unittest.TestCase, inherit the specialized 56 | # database classes that way, then they both get run 57 | class PushFishTestCase(unittest.TestCase): 58 | def setUp(self): 59 | self.uuid = str(uuid4()) 60 | from application import app 61 | cfg = Config.get_global_instance() 62 | 63 | app.config['TESTING'] = True 64 | app.config['TESTING_GCM'] = [] 65 | self.gcm_enable = True 66 | if not cfg.google_api_key: 67 | _LOGGER.warning("GCM API key is not provided, won't test GCM") 68 | self.gcm_enable = False 69 | self.mqtt_enable = True 70 | self.mqtt_address = cfg.mqtt_broker_address 71 | if not cfg.mqtt_broker_address: 72 | _LOGGER.warning("MQTT broker address is not provided, won't test MQTT") 73 | self.mqtt_enable = False 74 | 75 | self.gcm = app.config['TESTING_GCM'] 76 | self.app = app.test_client() 77 | self.app_real = app 78 | 79 | def test_service_create(self): 80 | name = "Hello test! {}".format(_random_str(5)) 81 | data = { 82 | "name": name, 83 | "icon": "http://i.imgur.com/{}.png".format(_random_str(7, False)) 84 | } 85 | rv = json.loads(self.app.post('/service', data=data).data) 86 | assert 'service' in rv 87 | return rv['service']['public'], rv['service']['secret'], name 88 | 89 | def test_subscription_new(self): 90 | public, secret, _ = self.test_service_create() 91 | data = dict(uuid=self.uuid, service=public) 92 | rv = self.app.post('/subscription', data=data) 93 | _failing_loader(rv.data) 94 | return public, secret 95 | 96 | def test_subscription_double(self): 97 | public, _ = self.test_subscription_new() 98 | data = dict(uuid=self.uuid, service=public) 99 | rv = self.app.post('/subscription', data=data) 100 | assert rv.status_code == 409 101 | data = json.loads(rv.data) 102 | assert 'error' in data 103 | assert data['error']['id'] == 4 104 | 105 | def test_subscription_delete(self): 106 | public, secret = self.test_subscription_new() 107 | rv = self.app.delete('/subscription?uuid={}&service={}'.format(self.uuid, public)) 108 | _failing_loader(rv.data) 109 | return public, secret 110 | 111 | def test_subscription_invalid_delete(self): 112 | # Without a just-deleted service there's a chance to get an existing 113 | # one, as a test database isn't created when running tests. 114 | public, _ = self.test_subscription_delete() 115 | rv = self.app.delete('/subscription?uuid={}&service={}'.format(self.uuid, public)) 116 | assert rv.status_code == 409 117 | data = json.loads(rv.data) 118 | assert 'error' in data 119 | assert data['error']['id'] == 11 120 | 121 | def test_subscription_list(self): 122 | public, _ = self.test_subscription_new() 123 | rv = self.app.get('/subscription?uuid={}'.format(self.uuid)) 124 | resp = _failing_loader(rv.data) 125 | assert 'subscriptions' in resp 126 | assert len(resp['subscriptions']) == 1 127 | assert resp['subscriptions'][0]['service']['public'] == public 128 | 129 | def test_message_send(self, public='', secret=''): 130 | if not public or not secret: 131 | public, secret = self.test_subscription_new() 132 | data = { 133 | "level": random.randint(0, 5), 134 | "message": "Test message - {}".format(_random_str(20)), 135 | "title": "Test Title - {}".format(_random_str(5)), 136 | "secret": secret, 137 | } 138 | rv = self.app.post('/message', data=data) 139 | _failing_loader(rv.data) 140 | return public, secret, data 141 | 142 | def test_message_send_no_subscribers(self): 143 | # We just want to know if the server "accepts" it 144 | public, secret, _ = self.test_service_create() 145 | self.test_message_send(public, secret) 146 | 147 | def test_message_receive(self, amount=-1): 148 | if amount <= 0: 149 | self.test_message_send() 150 | amount = 1 151 | 152 | rv = self.app.get('/message?uuid={}'.format(self.uuid)) 153 | resp = _failing_loader(rv.data) 154 | assert len(resp['messages']) is amount 155 | 156 | # Ensure it is marked as read 157 | rv = self.app.get('/message?uuid={}'.format(self.uuid)) 158 | resp = _failing_loader(rv.data) 159 | assert len(resp['messages']) is 0 160 | 161 | def test_message_receive_no_subs(self): 162 | self.test_message_send() 163 | rv = self.app.get('/message?uuid={}'.format(uuid4())) 164 | resp = _failing_loader(rv.data) 165 | assert len(resp['messages']) is 0 166 | 167 | def test_message_receive_multi(self): 168 | self.test_message_mark_read() 169 | 170 | for _ in range(3): 171 | public, secret = self.test_subscription_new() 172 | for _ in range(5): 173 | self.test_message_send(public, secret) 174 | 175 | self.test_message_receive(15) 176 | 177 | def test_message_mark_read(self): 178 | self.test_message_send() 179 | self.app.delete('/message?uuid={}'.format(self.uuid)) 180 | rv = self.app.get('/message?uuid={}'.format(self.uuid)) 181 | resp = _failing_loader(rv.data) 182 | assert len(resp['messages']) == 0 183 | 184 | def test_message_mark_read_double(self): 185 | self.test_message_mark_read() 186 | 187 | # Read again without sending 188 | self.app.delete('/message?uuid={}'.format(self.uuid)) 189 | rv = self.app.get('/message?uuid={}'.format(self.uuid)) 190 | resp = _failing_loader(rv.data) 191 | assert not resp['messages'] 192 | 193 | def test_message_mark_read_multi(self): 194 | # Stress test it a bit 195 | for _ in range(3): 196 | public, secret = self.test_subscription_new() 197 | for _ in range(5): 198 | self.test_message_send(public, secret) 199 | 200 | self.test_message_mark_read() 201 | 202 | def test_service_delete(self): 203 | public, secret = self.test_subscription_new() 204 | # Send a couple of messages, these should be deleted 205 | for _ in range(10): 206 | self.test_message_send(public, secret) 207 | 208 | rv = self.app.delete('/service?secret={}'.format(secret)) 209 | _failing_loader(rv.data) 210 | 211 | # Does the service not exist anymore? 212 | rv = self.app.get('/service?service={}'.format(public)) 213 | assert 'error' in json.loads(rv.data) 214 | 215 | # Has the subscriptioner been deleted? 216 | rv = self.app.get('/subscription?uuid={}'.format(self.uuid)) 217 | resp = _failing_loader(rv.data) 218 | assert public not in [l['service']['public'] for l in resp['subscriptions']] 219 | 220 | # check we on't receive the message anymore 221 | rv = self.app.get('/message?uuid={}'.format(self.uuid)) 222 | resp = _failing_loader(rv.data) 223 | assert not resp["messages"] 224 | 225 | def test_service_info(self): 226 | public, _, name = self.test_service_create() 227 | rv = self.app.get('/service?service={}'.format(public)) 228 | data = _failing_loader(rv.data) 229 | assert 'service' in data 230 | srv = data['service'] 231 | assert srv['name'] == name 232 | assert srv['public'] == public 233 | 234 | def test_service_info_secret(self): 235 | public, secret, name = self.test_service_create() 236 | rv = self.app.get('/service?secret={}'.format(secret)) 237 | data = _failing_loader(rv.data) 238 | assert 'service' in data 239 | srv = data['service'] 240 | assert srv['name'] == name 241 | assert srv['public'] == public 242 | 243 | def test_service_update(self): 244 | public, secret, _ = self.test_service_create() 245 | data = { 246 | "name": _random_str(10), 247 | "icon": "http://i.imgur.com/{}.png".format(_random_str(7, False)) 248 | } 249 | rv = self.app.patch('/service?secret={}'.format(secret), data=data).data 250 | _failing_loader(rv) 251 | 252 | # Test if patched 253 | rv = self.app.get('/service?service={}'.format(public)) 254 | rv = _failing_loader(rv.data)['service'] 255 | for key in data.keys(): 256 | assert data[key] == rv[key] 257 | 258 | def test_uuid_regex(self): 259 | rv = self.app.get('/service?service={}'.format(_random_str(20))).data 260 | assert 'error' in json.loads(rv) 261 | 262 | def test_service_regex(self): 263 | rv = self.app.get('/message?uuid={}'.format(_random_str(20))).data 264 | assert 'error' in json.loads(rv) 265 | 266 | def test_missing_arg(self): 267 | rv = json.loads(self.app.get('/message').data) 268 | assert 'error' in rv and rv['error']['id'] == 7 269 | rv = json.loads(self.app.get('/service').data) 270 | assert 'error' in rv and rv['error']['id'] == 7 271 | 272 | def test_gcm_register(self): 273 | if self.gcm_enable: 274 | reg_id = _random_str(40, unicode=False) 275 | data = {'uuid': self.uuid, 'regId': reg_id} 276 | rv = self.app.post('/gcm', data=data).data 277 | _failing_loader(rv) 278 | return reg_id 279 | else: 280 | _LOGGER.warning("GCM is disabled, not testing gcm_register") 281 | 282 | def test_gcm_unregister(self): 283 | if self.gcm_enable: 284 | self.test_gcm_register() 285 | rv = self.app.delete('/gcm', data={'uuid': self.uuid}).data 286 | _failing_loader(rv) 287 | else: 288 | _LOGGER.warning("GCM is disabled, not testing gcm_unregister") 289 | 290 | def test_gcm_register_double(self): 291 | if self.gcm_enable: 292 | self.test_gcm_register() 293 | self.test_gcm_register() 294 | else: 295 | _LOGGER.warning("GCM is disabled, not testing gcm_register_double") 296 | 297 | def test_gcm_send(self): 298 | if self.gcm_enable: 299 | reg_id = self.test_gcm_register() 300 | public, _, data = self.test_message_send() 301 | 302 | messages = [m['data'] for m in self.gcm 303 | if reg_id in m['registration_ids']] 304 | 305 | assert len(messages) is 1 306 | assert messages[0]['encrypted'] is False 307 | 308 | message = messages[0]['message'] 309 | assert message['service']['public'] == public 310 | assert message['message'] == data['message'] 311 | else: 312 | _LOGGER.warning("GCM is disabled, not testing gcm_send") 313 | 314 | def test_mqtt_register(self): 315 | if self.mqtt_enable: 316 | data = {'uuid': self.uuid} 317 | rv = self.app.post('/mqtt', data=data).data 318 | _failing_loader(rv) 319 | else: 320 | _LOGGER.warning("MQTT is disabled, not testing mqtt_register") 321 | 322 | def test_mqtt_unregister(self): 323 | if self.mqtt_enable: 324 | self.test_mqtt_register() 325 | rv = self.app.delete('/mqtt', data={'uuid': self.uuid}).data 326 | _failing_loader(rv) 327 | else: 328 | _LOGGER.warning("MQTT is disabled, not testing mqtt_unregister") 329 | 330 | def test_mqtt_register_double(self): 331 | if self.mqtt_enable: 332 | self.test_mqtt_register() 333 | self.test_mqtt_unregister() 334 | else: 335 | _LOGGER.warning("MQTT is disabled, not testing MQTT_register_double") 336 | 337 | def test_mqtt_send(self): 338 | """ 339 | test if message pushed to PushFish can be received from the mqtt broker 340 | """ 341 | if self.mqtt_enable: 342 | self.test_mqtt_register() 343 | 344 | url = self.mqtt_address 345 | if ":" in url: 346 | port = url.split(":")[1] 347 | url = url.split(":")[0] 348 | else: 349 | # default port 350 | port = 1883 351 | 352 | client = mqtt_api.Client() 353 | client.connect(url, port, 60) 354 | 355 | client.subscribe(self.uuid) 356 | client.on_message = _message_callback 357 | client.loop_start() 358 | public, _, data = self.test_message_send() 359 | sleep(2) 360 | client.loop_stop() 361 | client.disconnect() 362 | 363 | assert len(_messages_received) is 1 364 | 365 | mqtt_data = _messages_received[0] 366 | assert mqtt_data['topic'] == self.uuid 367 | message = mqtt_data['data']['message'] 368 | assert message['service']['public'] == public 369 | assert message['message'] == data['message'] 370 | else: 371 | _LOGGER.warning("MQTT is disabled, not testing mqtt_send") 372 | 373 | # def test_get_version(self): 374 | # version = self.app.get('/version').data 375 | # 376 | # assert len(version) is 7 377 | # with open('.git/refs/heads/master', 'r') as f: 378 | # assert f.read()[:7] == version 379 | 380 | def test_get_static(self): 381 | files = ['robots.txt', 'favicon.ico'] 382 | 383 | for f in files: 384 | path = os.path.join(self.app_real.root_path, 'static', f) 385 | with open(path, 'rb') as i: 386 | data = self.app.get('/{}'.format(f)).data 387 | assert data == i.read() 388 | 389 | 390 | def load_tests(loader, standard_tests, pattern): 391 | suite = unittest.TestSuite() 392 | test_class = PushFishTestCase 393 | tests = loader.loadTestsFromTestCase(test_class) 394 | suite.addTests(tests) 395 | return suite 396 | 397 | 398 | if __name__ == "__main__": 399 | unittest.main() 400 | --------------------------------------------------------------------------------