├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── ansible ├── deploy.yaml ├── install-app.yaml ├── install-caddy.yml ├── install-db.yaml ├── install-redis.yaml ├── myhosts.ini └── vars.yml ├── build.sh ├── conf ├── caddy.conf ├── favicon.ico ├── loginscript.sh ├── pydaemon.service ├── robots.txt ├── server-config-localdev.json ├── server-config.json └── uwsgi.ini ├── migrations ├── 001_users.py └── 002_movies.py ├── migrations_sqlite ├── 001_init.py └── 002_movies.py ├── py ├── account.py ├── api_account.py ├── api_dev.py ├── api_movies.py ├── bgtasks.py ├── config.py ├── cron.py ├── db.py ├── main.py ├── mule1.py ├── red.py ├── ui_auth.py ├── util.py └── webutil.py ├── requirements.txt ├── rsync.sh ├── run.sh ├── scripts └── dbmigrate.py ├── shell.sh ├── templates ├── auth.html └── example.html ├── test ├── api-list.jpg ├── auth.jpg ├── quick.sh ├── sample.log.txt ├── test_api.py └── test_redis.py └── www └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .DS_Store 3 | *.log 4 | *.db 5 | /data/ 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bullseye 2 | 3 | WORKDIR /app 4 | 5 | # uwsgi must be compiled - install necessary build tools, compile uwsgi 6 | # and then remove the build tools to minimize image size 7 | # (buildDeps are removed, deps are kept) 8 | RUN set -ex \ 9 | && buildDeps=' \ 10 | build-essential \ 11 | ' \ 12 | && deps=' \ 13 | htop \ 14 | ' \ 15 | && apt-get update && apt-get install -y $buildDeps $deps --no-install-recommends && rm -rf /var/lib/apt/lists/* \ 16 | && pip install uWSGI==2.0.24 \ 17 | && apt-get purge -y --auto-remove $buildDeps \ 18 | && find /usr/local -depth \ 19 | \( \ 20 | \( -type d -a -name test -o -name tests \) \ 21 | -o \ 22 | \( -type f -a -name '*.pyc' -o -name '*.pyo' \) \ 23 | \) -exec rm -rf '{}' + 24 | 25 | # install other py libs - not require compilation 26 | COPY requirements.txt /app/requirements.txt 27 | RUN pip install -r /app/requirements.txt 28 | 29 | # copy source files 30 | COPY conf /app/conf 31 | COPY py /app/py 32 | COPY migrations /app/migrations 33 | COPY migrations_sqlite /app/migrations_sqlite 34 | COPY scripts /app/scripts 35 | COPY templates /app/templates 36 | COPY test /app/test 37 | COPY conf/loginscript.sh /etc/profile 38 | 39 | # background spooler dir 40 | RUN mkdir /tmp/pysrv_spooler 41 | 42 | # we don't need this file with Docker (autoload is enabled) but uwsgi looks for it 43 | RUN echo `date +%s` >/app/RESTART 44 | 45 | EXPOSE 80 46 | 47 | 48 | # our server config file 49 | # - you should write your own config file and put OUTSIDE the repository 50 | # since the config file contains secrets 51 | # - here I use the sample template from repo 52 | # - it is also possible to override the config with env variables, either here 53 | # or in Amazon ECS or Kubernetes configuration 54 | # COPY conf/server-config-localdev.json /app/real-server-config.json 55 | # ENV PYSRV_DATABASE_HOST host.docker.internal 56 | # ENV PYSRV_REDIS_HOST host.docker.internal 57 | # ENV PYSRV_DATABASE_PASSWORD x 58 | 59 | # build either a production or dev image 60 | ARG BUILDMODE=production 61 | ENV ENVBUILDMODE=$BUILDMODE 62 | 63 | RUN echo "BUILDMODE $ENVBUILDMODE" 64 | 65 | # run in shell mode with ENV expansion 66 | CMD uwsgi --ini /app/conf/uwsgi.ini:uwsgi-$ENVBUILDMODE 67 | 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tomi Mickelsson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | RESTPie3 - Python REST API Server Starter Kit 3 | ============================================= 4 | 5 | This is a lightweight python3 REST API server that offers 6 | essential web service features in a simple package. This is not a framework, 7 | just **a practical and clean codebase that relies on a few core components** 8 | that do the job well. Fork and create your own REST API server quickly. 9 | 10 | Open sourced on Sep 2018 after years of production use at multiple sites. 11 | 12 | Update Sep 2020: Run in Raspberry with an SQLite database. 13 | 14 | Update May 2023: Python and libraries updated, Python 3.11 into use. Still a fine foundation for a new project - the architecture does not age, and simple outlives complex. 15 | 16 | Update March 2024: Ansible scripts for automatic install to cloud. 17 | 18 | **Table of contents** 19 | 20 | * [Features](#features) 21 | * [Building blocks](#building-blocks) 22 | * [Source files](#source-files) 23 | * [Run locally with Docker](#run-locally-with-docker) 24 | * [Develop locally with Docker](#develop-locally-with-docker) 25 | * [API methods](#api-methods) 26 | * [Authentication & authorization](#authentication--authorization) 27 | * [Session data](#session-data) 28 | * [Redis storage](#redis-storage) 29 | * [Background workers & cron](#background-workers--cron) 30 | * [Mules: extra servers](#mules) 31 | * [Logging](#logging) 32 | * [Tests](#tests) 33 | * [Deploy to cloud](#deploy-to-cloud) 34 | * [Security](#security) 35 | * [Scaling up](#scaling-up) 36 | * [Run in Raspberry](#run-in-raspberry) 37 | * [What about the front-end?](#what-about-the-front-end) 38 | * [Need help?](#need-help) 39 | * [License](#license) 40 | * [Screenshot API list](#screenshot) 41 | * [Screenshot Auth](#screenshot2) 42 | 43 | 44 | Features 45 | -------- 46 | 47 | A quick list of the features of this Python API server: 48 | 49 | * Simple and flexible server with minimum dependencies 50 | * Process-based request workers, not thread-based nor async 51 | * Secure server-side sessions with Redis storage 52 | * Robust worker management: restarts, timecapping, max life 53 | * Background tasks 54 | * Built-in cron 55 | * Automatic directory page listing the API methods and their docstrings [Screenshot](#screenshot) 56 | * Redis as a generic storage with expiring keys, lightweight queues 57 | * Email & password authentication with secure algorithms 58 | * User role model and authorization of API methods via simple decorator 59 | * Logging system with practical data for troubleshooting, detects slow 60 | requests, warnings&errors colorized 61 | * Server reload on code change 62 | * Database ORM and migrations 63 | * Database init schemas for PostgreSQL and SQLite 64 | * Docker image for the "big cloud" and local development 65 | * Fast rsync deployment of updates to Linux VPS servers 66 | * Tests for the API 67 | * Raspberry compatible 68 | * Simple UI for login/signup/forgot/reset password [Screenshot](#screenshot2) 69 | 70 | 71 | Building blocks 72 | --------------- 73 | 74 | The core building blocks of this server are mature open-source components that 75 | I have years of experience of. 76 | 77 | * [Python](http://python.org) is a high-level and versatile scripting language 78 | that provides powerful features with an exceptionally clear syntax. 79 | 80 | The language is well designed and has received increased fame and 81 | popularity over the recent years. Huge number of developers are picking 82 | Python in their work. In Sep 2017, a StackOverflow study writes about [The Incredible 83 | Growth of Python](https://stackoverflow.blog/2017/09/06/incredible-growth-python/): 84 | "Python has a solid claim to being the fastest-growing major programming 85 | language." In the [TIOBE index](http://www.tiobe.com/tiobe-index/) 86 | Python stands at position 3 as of Sep 2018. 87 | 88 | Having been coding Python professionally for close to two decades, I can say 89 | it has boosted my productivity and still is my primary language for many 90 | tasks, including developing the business logic in the back-end. 91 | 92 | * [Flask](http://flask.pocoo.org/) is the Python web framework. Flask is 93 | considered as an unopinionated micro-framework that only provides the 94 | essentials of a web framework without enforcing other components (like 95 | database, orm, admin interface, sessions etc.) As a webdev veteran I 96 | appreciate this flexibility since I do want to pick the best of breed 97 | components myself. The core stays but other needs may vary from project to 98 | project, from Raspberry to the AWS cloud. The flexibility lets me be in 99 | control and simplify. 100 | 101 | * [uwsgi](https://uwsgi-docs.readthedocs.io/en/latest/) is the master daemon 102 | that runs and supervises the Python worker processes. uwsgi has a list of 103 | power features that are essential to a robust back-end: timecapped requests, 104 | recycling of workers, background tasks, cron jobs, timers, logging, auto 105 | reloads on code change, run-as privileges. uwsgi is configured via the 106 | [uwsgi.ini](conf/uwsgi.ini) file. 107 | 108 | * [PostgreSQL](http://postgresql.org) is the main database, "the most advanced 109 | open source database" that is getting stronger every year. PostgreSQL is a 110 | powerhouse of features with a rock-solid implementation. Personally I enjoy 111 | the JSON functionality the most, since it provides good amount of 112 | flexibility to the relational model that I still prefer in a master database 113 | over the schema-free solutions. 114 | 115 | Someone wrote an article saying [PostgreSQL is the worlds' best database](https://www.2ndquadrant.com/en/blog/postgresql-is-the-worlds-best-database/). 116 | 117 | Note that the code also supports [SQLite](https://www.sqlite.org/index.html) 118 | database. SQLite maybe convenient in a lighter setup if the full power of 119 | PostgreSQL is not needed such as in a Raspberry. 120 | 121 | * [Redis](https://redis.io/) is a persistent in-memory database that is used 122 | as a storage for server-side session data and as a lightweight caching and 123 | queueing system. Fast and solid. 124 | 125 | * [Peewee](http://docs.peewee-orm.com/en/latest/) is a straightforward 126 | database ORM library. It is small and easy to learn, and has all the 127 | necessary features. I favor the simplicity of the ActiveRecord pattern with 128 | 1-1 mapping of classes and database, as opposed to more complex data mapper 129 | pattern that is followed by the big 130 | [SQLAlchemy](https://www.sqlalchemy.org/) library. I know SQL and like to 131 | operate at the row level, and have explicit control. 132 | Peewee makes database access a breeze and allows you to execute raw 133 | SQL if you need the full power of the database. Peewee supports SQLite, 134 | MySQL and PostgreSQL. 135 | 136 | For scheme migrations, 137 | [Peewee-migrate](https://github.com/klen/peewee_migrate) is an easy choice 138 | that fits well with Peewee. 139 | 140 | 141 | If you'd like to replace some of these components, it is possible, this is a 142 | small codebase. 143 | 144 | 145 | Source files 146 | ------------ 147 | 148 | The whole of this server fits into a small set of files: 149 | 150 | ``` 151 | ├── /ansible/ # ansible files for automated cloud install 152 | ├── /conf/ # configuration files 153 | │ ├── /caddy.conf # config for Caddy www proxy that ansible setups 154 | │ ├── /favicon.ico # site icon 155 | │ ├── /loginscript.sh # docker shell login script, sets paths 156 | │ ├── /pydaemon.service # systemd daemon config 157 | │ ├── /robots.txt # deny all from robots 158 | │ ├── /server-config-localdev.json # main server config for localdev in docker 159 | │ ├── /server-config.json # main server config, ansible fills and copies to cloud 160 | │ └── /uwsgi.ini # uwsgi daemon config, for localdev & server 161 | ├── /migrations/ # db migrations - postgresql 162 | │ ├── /001_users.py # users table, the foundation 163 | │ └── /002_movies.py # movies table, just as an example 164 | ├── /migrations_sqlite/ # db migrations - sqlite 165 | │ ├── /001_init.py # users table, the foundation 166 | │ └── /002_movies.py # movies table, just as an example 167 | ├── /py/ # python modules 168 | │ ├── /account.py # account related: passwords, user session 169 | │ ├── /api_account.py # API-endpoints for login/signup/logout 170 | │ ├── /api_dev.py # API-endpoints for dev/testing/api list 171 | │ ├── /api_movies.py # API-endpoints for movies basic crud 172 | │ ├── /bgtasks.py # background tasks, 1 sample method 173 | │ ├── /config.py # central config for the app 174 | │ ├── /cron.py # cron methods: run every min, hourly, daily 175 | │ ├── /db.py # database classes and methods 176 | │ ├── /main.py # server main 177 | │ ├── /red.py # redis: get/set keyvals, lists, atomic 178 | │ ├── /ui_auth.py # quick auth pages 179 | │ ├── /util.py # misc utility functions 180 | │ └── /webutil.py # core web flow: before/after request, auth, role check 181 | ├── /scripts/ # scripts 182 | │ └── /dbmigrate.py # db migration 183 | ├── /templates/ # templates (if you really need them) 184 | │ ├── /auth.html # login/signup form 185 | │ └── /example.html # very basic jinja2 example 186 | ├── /test/ # test scripts 187 | │ ├── /quick.sh # quick adhoc curl example 188 | │ ├── /test_api.py # test API methods 189 | │ ├── /test_redis.py # test redis module 190 | │ └── /sample.log.txt # sample logging output from api test 191 | ├── /www/ # static files for Caddy 192 | ├── build.sh # build Docker image in dev mode 193 | ├── Dockerfile # docker image config 194 | ├── requirements.txt # python 3rd party dependencies 195 | ├── rsync.sh # rsync sources to server and reload (instead of ansible) 196 | ├── run.sh # run server locally with Docker in dev mode 197 | └── shell.sh # run interactive shell inside docker instance 198 | ``` 199 | 200 | So how do you get started with your own project? I suggest to take this route: 201 | 202 | * browse briefly the source files, understand their role 203 | * read and throw away `002_movies.py` and `api_movies.py`, they exist only as 204 | a sample 205 | * discard `cron.py, bgtasks.py` if you don't need background processing 206 | * discard `templates` if you only create an API server 207 | * write your own business logic: 208 | * create data classes and methods in `db.py or db_x.py` 209 | * create API modules in `api_x.py` 210 | * create database migrations 211 | * create tests 212 | 213 | 214 | Run locally with Docker 215 | ----------------------- 216 | 217 | RESTPie3 is easy to run locally via Docker. 218 | The base image is an [official python image](https://hub.docker.com/_/python) 219 | variant **python:3.11-slim-bullseye**. 220 | 221 | If you already have Docker installed, the quick steps to run RESTPie3 with 222 | SQLite and Redis are: 223 | 224 | docker pull redis:5 225 | 226 | # create + start the redis instance 227 | docker run -d --name redis -p 6379:6379 redis:5 228 | 229 | # download and build RESTPie3 230 | git clone https://github.com/tomimick/restpie3 231 | cd restpie3 232 | ./build.sh 233 | 234 | # start RESTPie3 235 | ./run.sh 236 | 237 | # in another term, create initial database schema 238 | docker exec -it restpie-dev bash -l -c 'python /app/scripts/dbmigrate.py' 239 | 240 | 241 | If all went OK, RESTPie3, SQLite and Redis are running and you should be able to list 242 | the REST API at http://localhost:8100/api/list 243 | 244 | The SQLite database is empty at this point so empty lists are returned from 245 | the API. You are also logged out so some of the API end-points can't be 246 | accessed. To quickly test the API, you can invoke this example script which 247 | uses curl to do a signup and insert a new movie in the database: 248 | 249 | ./test/quick.sh 250 | 251 | For a serious setup you want to have full PostgreSQL. Do the setup like this: 252 | 253 | docker pull postgres:15 254 | 255 | # create + start a postgres instance - use your own db + password! 256 | # the params here must match the ones in conf/server-config-localdev.json 257 | docker run -d --name pos-restpie -p 5432:5432 -e POSTGRES_DB=tmdb -e POSTGRES_USER=tm -e POSTGRES_PASSWORD=MY_PASSWORD postgres:12 258 | 259 | # activate the uuid extension 260 | docker exec -it pos-restpie psql -U tm -d tmdb -c 'create extension "uuid-ossp"' 261 | 262 | # and then in server-config-localdev.json 263 | # set PYSRV_DATABASE_HOST (see PYSRV_DATABASE_HOST_POSTGRESQL) 264 | 265 | To start and stop these docker instances, invoke: 266 | 267 | docker start redis 268 | docker start pos-restpie 269 | docker start restpie-dev 270 | docker stop redis 271 | docker stop pos-restpie 272 | docker stop restpie-dev 273 | 274 | 275 | 276 | Develop locally with Docker 277 | --------------------------- 278 | 279 | Docker is great for packaging software to be run in the cloud, but it is also 280 | beneficial while developing the software. With Docker you can isolate and play 281 | easily with different dev environments and services without installing 282 | anything on the local machine and without facing ugly local version conflicts. 283 | Running the same docker image locally also ensures the environment is 284 | identical to the release environment, which makes a lot of sense. 285 | 286 | ./run.sh 287 | 288 | The above command runs the dev instance in the foreground so you are able to 289 | see the logging output in the console and detect errors immediately. You can 290 | stop the server with CTRL+C. When the instance ends, its data is deleted (the 291 | --rm option) - this is good as we don't want to create a long list of dangling 292 | temporary instances. 293 | 294 | Now the COOL thing in the dev mode here is that we are using Docker volumes to 295 | map a local root folder containing all source files to `/app/` folder 296 | inside the Docker instance. This makes it possible to use any local file 297 | editor to edit the python sources and when a file is saved, the server inside 298 | the Docker instance reloads itself automatically! 299 | 300 | To see the executed SQL statements of the server in the console, you can set 301 | the PYSRV_LOG_SQL env variable: 302 | 303 | docker run -it --rm --name restpie-dev -p 8100:80 -v `pwd`/:/app/ -e PYSRV_LOG_SQL=1 restpie-dev-image 304 | 305 | 306 | If you want to run a shell inside the dev instance, invoke in another terminal 307 | session, while dev instance is running: 308 | 309 | docker exec -it restpie-dev bash -l 310 | 311 | # or just 312 | ./shell.sh 313 | 314 | # see files in the instance file system 315 | ls 316 | ll 317 | 318 | # see running processes 319 | htop 320 | 321 | # run python files 322 | python scripts/something.py 323 | 324 | You can modify the [login script](conf/loginscript.sh) to set paths and 325 | aliases etc for this interactive shell. 326 | 327 | 328 | API methods 329 | ----------- 330 | 331 | The available API methods are implemented in api_x.py modules: 332 | 333 | * `api_account.py` contains the core email/password login/signup/logout 334 | methods that you most likely need anyway. 335 | * `api_dev.py` contains misc methods for testing and developing which you can 336 | discard after learning the mechanics. 337 | * `api_movies.py` is just a sample module to demonstrate a basic CRUD REST 338 | API. You definately want to discard this and transform into your actual 339 | data models - just read and learn it. 340 | 341 | The server has built-in little [introspection](#screenshot) for listing the available APIs as 342 | a HTML page. You just declare, implement and document the API methods 343 | normally with the Flask decorator, docstrings and the methods will be 344 | automatically listed at 345 | [localhost:8100/api/list](http://localhost:8100/api/list). This is a neat way 346 | to document your server API. You can decide whether you want to disable this 347 | functionality in production or not. 348 | 349 | To parse and serialize JSON data I am simply using the Flask and Peewee 350 | primitives: `jsonify, JSONEncoder and Peewee 351 | model_to_dict/dict_to_model`. I am not using other libraries such as 352 | [Flask-RestPlus](https://flask-restplus.readthedocs.io/en/stable/) or 353 | [Connexion](https://github.com/zalando/connexion). I have used 354 | [Flask-Restful](https://flask-restful.readthedocs.io/en/latest/) before but I 355 | am not sure these libs add value. You might be able to reduce the number 356 | of code lines a little with them but possibly loosing simplicity and control. 357 | In my opinion Flask already provides the core I need. Adding one of these libs 358 | is perfectly possible though should you want it. 359 | 360 | Also I am not using [Swagger](https://swagger.io/) here but I do have felt the 361 | temptation! The first question with Swagger would be which way to go: To first 362 | create the API spec manually and then generate the method stubs, or first 363 | write the methods with annotations and then generate the API spec? I am not 364 | sure about the order and [neither is the community](https://news.ycombinator.com/item?id=14035936). Swagger maybe good for big projects but this is a 365 | small and cute project :) 366 | 367 | 368 | Authentication & authorization 369 | ------------------------------ 370 | 371 | Simple email-password based signup and login authentication is included in the 372 | server. It relies on [PassLib](https://passlib.readthedocs.io/en/stable/) 373 | Python library for strong uptodate hashing algorithms. 374 | 375 | A Python decorator called `login_required` can be inserted before an API 376 | method that controls that the method requires an authenticated user session, 377 | and optionally to specify if the method requires a certain user level. User 378 | accounts have a role-field whose value is one of: 379 | 380 | ```python 381 | user.role = (disabled, readonly, editor, admin, superuser) 382 | ``` 383 | 384 | You can set the user roles that are meaningful for your project, see the 385 | migration [001_users.py](migrations/001_users.py). If this simple linear role 386 | model is too limited, you can introduce a capability model perhaps with a text 387 | array field, similar to `user.tags`. 388 | 389 | For example, this method requires that the user is logged on and has a role 390 | editor: 391 | 392 | ```python 393 | @app.route('/api/movies/', methods = ['POST']) 394 | @login_required(role='editor') 395 | def movie_create(): 396 | """Creates a movie and returns it.""" 397 | #...code here... 398 | ``` 399 | 400 | If the user does not have an editor role or above, the API server returns 401 401 | Unauthorized to the client. 402 | 403 | API methods which don't have a `login_required` decorator attached are 404 | available for anybody, including non authenticated visitors. 405 | 406 | Accounts with role=disabled are stopped at the door and not let in to the 407 | server at all. 408 | 409 | If you want to support Facebook or Google OAuth authentication method, I 410 | recommend you use the [rauth](https://github.com/litl/rauth) library. 411 | 412 | By default the server allows accessing the API from all domains, CORS 413 | `Access-Control-Allow-Origin value='*'` but it can be set in the config file. 414 | 415 | 416 | Session data 417 | ------------ 418 | 419 | Server-side session data is stored in Redis. Data written into a session is 420 | not visible at the client side. 421 | 422 | Flask provides a thread-global session object that acts like a dictionary. You 423 | set keys to values in the session. A value can be any object that can be 424 | [pickled](https://docs.python.org/3/library/pickle.html). Modifications to the 425 | session data are automatically saved to Redis by Flask at the end of the 426 | request. 427 | 428 | This starter stores two core data in the session: `userid` and `role` of the 429 | user. (Role-field is in session for performance reason: otherwise we would 430 | need to query it from the database with EVERY request that specifies 431 | login_required. Note that if the user role changes, you need to update it in 432 | session too.) 433 | 434 | A common operation in an API method is to access the calling user object, 435 | myself. There is a call `webutil.get_myself()` that loads myself from the 436 | database, or None for a visitor. 437 | 438 | Flask also provides a thread-global object called `g` where you can store 439 | data, but this data is *only stored for the duration of the request.* This 440 | data is not stored in Redis and is discarded when the request ends. `g` can be 441 | used for caching common data during the request, but don't overuse it. 442 | 443 | Redis is a persistent storage, unlike memcached, which means that if the 444 | server gets rebooted, the user sessions will be restored and logged-in users 445 | do not need to relogin. 446 | 447 | By default, the session is remembered for 1 month. If there is no user 448 | activity during 1 month, the session gets deleted. This time is controlled by 449 | PERMANENT_SESSION_LIFETIME in [config.py](py/config.py). 450 | 451 | 452 | Redis storage 453 | ------------- 454 | 455 | You can also use Redis for other than session data. Redis can act as a 456 | convenient schema-free storage for various kinds of data, perhaps for 457 | temporary data, or for lists whose size can be limited, or act as a 458 | distributed cache within a cluster of servers. 459 | 460 | A typical case might be that a background worker puts the calculation results 461 | into Redis where the data is picked from by an API method (if the result is 462 | secondary in nature and does not belong to the master database). 463 | 464 | The module [red.py](py/red.py) provides simple methods for using Redis: 465 | 466 | ```python 467 | # store a value into Redis (here value is a dict but can be anything) 468 | value = {"type":"cat", "name":"Sophia"} 469 | red.set_keyval("mykey", value) 470 | 471 | # get a value 472 | value = red.get_keyval("mykey") 473 | 474 | # store a value that will expire/disappear after 70 minutes: 475 | red.set_keyval("cron_calculation_cache", value, 70*60) 476 | ``` 477 | 478 | To append data into a list: 479 | 480 | ```python 481 | # append item into a list 482 | item = {"action":"resize", "url":"https://example.org/a.jpg"} 483 | red.list_append("mylist", item) 484 | 485 | # take first item from a list 486 | item = red.list_pop("mylist") 487 | 488 | # append item into a FIFO list with a max size of 100 489 | # (discards the oldest items first) 490 | red.list_append("mylist", data_item, 100) 491 | ``` 492 | 493 | red.py can be extended to cover more functionality that 494 | [Redis provides](https://redis.io/commands). 495 | 496 | 497 | Background workers & cron 498 | ------------------------- 499 | 500 | uwsgi provides a simple mechanism to run long running tasks in background 501 | worker processes. 502 | 503 | In any Python module (like in [bgtasks.py](py/bgtasks.py)) you have code 504 | to be run in a background worker: 505 | 506 | ```python 507 | @spool(pass_arguments=True) 508 | def send_email(*args, **kwargs): 509 | """A background worker that is executed by spooling arguments to it.""" 510 | #...code here... 511 | ``` 512 | 513 | You start the above method in a background worker process like this: 514 | 515 | ```python 516 | bgtasks.send_email.spool(email="tomi@tomicloud.com", 517 | subject="Hello world!", template="welcome.html") 518 | ``` 519 | 520 | The number of background worker processes is controlled by `spooler-processes` 521 | configuration in [uwsgi.ini](conf/uwsgi.ini). The spooled data is written and 522 | read into a temp file on disk, not in Redis. 523 | 524 | Crons are useful for running background tasks in specified times, like in 525 | every hour or every night. uwsgi has an easy built-in support for crons. To 526 | have a nightly task you simple code: 527 | 528 | ```python 529 | @cron(0,2,-1,-1,-1) 530 | #(minute, hour, day, month, weekday) - in server local time 531 | def daily(num): 532 | """Runs every night at 2:00AM.""" 533 | #...code here... 534 | ``` 535 | 536 | Mules 537 | ----- 538 | 539 | Mules are independent background worker processes/servers that start and stop 540 | along with the main API server. 541 | 542 | The benefit of mules is ease of setup and ease of sharing code and 543 | environment. You can develop extra servers with little effort - uwsgi manages 544 | the config and lifetimes of mules. If a mule exits for some reason, it is 545 | automatically restarted by uwsgi. (It is also possible to communicate between 546 | mules - read more [in the 547 | docs](https://uwsgi-docs.readthedocs.io/en/latest/Mules.html)). 548 | 549 | In the included toy example [mule1.py](py/mule1.py) a TCP server is created 550 | that listens on port 9999 and echoes back whatever it receives from TCP 551 | clients. You can test it by sending data to it with netcat-tool like this: 552 | 553 | echo "hello world" | nc 192.168.100.10 9999 554 | 555 | 556 | You can have any number of different mules, each running in their own process. 557 | They are configured in [uwsgi.ini](conf/uwsgi.ini): 558 | 559 | mule = py/mule1.py 560 | mule = py/mule2.py 561 | 562 | 563 | 564 | Logging 565 | ------- 566 | 567 | Logging is an essential tool in monitoring and troubleshooting the server 568 | operation. This server automatically logs several useful data in the log file, 569 | including the request path, method and parameters, and return codes and 570 | tracebacks. Userid and origin ip address is logged too. 571 | 572 | Secrets should not be written into a log file to prevent unnecessary leakage 573 | of data. Currently this server automatically prevents logging keys 574 | ["password", "passwd", "pwd"] if they exist in the input parameters of an API 575 | method. You should extend this list to cover your secret keys. 576 | 577 | In local development, the log is dumped to the console, and at server the log 578 | is dumped to a file `/app/app.log`. It is also possible to send the logging 579 | output to a remote `rsyslog`. This can be useful in a cluster setup. 580 | 581 | Slow queries that take more than 1 second to finish are logged with a warning 582 | SLOW! and the log line includes the actual amount of time it took to respond. 583 | It is good to monitor where the pressure starts to cumulate in a server and 584 | then optimize the code or put the execution into a background worker. 585 | 586 | Requests that take more than 20 seconds get terminated and the following line 587 | is logged: `HARAKIRI ON WORKER 1 (pid: 47704, try: 1)`. This harakiri time is 588 | configurable. 589 | 590 | To log the executed SQL statements during development, see PYSRV_LOG_SQL 591 | above. 592 | 593 | Note that as the server logs may contain sensitive data, you should not keep 594 | the production logs for too long time, and you should mention in the policy 595 | statement what data you collect. The GDPR legislation in Europe has a saying 596 | on this. 597 | 598 | To monitor the log file in realtime at server you can invoke: 599 | 600 | tail -f -n 500 /app/app.log 601 | 602 | To see the logged errors only, in most recent first order: 603 | 604 | tac /app/app.log | grep "^ERR " | less -r 605 | 606 | This starter comes with a simple log line colorizer out of the box, hilighting 607 | warnings and errors with a simple logging.Formatter. This is convenient but 608 | the escape codes are inserted into the log file. If you want to have a more 609 | powerful logging colorizer, take a look of 610 | [grc](https://github.com/garabik/grc) for example. 611 | 612 | As an example, the server logs [this output](test/sample.log.txt) when the 613 | included API tests are run. 614 | 615 | 616 | Tests 617 | ----- 618 | 619 | The test folder contains two test scripts for testing the server API and the 620 | Redis module. Keeping tests up-todate and running them frequently or 621 | automatically during the process is a safety net that protects you from 622 | mistakes and bugs. With dynamic languages such as Python or Javascript or 623 | Ruby, tests are even more important than with compiled languages. 624 | 625 | For locally run tests I expose a method `/apitest/dbtruncate` in 626 | [api_dev.py](py/api_dev.py) that truncates the data in the database tables 627 | before running the API tests. If you like to write tests in a different way, 628 | just remove it. 629 | 630 | Run tests inside the DEV instance: 631 | 632 | docker exec -it restpie-dev bash -l -c 'python /app/test/test_api.py' 633 | docker exec -it restpie-dev bash -l -c 'python /app/test/test_redis.py' 634 | 635 | 636 | Deploy to Cloud 637 | --------------- 638 | 639 | There are endless ways to deploy server software to cloud, X amount of tools and Y amount of different kinds of online services. For an early startup or for a hobby project my advice is this: start small and easy - buy a cheap virtual private server, VPS, or a few of them, from a big or a small cloud vendor and run your backend services there. Have control of your infra. Have simple scripts to automate things. Then automate more via Github Actions and so on. There is time later to grow your infra if you get past product-market-fit challenges and actually enjoy a bigger business. 640 | 641 | [Ansible](https://www.ansible.com/) is one of the best lightweigt tools to automate infra tasks. It simply runs commands over the SSH against all your servers. 642 | 643 | I have prepared Ansible scripts to setup RESTPie3 quickly on a Linux server(s). These steps have been verified to work with Debian 12.5 "bookworm". 644 | 645 | The scripts setup Redis + PostgreSQL + RESTPie3 + Caddy on your server(s). [Caddy](https://caddyserver.com) is a solid and simple www server written in Go that can run as a proxy in front of uwsgi. 646 | 647 | Prerequisites: 648 | - Have a root ssh access to your server(s) via ssh publickey, and use ssh-agent to store your private key so you don't have to type the passphare all the time. After setup you should disable server root access. 649 | - Update your server to the latest versions: `apt update & apt upgrade` 650 | - Locally create a new ssh-key for the user-level Linux account that is later used for deploys 651 | `ssh-keygen -t rsa -f ~/.ssh/id_myapp -C 'restpie3 deploy account'` 652 | - If you run OSX, update local rsync to latest 3.x: `brew install rsync` 653 | - Install Ansible tools and extensions locally (OSX: `brew install ansible`) 654 | - Put your desired config/secrets into [vars.yml](ansible/vars.yml) Note: do NOT commit secrets into git! Move this file elsewhere or put into `.gitignore` 655 | - Write the IP-address or domain name of your server into [myhosts.ini](ansible/myhosts.ini). If you have multiple servers, write them all here. 656 | 657 | Then run these scripts: 658 | 659 | cd restpie3/ansible/ 660 | ansible-playbook -i myhosts.ini install-redis.yaml 661 | ansible-playbook -i myhosts.ini install-caddy.yaml 662 | ansible-playbook -i myhosts.ini install-db.yaml 663 | ansible-playbook -i myhosts.ini install-app.yaml 664 | 665 | Now you should have a running RESTPie3 at the server you configured. Point your browser there. You should see text `RESTPie3 setup works!` 666 | 667 | Your code updates from local machine to server can be deployed with: 668 | 669 | ansible-playbook -i myhosts.ini deploy.yaml 670 | 671 | Later you might want to run this script from Github Action to have consistent automation. 672 | 673 | For a production setup you should harden the security of this simple setup with the usual steps like: create a private network behind a load balancer, disable ssh root login, carefully build access control, etc. 674 | 675 | 676 | Security 677 | -------- 678 | 679 | A few words about security practices in this software: 680 | 681 | * no secrets are stored in code but in an external configuration file or in 682 | environment variables 683 | * all requests are logged and can be inspected later 684 | * no secrets are leaked to the log file 685 | * server based sessions with an opaque uuid cookie 686 | * no Javascript access to uuid cookie 687 | * config to require https for cookies (do enable in production) 688 | * strong algorithms are used for password auth 689 | * only salt and password hash is stored in the database 690 | * session can be either permanent or end on browser close 691 | * sensitive user objects have a random uuid and not an integer id 692 | * possible to add a fresh cookie check by saving IP+agent string to session 693 | * database code is not vulnerable to SQL injections 694 | * authorization is enforced via user roles and function decorator, not 695 | requiring complex code 696 | * uwsgi supports running code as a lower privileged user 697 | 698 | Of course the overall security of the service is also heavily dependent on the 699 | configuration of the server infrastructure and access policies. 700 | 701 | 702 | Scaling up 703 | ---------- 704 | 705 | While the codebase of this server is small, it has decent muscle. The 706 | *stateless* design makes it perfectly possible to scale it up to work in a 707 | bigger environment. There is nothing in the code that "does not scale". The 708 | smallness and flexibility of the software makes it actually easier to scale, 709 | allowing easier switch of components in case that is needed. For 710 | example, you can start with the uwsgi provided background tasks framework and 711 | replace that later on with, say [Celery](http://www.celeryproject.org/), if 712 | you see it scales and fits better. 713 | 714 | In a traditional setup you first scale up vertically, you "buy a bigger 715 | server". This code contains a few measures that helps scaling up 716 | vertically: 717 | 718 | * you can add worker processes to match the single server capacity 719 | * you can add more background processes if there is a lot of background jobs 720 | * slow requests are logged so you see when they start to cumulate 721 | * stuck requests never halt the whole server - stuck workers are killed after 722 | 20 seconds by default, freeing resources to other workers (harakiri) 723 | * you can optimize SQL, write raw queries, let the database do work better 724 | * you can cache data in Redis 725 | 726 | When you outgrow the biggest single server, and start adding servers, you 727 | scale horizontally. The core API server component can be cloned into a cluster 728 | of servers where each of them operates independently of the others. Scaling 729 | the API server here is easy, it is the other factors that become harder, like 730 | database scaling and infra management. 731 | 732 | 733 | Run in Raspberry 734 | ---------------- 735 | 736 | RESTPie3 runs fine in Raspberry. RESTPie3 is lightweight, does not consume 737 | much resources, and supports robust daemon and worker management that is 738 | important in a setup that may have more hiccups such as power outages or 739 | connectivity issues. 740 | 741 | In a Raspberry setup there usually is less need for a big database. Hence 742 | RESTPie3 also supports SQLite which is a small but solid zero configuration 743 | SQL database. 744 | 745 | To activate SQLite mode, configure server-config.json like this: 746 | 747 | "PYSRV_DATABASE_HOST": "/app/data/mydb.sqlite", 748 | 749 | Then follow steps in "Run and Develop locally with Docker". Invoke ./run.sh. 750 | Then initialize SQLite database inside the container, from another terminal: 751 | 752 | docker exec -it restpie-dev bash -l -c 'python3 /app/scripts/dbmigrate.py' 753 | 754 | The SQLite database file will be created into RESTPiE3/data/mydb.sqlite. Note 755 | that this file is outside the container, accessed via volume so the file is 756 | not destroyed when the image is destroyed. 757 | 758 | Local container should now run with SQLite database. 759 | 760 | **Setup Raspberry** 761 | 762 | The setup steps for Raspberry are similar as with any Linux host. Here's the 763 | steps in short. I assume you already have a working SSH connection to 764 | Raspberry with pubkey configured. 765 | 766 | ```console 767 | # in raspberry: 768 | sudo apt-get update 769 | sudo apt-get install redis-server 770 | sudo apt-get install python3-pip 771 | sudo mkdir /app/ 772 | sudo chown pi /app 773 | 774 | # in local machine: 775 | pico rsync.sh # write your own HOST 776 | # then transfer files to raspberry /app/ 777 | ./rsync.sh 778 | 779 | # in raspberry: 780 | pico /app/requirements.txt # remove psycopg2-binary 781 | sudo pip3 install -r /app/requirements.txt 782 | sudo pip3 install uwsgi 783 | cd /app/ 784 | export PYTHONPATH=/app/py/ 785 | cp conf/server-config.json real-server-config.json 786 | export PYSRV_CONFIG_PATH=/app/real-server-config.json 787 | 788 | # init sqlite database 789 | mkdir data 790 | python3 scripts/dbmigrate.py 791 | # empty database was created at /app/data/mydb.sqlite 792 | pico /app/real-server-config.json # make sure all is correct, change PYSRV_REDIS_HOST to "localhost:6379" 793 | 794 | # setup server as a service, to start on reboot 795 | sudo cp conf/pydaemon.service /etc/systemd/system/ 796 | sudo systemctl enable pydaemon 797 | sudo systemctl daemon-reload 798 | sudo systemctl start pydaemon 799 | 800 | # in local machine: 801 | # edit sources, then rsync... 802 | ./rsync.sh 803 | # server reloads itself automatically 804 | ``` 805 | 806 | 807 | What about the front-end? 808 | ------------------------- 809 | 810 | This is primarily a back-end Python server that provides a REST API to the 811 | world. There is no front-end implementation in this project apart from the 812 | four quick auth pages. This is because the focus is on creating a good REST 813 | API server that serves web front-ends and native mobile apps, but also because 814 | I think that it is good to modularize the front-end and back-end code cleanly 815 | into separate code bases. 816 | 817 | There is a simple [example.html](templates/example.html) page if you just want 818 | to quickly output an HTML page that is generated at server side in the old 819 | fashioned way. 820 | 821 | Also included are simple HTML pages for [login, signup, forgot password and reset password](templates/auth.html) since these pages are usually 822 | required in every project and it is boring to always build them from 823 | scratch. Only HTML and CSS is used with zero lines of Javascript. 824 | It is easy to start the project with them and create something fancier 825 | later if needed. 826 | 827 | 828 | Need help? 829 | ---------- 830 | 831 | This starter is intended to provide you a quick start in building a great 832 | foundation for an API back-end. Take this code and see if it works for you. 833 | This server is not a toy - it is a practical, solid server that is based on my 834 | experience in building full-stack services over the years. 835 | 836 | If you need dev power in building your great service, back or front, you can 837 | [email me](mailto:tomi.mickelsson@gmail.com) to ask if I am available for 838 | freelancing work. Check my blog at [tomicloud.com](https://tomicloud.com). 839 | 840 | 841 | License 842 | ------- 843 | MIT License 844 | 845 | 846 | Screenshot 847 | ---------- 848 | 849 | [/api/list](http://localhost:8100/api/list) returns an HTML page that lists 850 | the available API methods and their docstring automatically. 851 | 852 | ![API-listing](test/api-list.jpg) 853 | 854 | Screenshot2 855 | ----------- 856 | 857 | [/auth/signup](http://localhost:8100/auth/signup) quick auth pages for 858 | login/signup/forgot password/reset password. 859 | 860 | ![](test/auth.jpg) 861 | 862 | -------------------------------------------------------------------------------- /ansible/deploy.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Deploy software to server 3 | hosts: apphosts 4 | # note: deploy happens via regular user, not root, which should be disabled 5 | 6 | vars_files: 7 | - vars.yml 8 | 9 | tasks: 10 | - name: Synch files to server 11 | ansible.posix.synchronize: 12 | src: "{{ src_folder }}" 13 | dest: /app/ 14 | rsync_opts: 15 | - "--exclude=.git" 16 | - "--chown={{ app_user }}:staff" 17 | 18 | - name: DB migration 19 | ansible.builtin.shell: 20 | cmd: /app/PYENV/bin/python3 /app/scripts/dbmigrate.py 21 | chdir: /app/ 22 | environment: 23 | PYTHONPATH: /app/py/ 24 | PYSRV_CONFIG_PATH: /app/real-server-config.json 25 | PATH: "/app/PYENV/bin/:{{ ansible_env.PATH }}" 26 | when: 0 27 | # when toggles true/false 28 | 29 | - name: Restart server 30 | ansible.builtin.file: 31 | path: /app/RESTART 32 | state: touch 33 | mode: u=rw,g=r,o=r 34 | -------------------------------------------------------------------------------- /ansible/install-app.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Install our python app server 3 | hosts: apphostsroot 4 | 5 | vars_files: 6 | - vars.yml 7 | 8 | tasks: 9 | - name: Create unix user 10 | ansible.builtin.user: 11 | name: "{{ app_user }}" 12 | groups: "sudo" 13 | 14 | - name: Upload ssh pubkey to authorized key 15 | ansible.posix.authorized_key: 16 | user: "{{ app_user }}" 17 | state: present 18 | key: "{{ lookup('file', '~/.ssh/id_myapp.pub') }}" 19 | 20 | - name: Create /app dir 21 | ansible.builtin.file: 22 | path: /app/ 23 | state: directory 24 | owner: "{{ app_user }}" 25 | mode: u=rw,g=r,o=r 26 | 27 | - name: Install packages 28 | ansible.builtin.apt: 29 | pkg: 30 | - python3-pip 31 | - python3-venv 32 | - uwsgi-core 33 | - uwsgi-plugin-python3 34 | - htop 35 | 36 | - name: Create python virtual env 37 | ansible.builtin.command: "python3 -m venv /app/PYENV/" 38 | 39 | - name: Copy source files to server 40 | ansible.posix.synchronize: 41 | src: "{{ src_folder }}" 42 | dest: /app/ 43 | owner: false 44 | group: false 45 | rsync_opts: 46 | - "--exclude=.git" 47 | 48 | - name: Create RESTART file 49 | ansible.builtin.file: 50 | path: /app/RESTART 51 | state: touch 52 | 53 | - name: Recursively change ownership of dir 54 | ansible.builtin.file: 55 | path: /app/ 56 | state: directory 57 | recurse: yes 58 | owner: "{{ app_user }}" 59 | group: "staff" 60 | 61 | - name: Install py libs 62 | ansible.builtin.pip: 63 | requirements: /app/requirements.txt 64 | virtualenv: /app/PYENV/ 65 | 66 | - name: Copy srv config file with correct data 67 | ansible.builtin.template: 68 | src: "{{ src_folder }}/conf/server-config.json" 69 | dest: /app/real-server-config.json 70 | owner: "{{ app_user }}" 71 | group: root 72 | mode: u=rw,g=r,o=r 73 | 74 | - name: Copy service file 75 | ansible.builtin.copy: 76 | src: "{{ src_folder }}/conf/pydaemon.service" 77 | dest: /etc/systemd/system/ 78 | owner: root 79 | group: root 80 | mode: u=rwx,g=rx,o=rx 81 | 82 | - name: Enable pydaemon service 83 | ansible.builtin.systemd_service: 84 | name: pydaemon 85 | enabled: true 86 | masked: no 87 | 88 | - name: Start pydaemon 89 | ansible.builtin.systemd_service: 90 | daemon_reload: true 91 | state: started 92 | name: pydaemon 93 | 94 | - name: Init db schema 95 | ansible.builtin.shell: 96 | cmd: /app/PYENV/bin/python3 /app/scripts/dbmigrate.py 97 | chdir: /app/ 98 | environment: 99 | PYTHONPATH: /app/py/ 100 | PYSRV_CONFIG_PATH: /app/real-server-config.json 101 | PATH: "/app/PYENV/bin/:{{ ansible_env.PATH }}" 102 | -------------------------------------------------------------------------------- /ansible/install-caddy.yml: -------------------------------------------------------------------------------- 1 | 2 | - name: Install Caddy www server 3 | hosts: wwwhost 4 | # assumes you have static files at /app/www/ 5 | 6 | vars_files: 7 | - vars.yml 8 | 9 | tasks: 10 | - name: Install Caddy 11 | ansible.builtin.apt: 12 | pkg: 13 | - caddy 14 | 15 | - name: Copy our caddy config 16 | ansible.builtin.copy: 17 | src: "{{ src_folder }}/conf/caddy.conf" 18 | dest: /etc/caddy/Caddyfile 19 | owner: root 20 | group: root 21 | mode: u=rw,g=r,o=r 22 | 23 | - name: Restart caddy 24 | ansible.builtin.systemd_service: 25 | state: restarted 26 | name: caddy 27 | -------------------------------------------------------------------------------- /ansible/install-db.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Install PostgreSQL database server 3 | hosts: dbhost 4 | become: yes 5 | 6 | vars_files: 7 | - vars.yml 8 | 9 | tasks: 10 | - name: Install PostgreSQL server 11 | ansible.builtin.apt: 12 | name: postgresql 13 | 14 | - name: Install pip 15 | ansible.builtin.apt: 16 | pkg: 17 | - python3-pip 18 | 19 | - name: Install psycopg2 for community plugin below 20 | ansible.builtin.pip: 21 | name: psycopg2-binary 22 | extra_args: --break-system-packages 23 | 24 | - name: Create db user 25 | community.postgresql.postgresql_user: 26 | name: "{{ db_user }}" 27 | password: "{{ db_password }}" 28 | become: yes 29 | become_user: postgres 30 | 31 | - name: Create app db 32 | community.postgresql.postgresql_db: 33 | name: "{{ db_name }}" 34 | owner: "{{ db_user }}" 35 | become: yes 36 | become_user: postgres 37 | 38 | - name: Add uuid extension 39 | community.postgresql.postgresql_ext: 40 | name: uuid-ossp 41 | db: "{{ db_name }}" 42 | become: yes 43 | become_user: postgres 44 | -------------------------------------------------------------------------------- /ansible/install-redis.yaml: -------------------------------------------------------------------------------- 1 | 2 | - name: Install Redis server 3 | hosts: redishost 4 | 5 | tasks: 6 | - name: Install Redis 7 | ansible.builtin.apt: 8 | name: redis 9 | -------------------------------------------------------------------------------- /ansible/myhosts.ini: -------------------------------------------------------------------------------- 1 | [apphosts] 2 | 10.10.10.10 ansible_ssh_user=myapp 3 | 4 | [apphostsroot] 5 | 10.10.10.10 ansible_ssh_user=root 6 | 7 | [redishost] 8 | 10.10.10.10 ansible_ssh_user=root 9 | 10 | [dbhost] 11 | 10.10.10.10 ansible_ssh_user=root 12 | 13 | [wwwhost] 14 | 10.10.10.10 ansible_ssh_user=root 15 | -------------------------------------------------------------------------------- /ansible/vars.yml: -------------------------------------------------------------------------------- 1 | --- 2 | redis_host: localhost 3 | db_host: localhost 4 | src_folder: /Users/x/restpie3/ 5 | app_user: myapp 6 | db_user: my-db-account 7 | db_password: my-password-here 8 | db_name: mydb 9 | 10 | # this file has your secrets - keep it safe, do not commit into repository! 11 | # do not use the default values here! 12 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # run in dev mode 3 | 4 | docker build --build-arg BUILDMODE=debug-docker -t restpie-dev-image . 5 | 6 | -------------------------------------------------------------------------------- /conf/caddy.conf: -------------------------------------------------------------------------------- 1 | # The Caddyfile is an easy way to configure your Caddy web server. 2 | # 3 | # Unless the file starts with a global options block, the first 4 | # uncommented line is always the address of your site. 5 | # 6 | # To use your own domain name (with automatic HTTPS), first make 7 | # sure your domain's A/AAAA DNS records are properly pointed to 8 | # this machine's public IP, then replace ":80" below with your 9 | # domain name. 10 | 11 | :80 { 12 | # Set this path to your site's directory. 13 | root * /app/www/ 14 | 15 | # Enable the static file server. 16 | file_server 17 | 18 | # Another common task is to set up a reverse proxy: 19 | # reverse_proxy localhost:8080 20 | 21 | # Or serve a PHP site through php-fpm: 22 | # php_fastcgi localhost:9000 23 | 24 | # forward to restpie3 25 | handle_path /api/* { 26 | rewrite * /api{uri} 27 | reverse_proxy localhost:8110 28 | } 29 | } 30 | 31 | # Refer to the Caddy docs for more information: 32 | # https://caddyserver.com/docs/caddyfile 33 | -------------------------------------------------------------------------------- /conf/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomimick/restpie3/890c5bdb15ce2ebbcc357797a0a4e8eb7f1ffc92/conf/favicon.ico -------------------------------------------------------------------------------- /conf/loginscript.sh: -------------------------------------------------------------------------------- 1 | # this is /etc/profile 2 | # - a login script when running the interactive shell inside the container 3 | 4 | export PYTHONPATH=/app/py 5 | export PYSRV_CONFIG_PATH=/app/conf/server-config.json 6 | export FLASK_ENV=development 7 | alias l='ls' 8 | alias ll='ls -l' 9 | 10 | -------------------------------------------------------------------------------- /conf/pydaemon.service: -------------------------------------------------------------------------------- 1 | 2 | # systemd service configuration - uwsgi daemon 3 | # 4 | # https://www.digitalocean.com/community/tutorials/understanding-systemd-units-and-unit-files 5 | # https://www.digitalocean.com/community/tutorials/how-to-serve-flask-applications-with-uwsgi-and-nginx-on-ubuntu-16-04 6 | 7 | # make start on boot: systemctl enable mydaemon 8 | 9 | [Unit] 10 | Description=pysrv uwsgi daemon 11 | After=network.target 12 | 13 | [Service] 14 | User=root 15 | #User=myapp # user privileges are set by uwsgi 16 | #Group=mygroup 17 | # note: create /tmp/pysrv_spooler on reboot 18 | ExecStartPre=/bin/mkdir -p /tmp/pysrv_spooler; 19 | ExecStart=uwsgi --ini /app/conf/uwsgi.ini:uwsgi-production 20 | RuntimeDirectory=mydaemon 21 | Restart=always 22 | RestartSec=3 23 | KillSignal=SIGQUIT 24 | 25 | [Install] 26 | WantedBy=multi-user.target 27 | 28 | -------------------------------------------------------------------------------- /conf/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | 4 | -------------------------------------------------------------------------------- /conf/server-config-localdev.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "python server config, for localdev", 3 | 4 | "PYSRV_IS_PRODUCTION": "", 5 | 6 | "PYSRV_DATABASE_HOST": "/app/data/mydb.sqlite", 7 | "PYSRV_DATABASE_HOST_POSTGRESQL": "host.docker.internal", 8 | "PYSRV_DATABASE_HOST_SQLITE": "/app/data/mydb.sqlite", 9 | "PYSRV_DATABASE_PORT": "5432", 10 | "PYSRV_DATABASE_NAME": "tmdb", 11 | "PYSRV_DATABASE_USER": "tm", 12 | "PYSRV_DATABASE_PASSWORD": "MY_PASSWORD", 13 | 14 | "PYSRV_COOKIE_HTTPS_ONLY": false, 15 | "PYSRV_REDIS_HOST": "host.docker.internal:6379", 16 | "PYSRV_DOMAIN_NAME": "", 17 | "PYSRV_CORS_ALLOW_ORIGIN": "*" 18 | } 19 | -------------------------------------------------------------------------------- /conf/server-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "python server config, modified by ansible", 3 | 4 | "PYSRV_IS_PRODUCTION": "", 5 | 6 | "PYSRV_DATABASE_HOST": "{{ db_host }}", 7 | "PYSRV_DATABASE_PORT": "5432", 8 | "PYSRV_DATABASE_NAME": "{{ db_name }}", 9 | "PYSRV_DATABASE_USER": "{{ db_user }}", 10 | "PYSRV_DATABASE_PASSWORD": "{{ db_password }}", 11 | 12 | "PYSRV_COOKIE_HTTPS_ONLY": false, 13 | "PYSRV_REDIS_HOST": "{{ redis_host }}", 14 | "PYSRV_DOMAIN_NAME": "", 15 | "PYSRV_CORS_ALLOW_ORIGIN": "*" 16 | } 17 | 18 | -------------------------------------------------------------------------------- /conf/uwsgi.ini: -------------------------------------------------------------------------------- 1 | 2 | # uwsgi daemon config 3 | # https://uwsgi-docs.readthedocs.io/en/latest/Options.html 4 | 5 | # old: local dev - plain python, no docker 6 | [uwsgi-debug] 7 | env = FLASK_ENV=development 8 | env = PYSRV_CONFIG_PATH=conf/server-config.json 9 | http = localhost:8100 10 | master = 1 11 | wsgi-file = py/main.py 12 | callable = app 13 | # processes = 1, otherwise autoreload fails 14 | processes = 1 15 | stats = 127.0.0.1:9100 16 | virtualenv = $(SERVER_VIRTUALENV) 17 | py-autoreload = 1 18 | #harakiri=10 - disable locally, otherwise autoreload fails 19 | disable-logging=1 20 | spooler-quiet=1 21 | spooler-processes=1 22 | spooler-frequency=5 23 | spooler-harakiri=600 24 | spooler = /tmp/my_spooler 25 | # few static files - serve the frontend from elsewhere 26 | static-map = /robots.txt=conf/robots.txt 27 | static-map = /favicon.ico=conf/favicon.ico 28 | 29 | 30 | # local dev with docker - py-autoreload enabled 31 | [uwsgi-debug-docker] 32 | env = FLASK_ENV=development 33 | env = PYSRV_CONFIG_PATH=/app/conf/server-config-localdev.json 34 | http = 0.0.0.0:80 35 | # if using nginx and uwsgi_pass: 36 | # uwsgi-socket = localhost:8010 37 | master = 1 38 | wsgi-file = py/main.py 39 | callable = app 40 | processes = 4 41 | chdir = /app/ 42 | pythonpath = /app/py/ 43 | py-autoreload = 1 44 | disable-logging=1 45 | spooler-quiet=1 46 | spooler-processes=3 47 | spooler-frequency=5 48 | spooler-max-tasks=100 49 | spooler-harakiri=600 50 | spooler = /tmp/pysrv_spooler 51 | vacuum = true 52 | logger = stdio 53 | # workers live max this many requests and secs 54 | max-requests=100 55 | max-worker-lifetime=36000 56 | # few static files - serve the frontend from elsewhere 57 | static-map = /robots.txt=conf/robots.txt 58 | static-map = /favicon.ico=conf/favicon.ico 59 | mule = py/mule1.py 60 | 61 | # test/production server config - plain python and docker 62 | [uwsgi-production] 63 | plugins = corerouter,python3,logfile,spooler,http 64 | env=PYSRV_CONFIG_PATH=/app/real-server-config.json 65 | # http :8110 is externally available, localhost:8110 is not 66 | http = localhost:8110 67 | # if using nginx and uwsgi_pass: 68 | # uwsgi-socket = localhost:8010 69 | master = 1 70 | wsgi-file = py/main.py 71 | callable = app 72 | processes = 4 73 | chdir = /app/ 74 | pythonpath = /app/py/ 75 | virtualenv = /app/PYENV/ 76 | # deploy-script touches this file and uwsgi restarts 77 | touch-reload=/app/RESTART 78 | harakiri=20 79 | disable-logging=1 80 | spooler-quiet=1 81 | spooler-processes=3 82 | spooler-frequency=5 83 | spooler-max-tasks=100 84 | spooler-harakiri=600 85 | spooler = /tmp/pysrv_spooler 86 | vacuum = true 87 | # log to file (and stdout too so docker run locally works) 88 | logger = file:/app/app.log 89 | logger = stdio 90 | # run as this user - MUST SET LOWER PRIVILEGES! (Port 80 requires root) 91 | uid=myapp 92 | ; gid=appgroup 93 | # workers live max this many requests and secs 94 | max-requests=100 95 | max-worker-lifetime=36000 96 | # few static files - serve the frontend from elsewhere 97 | static-map = /robots.txt=conf/robots.txt 98 | static-map = /favicon.ico=conf/favicon.ico 99 | mule = py/mule1.py 100 | 101 | -------------------------------------------------------------------------------- /migrations/001_users.py: -------------------------------------------------------------------------------- 1 | """Peewee migrations -- 001_create.py. 2 | 3 | Some examples: 4 | 5 | > Model = migrator.orm['model_name'] # Return model in current state by name 6 | 7 | > migrator.sql(sql) # Run custom SQL 8 | > migrator.python(func, *args, **kwargs) # Run python code 9 | > migrator.create_model(Model) # Create a model 10 | > migrator.remove_model(model, cascade=True) # Remove a model 11 | > migrator.add_fields(model, **fields) # Add fields to a model 12 | > migrator.change_fields(model, **fields) # Change fields 13 | > migrator.remove_fields(model, *field_names, cascade=True) 14 | > migrator.rename_field(model, old_field_name, new_field_name) 15 | > migrator.rename_table(model, new_table_name) 16 | > migrator.add_index(model, *col_names, unique=False) 17 | > migrator.drop_index(model, *col_names) 18 | > migrator.add_not_null(model, *field_names) 19 | > migrator.drop_not_null(model, *field_names) 20 | > migrator.add_default(model, field_name, default) 21 | """ 22 | 23 | def migrate(migrator, database, fake=False, **kwargs): 24 | """Write your migrations here.""" 25 | 26 | # create extension manually - you must be a superuser to do this 27 | # is needed by uuid_generate_v4() 28 | 29 | # migrator.sql("""CREATE EXTENSION IF NOT EXISTS "uuid-ossp";""") 30 | 31 | 32 | migrator.sql("""CREATE TYPE type_user_role AS ENUM ( 33 | 'disabled', 34 | 'readonly', 35 | 'editor', 36 | 'admin', 37 | 'superuser') 38 | """) 39 | 40 | 41 | migrator.sql("""CREATE TABLE users ( 42 | 43 | id uuid PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(), 44 | 45 | created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 46 | modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 47 | 48 | email text UNIQUE, 49 | password text, 50 | first_name text, 51 | last_name text, 52 | role type_user_role DEFAULT 'readonly', 53 | tags text[] 54 | )""") 55 | # normal integer-id: id serial PRIMARY KEY NOT NULL, 56 | 57 | 58 | def rollback(migrator, database, fake=False, **kwargs): 59 | """Write your rollback migrations here.""" 60 | 61 | -------------------------------------------------------------------------------- /migrations/002_movies.py: -------------------------------------------------------------------------------- 1 | 2 | # 002_movies.py 3 | 4 | def migrate(migrator, database, fake=False, **kwargs): 5 | 6 | # an example class for demonstrating CRUD... 7 | 8 | migrator.sql("""CREATE TABLE movies( 9 | id serial PRIMARY KEY NOT NULL, 10 | created timestamp not null default CURRENT_TIMESTAMP, 11 | modified timestamp not null default CURRENT_TIMESTAMP, 12 | 13 | creator uuid REFERENCES users(id), 14 | 15 | title text, 16 | director text 17 | )""") 18 | 19 | def rollback(migrator, database, fake=False, **kwargs): 20 | 21 | migrator.sql("""DROP TABLE movies""") 22 | 23 | -------------------------------------------------------------------------------- /migrations_sqlite/001_init.py: -------------------------------------------------------------------------------- 1 | # 001_init.py 2 | 3 | def migrate(migrator, database, fake=False, **kwargs): 4 | """Write your migrations here.""" 5 | 6 | migrator.sql("""CREATE TABLE users ( 7 | 8 | id INTEGER PRIMARY KEY, 9 | 10 | created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 11 | modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 12 | 13 | email text UNIQUE, 14 | password text, 15 | first_name text, 16 | last_name text, 17 | role text DEFAULT 'readonly', 18 | tags text 19 | )""") 20 | 21 | def rollback(migrator, database, fake=False, **kwargs): 22 | """Write your rollback migrations here.""" 23 | 24 | -------------------------------------------------------------------------------- /migrations_sqlite/002_movies.py: -------------------------------------------------------------------------------- 1 | # 002_movies.py 2 | 3 | def migrate(migrator, database, fake=False, **kwargs): 4 | 5 | migrator.sql("""CREATE TABLE movies( 6 | id INTEGER PRIMARY KEY, 7 | 8 | created timestamp not null default CURRENT_TIMESTAMP, 9 | modified timestamp not null default CURRENT_TIMESTAMP, 10 | 11 | creator integer REFERENCES users(id), 12 | 13 | title text, 14 | director text 15 | )""") 16 | 17 | def rollback(migrator, database, fake=False, **kwargs): 18 | 19 | migrator.sql("""DROP TABLE movies""") 20 | 21 | -------------------------------------------------------------------------------- /py/account.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # account.py: user account related operations, passwords 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | import sys 9 | import re 10 | from flask import request, session 11 | from passlib.context import CryptContext 12 | 13 | import logging 14 | log = logging.getLogger("account") 15 | 16 | 17 | pwd_context = CryptContext( 18 | schemes=["pbkdf2_sha256", "bcrypt"] # list of supported algos 19 | ) 20 | 21 | 22 | def build_session(user_obj, is_permanent=True): 23 | """On login+signup, builds the server-side session dict with the data we 24 | need. userid being the most important.""" 25 | 26 | assert user_obj 27 | assert user_obj.id 28 | 29 | # make sure session is empty 30 | session.clear() 31 | 32 | # fill with relevant data 33 | session['userid'] = user_obj.id 34 | session['role'] = user_obj.role # if you update user.role, update this too 35 | 36 | # remember session even over browser restarts? 37 | session.permanent = is_permanent 38 | 39 | # could also store ip + browser-agent to verify freshness 40 | # of the session: only allow most critical operations with a fresh 41 | # session 42 | 43 | 44 | def hash_password(password): 45 | """Generate a secure hash out of the password. Salts automatically.""" 46 | 47 | return pwd_context.hash(password) 48 | 49 | 50 | def check_password(hash, password): 51 | """Check if given plaintext password matches with the hash.""" 52 | 53 | return pwd_context.verify(password, hash) 54 | 55 | 56 | def check_password_validity(passwd): 57 | """Validates the given plaintext password. Returns None for success, 58 | error text on error.""" 59 | 60 | err = None 61 | 62 | if not passwd or len(passwd) < 6: 63 | err = "Password must be atleast 6 characters" 64 | 65 | elif not re.search(r"[a-z]", passwd) \ 66 | or not re.search(r"[A-Z]", passwd) \ 67 | or not re.search(r"[0-9]", passwd): 68 | err = "Password must contain a lowercase, an uppercase, a digit" 69 | 70 | if err: 71 | log.error("password validity: %s", err) 72 | 73 | return err 74 | 75 | 76 | def new_signup_steps(user_obj): 77 | """Perform steps for a new signup.""" 78 | 79 | # user_obj.signup_ip = webutil.get_ip() 80 | # user_app.save() 81 | 82 | # send welcome email etc... 83 | 84 | -------------------------------------------------------------------------------- /py/api_account.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # api.py: REST API for basic account related stuff: signup/login/logout 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | from flask import request, session, g, jsonify 9 | 10 | import db 11 | import webutil 12 | import account 13 | from webutil import app, login_required, get_myself 14 | 15 | import logging 16 | log = logging.getLogger("api") 17 | 18 | 19 | @app.route('/api/login', methods = ['POST']) 20 | def login(): 21 | """Logs the user in with email+password. 22 | On success returns the user object, 23 | on error returns 400 and json with err-field.""" 24 | 25 | input = request.json or {} 26 | email = input.get('email') 27 | password = input.get('password') 28 | 29 | if not email or not password: 30 | return webutil.warn_reply("Missing input") 31 | 32 | u = db.get_user_by_email(email) 33 | if not u or not account.check_password(u.password, password): 34 | # error 35 | return webutil.warn_reply("Invalid login credentials") 36 | else: 37 | # success 38 | account.build_session(u, is_permanent=input.get('remember', True)) 39 | 40 | log.info("LOGIN OK agent={}".format(webutil.get_agent())) 41 | return jsonify(u), 200 42 | 43 | 44 | @app.route('/api/signup', methods = ['POST']) 45 | def signup(): 46 | """Signs a new user to the service. On success returns the user object, 47 | on error returns 400 and json with err-field.""" 48 | 49 | input = request.json or {} 50 | email = input.get('email') 51 | password = input.get('password') 52 | fname = input.get('fname') 53 | lname = input.get('lname') 54 | company = input.get('company') 55 | 56 | if not email or not password or not fname or not lname: 57 | return webutil.warn_reply("Invalid signup input") 58 | 59 | u = db.get_user_by_email(email) 60 | if u: 61 | msg = "Signup email taken: {}".format(email) 62 | return webutil.warn_reply(msg) 63 | 64 | err = account.check_password_validity(password) 65 | if err: 66 | return jsonify({"err":err}), 400 67 | 68 | # create new user 69 | u = db.User() 70 | u.email = email 71 | u.company = company 72 | u.first_name = fname 73 | u.last_name = lname 74 | u.password = account.hash_password(password) 75 | u.tags = [] 76 | u.role = 'editor' # set default to what makes sense to your app 77 | u.save(force_insert=True) 78 | 79 | account.new_signup_steps(u) 80 | account.build_session(u, is_permanent=input.get('remember', True)) 81 | 82 | log.info("SIGNUP OK agent={}".format(webutil.get_agent())) 83 | 84 | return jsonify(u), 201 85 | 86 | 87 | @app.route('/api/logout', methods = ['POST']) 88 | @login_required 89 | def logout(): 90 | """Logs out the user, clears the session.""" 91 | session.clear() 92 | return jsonify({}), 200 93 | 94 | 95 | @app.route('/api/me') 96 | @login_required 97 | def me(): 98 | """Return info about me. Attach more data for real use.""" 99 | 100 | me = get_myself() 101 | reply = {"me": me} 102 | 103 | return jsonify(reply), 200 104 | 105 | 106 | @app.route('/api/users') 107 | @login_required(role='superuser') 108 | def users(): 109 | """Search list of users. Only for superusers""" 110 | 111 | input = request.args or {} 112 | page = input.get('page') 113 | size = input.get('size') 114 | search = input.get('search') 115 | 116 | reply = db.query_users(page, size, search) 117 | 118 | return jsonify(reply), 200 119 | 120 | -------------------------------------------------------------------------------- /py/api_dev.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # api_dev.py: misc methods for testing and development 5 | # - remove if not needed, and make sure there is no risk for production 6 | # 7 | # Author: Tomi.Mickelsson@iki.fi 8 | 9 | from flask import jsonify, redirect, render_template 10 | import datetime 11 | import html 12 | 13 | import db 14 | import config 15 | import bgtasks 16 | import red 17 | from webutil import app 18 | 19 | import logging 20 | log = logging.getLogger("api") 21 | 22 | 23 | @app.route('/', methods = ['GET']) 24 | def index(): 25 | """Just a redirect to api list.""" 26 | return redirect('/api/list') 27 | 28 | 29 | @app.route('/api/list', methods = ['GET']) 30 | def list_api(): 31 | """List the available REST APIs in this service as HTML. Queries 32 | methods directly from Flask, no need to maintain separate API doc. 33 | (Maybe this could be used as a start to generate Swagger API spec too.)""" 34 | 35 | # decide whether available in production 36 | # if config.IS_PRODUCTION: 37 | # return "not available in production", 400 38 | 39 | # build HTML of the method list 40 | apilist = [] 41 | rules = sorted(app.url_map.iter_rules(), key=lambda x: str(x)) 42 | for rule in rules: 43 | f = app.view_functions[rule.endpoint] 44 | docs = f.__doc__ or '' 45 | module = f.__module__ + ".py" 46 | 47 | # remove noisy OPTIONS 48 | methods = sorted([x for x in rule.methods if x != "OPTIONS"]) 49 | url = html.escape(str(rule)) 50 | if not "/api/" in url and not "/auth/" in url: 51 | continue 52 | apilist.append("
{} {}
{} {}
".format( 53 | url, url, methods, docs, module)) 54 | 55 | header = """ 56 | RESTPie3 57 | """ 64 | title = """ 65 |
66 |

REST API ({} end-points)

67 |

IS_PRODUCTION={} IS_LOCAL_DEV={} Started ago={}

68 | """.format(len(apilist), config.IS_PRODUCTION, config.IS_LOCAL_DEV, 69 | config.started_ago(True)) 70 | footer = "
" 71 | 72 | return header + title + "
".join(apilist) + footer 73 | 74 | 75 | if config.IS_LOCAL_DEV: 76 | @app.route('/apitest/dbtruncate', methods = ['POST']) 77 | def truncate(): 78 | """For testing: Empty all data from all tables. An external test script 79 | can call this at start. Only accessible in local dev machine.""" 80 | 81 | cursor = db.database.execute_sql("truncate users, movies") 82 | return jsonify({}), 200 83 | 84 | 85 | @app.route('/apitest/sendemail', methods = ['GET']) 86 | def send(): 87 | """For testing: Example of activating a background task.""" 88 | 89 | log.info("executing a background task") 90 | 91 | bgtasks.send_email.spool(email="tomi@tomicloud.com", 92 | subject="Hello world!", template="welcome.html") 93 | 94 | return jsonify({"reply":"background task will start"}), 200 95 | 96 | 97 | @app.route('/apitest/counter', methods = ['GET']) 98 | def testcounter(): 99 | """For testing: Increment redis counter.""" 100 | 101 | num = red.incr("testcounter") 102 | return jsonify({"counter":num}), 200 103 | 104 | 105 | @app.route('/examplehtml', methods = ['GET']) 106 | def htmlpage(): 107 | """For testing: Example HTML page, if you want to use templates.""" 108 | 109 | # just some data for the template 110 | clock = datetime.datetime.now() 111 | 112 | return render_template('example.html', clock=clock) 113 | 114 | -------------------------------------------------------------------------------- /py/api_movies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # api.py: REST API for movies 5 | # - this module is just demonstrating how to handle basic CRUD 6 | # - GET operations are available for visitors, editing requires login 7 | 8 | # Author: Tomi.Mickelsson@iki.fi 9 | 10 | from flask import request, jsonify, g 11 | from playhouse.shortcuts import dict_to_model, update_model_from_dict 12 | 13 | import db 14 | import util 15 | from webutil import app, login_required, get_myself 16 | 17 | import logging 18 | log = logging.getLogger("api.movies") 19 | 20 | 21 | @app.route('/api/movies/', methods = ['GET']) 22 | def movie_query(): 23 | """Returns list of movies that match the given search criteria: page, limit, 24 | search, creator.""" 25 | 26 | input = request.args 27 | page = input.get("page") 28 | limit = input.get("limit") 29 | s = input.get("search") 30 | u = input.get("creator") 31 | 32 | movielist = db.query_movies(search=s, creator=u, page=page, limit=limit) 33 | 34 | return jsonify(movielist), 200 35 | 36 | 37 | @app.route('/api/movies/', methods = ['GET']) 38 | def movie_get(id): 39 | """Returns a single movie, or 404.""" 40 | 41 | m = db.get_movie(id) 42 | return jsonify(m), 200 43 | 44 | 45 | @app.route('/api/movies/', methods = ['POST']) 46 | @login_required(role='editor') 47 | def movie_create(): 48 | """Creates a movie and returns it.""" 49 | 50 | input = request.json 51 | input.pop("id", 0) # ignore id if given, is set by db 52 | 53 | m = dict_to_model(db.Movie, input) 54 | m.modified = m.created = util.utcnow() 55 | m.creator = get_myself() 56 | m.save() 57 | 58 | return jsonify(m), 201 59 | 60 | 61 | @app.route('/api/movies/', methods = ['PUT']) 62 | @login_required(role='editor') 63 | def movie_update(id): 64 | """Updates a movie and returns it.""" 65 | 66 | input = request.json 67 | # don't update created/creator-fields 68 | input.pop("created", 0) 69 | input.pop("creator", 0) 70 | 71 | m = db.get_movie(id) 72 | update_model_from_dict(m, input) 73 | m.modified = util.utcnow() 74 | m.save() 75 | 76 | return jsonify(m), 200 77 | 78 | 79 | @app.route('/api/movies/', methods = ['DELETE']) 80 | @login_required(role='editor') 81 | def movie_delete(id): 82 | """Deletes a movie.""" 83 | 84 | m = db.get_movie(id) 85 | m.delete_instance() 86 | 87 | return jsonify(m), 200 88 | 89 | -------------------------------------------------------------------------------- /py/bgtasks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # bgtasks.py: background tasks, which are run in separate worker processes 5 | # - execute these just by spooling arguments to functions, like: 6 | # bgtasks.send_email.spool(arg_list_here) 7 | # 8 | # Author: Tomi.Mickelsson@iki.fi 9 | 10 | from uwsgidecorators import spool 11 | 12 | import db 13 | import util 14 | import webutil 15 | import config 16 | import time 17 | import uwsgi 18 | 19 | import logging 20 | log = logging.getLogger("bgtasks") 21 | 22 | 23 | @spool(pass_arguments=True) 24 | def send_email(*args, **kwargs): 25 | """A background worker that is executed by spooling arguments to it.""" 26 | 27 | log.info("send_email started, got arguments: {} {}".format(args, kwargs)) 28 | 29 | try: 30 | log.info("processing emails...") 31 | 32 | # do the stuff... 33 | time.sleep(3) 34 | 35 | log.info("processing done!") 36 | 37 | except: 38 | log.exception("send_email") 39 | 40 | # returning SPOOL_OK here signals to uwsgi to not retry this task 41 | # if the exception propagates up, uwsgi will call us again in 42 | # N secs, configured in uwsgi.ini: spooler-frequency 43 | return uwsgi.SPOOL_OK 44 | 45 | -------------------------------------------------------------------------------- /py/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # config.py: configuration data of this app 5 | # - other modules should read config from here 6 | # - the config is first read from a json file 7 | # - env variables may override (in docker setup etc) 8 | 9 | import sys 10 | import os 11 | import redis 12 | import json 13 | import time 14 | 15 | # first load config from a json file, 16 | srvconf = json.load(open(os.environ["PYSRV_CONFIG_PATH"])) 17 | 18 | # then override with env variables 19 | for k, v in os.environ.items(): 20 | if k.startswith("PYSRV_"): 21 | print("env override ", k) 22 | srvconf[k] = v 23 | 24 | # grand switch to production! 25 | IS_PRODUCTION = bool(srvconf['PYSRV_IS_PRODUCTION'] or False) 26 | 27 | # local dev flag 28 | IS_LOCAL_DEV = os.environ.get("FLASK_ENV") == "development" and not IS_PRODUCTION 29 | # IS_LOCAL_DEV = False 30 | 31 | print("\nCONFIG: prod={},localdev={} ({})\n".format( 32 | IS_PRODUCTION, IS_LOCAL_DEV, srvconf["name"])) 33 | 34 | # database config 35 | DATABASE_HOST = srvconf['PYSRV_DATABASE_HOST'] 36 | DATABASE_PORT = srvconf['PYSRV_DATABASE_PORT'] 37 | DATABASE_NAME = srvconf['PYSRV_DATABASE_NAME'] 38 | DATABASE_USER = srvconf['PYSRV_DATABASE_USER'] 39 | DATABASE_PASSWORD = srvconf['PYSRV_DATABASE_PASSWORD'] 40 | IS_SQLITE = DATABASE_HOST.startswith("/") 41 | 42 | # Flask + session config 43 | # http://flask.pocoo.org/docs/1.0/config/ 44 | # https://pythonhosted.org/Flask-Session/ 45 | redishost = srvconf['PYSRV_REDIS_HOST'] 46 | 47 | flask_config = dict( 48 | # app config 49 | TESTING = IS_LOCAL_DEV, 50 | SECRET_KEY = None, # we have server-side sessions 51 | 52 | # session config - hardcoded to Redis 53 | SESSION_TYPE = 'redis', 54 | SESSION_REDIS = redis.from_url('redis://{}'.format(redishost)), 55 | SESSION_COOKIE_NAME = "mycookie", 56 | SESSION_COOKIE_SECURE = srvconf['PYSRV_COOKIE_HTTPS_ONLY'] if not IS_LOCAL_DEV else False, # require https? 57 | SESSION_COOKIE_HTTPONLY = True, # don't allow JS cookie access 58 | SESSION_KEY_PREFIX = 'pysrv', 59 | PERMANENT_SESSION_LIFETIME = 60*60*24*30, # 1 month 60 | SESSION_COOKIE_DOMAIN = srvconf['PYSRV_DOMAIN_NAME'] or None if not IS_LOCAL_DEV else None, 61 | ) 62 | 63 | # dump sql statements in log file? 64 | PYSRV_LOG_SQL = srvconf.get('PYSRV_LOG_SQL') 65 | 66 | # allow API access to this domain 67 | CORS_ALLOW_ORIGIN = srvconf.get('PYSRV_CORS_ALLOW_ORIGIN', '*') 68 | 69 | START_TIME = int(time.time()) 70 | 71 | 72 | def started_ago(as_string=False): 73 | """Returns how many seconds ago the server was started. Or as a string.""" 74 | 75 | ago = int(time.time()) - START_TIME 76 | if as_string: 77 | return "{}d {:02d}:{:02d}:{:02d}".format(int(ago/60/60/24), 78 | int(ago/60/60)%24, int(ago/60)%60, ago%60) 79 | else: 80 | return ago 81 | 82 | -------------------------------------------------------------------------------- /py/cron.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # cron.py: cron tasks (called by uwsgi daemon, not linux cron) 5 | # 6 | # - Note that these funcs are subject to the same "harakiri" timeout as 7 | # regular requests. If you need to run longer, spool a background worker 8 | # from a cron func. Spooler workers have a timeout "spooler-harakiri" 9 | # specified in uwsgi.ini. 10 | # 11 | # Author: Tomi.Mickelsson@iki.fi 12 | 13 | from uwsgidecorators import timer, cron, filemon 14 | import datetime 15 | 16 | import db 17 | import util 18 | import webutil 19 | import config 20 | import red 21 | 22 | import logging 23 | log = logging.getLogger("cron") 24 | 25 | 26 | @timer(60) 27 | def every_minute(num): 28 | """Runs every minute.""" 29 | 30 | log.info("every_minute") 31 | 32 | 33 | @cron(0,-1,-1,-1,-1) 34 | #(minute, hour, day, month, weekday) - in local time 35 | def every_hour(num): 36 | """Runs every hour at X:00.""" 37 | 38 | log.info("every_hour") 39 | 40 | 41 | @cron(0,2,-1,-1,-1) 42 | def daily(num): 43 | """Runs every night at 2:00AM.""" 44 | 45 | log.info("daily task here - it is 2:00AM") 46 | 47 | # if you have a cluster of servers, this check ensures that 48 | # only one server performs the task 49 | today = str(datetime.date.today()) 50 | if today != red.get_set("nightlycron", today): 51 | daily_single_server() 52 | 53 | def daily_single_server(): 54 | 55 | log.info("daily task here - only 1 server in a cluster runs this") 56 | 57 | 58 | # @filemon("/tmp/foobar") 59 | # def file_has_been_modified(num): 60 | # """Runs when a file has been modified.""" 61 | # log.info("cron task: file has been modified") 62 | 63 | -------------------------------------------------------------------------------- /py/db.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | """db.py: Models and functions for accessing the database 5 | - using peewee orm 6 | - preferably have all SQL in this file 7 | 8 | Author: Tomi.Mickelsson@iki.fi 9 | 10 | http://docs.peewee-orm.com/en/latest/peewee/querying.html 11 | http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#postgres-ext 12 | """ 13 | 14 | from peewee import * 15 | from playhouse.shortcuts import model_to_dict 16 | 17 | from flask import abort 18 | import config 19 | 20 | import logging 21 | log = logging.getLogger("db") 22 | 23 | if config.IS_SQLITE: 24 | # config.DATABASE_HOST is full path to sqlite file 25 | database = SqliteDatabase(config.DATABASE_HOST, pragmas={}) 26 | else: 27 | from playhouse.postgres_ext import PostgresqlExtDatabase, ArrayField, BinaryJSONField, BooleanField, JSONField 28 | # support for arrays of uuid 29 | import psycopg2.extras 30 | psycopg2.extras.register_uuid() 31 | 32 | database = PostgresqlExtDatabase(config.DATABASE_NAME, 33 | user=config.DATABASE_USER, password=config.DATABASE_PASSWORD, 34 | host=config.DATABASE_HOST, port=config.DATABASE_PORT) 35 | 36 | 37 | # -------------------------------------------------------------------------- 38 | # Base model and common methods 39 | 40 | class BaseModel(Model): 41 | """Base class for all database models.""" 42 | 43 | # exclude these fields from the serialized dict 44 | EXCLUDE_FIELDS = [] 45 | 46 | def serialize(self): 47 | """Serialize the model into a dict.""" 48 | d = model_to_dict(self, recurse=False, exclude=self.EXCLUDE_FIELDS) 49 | d["id"] = str(d["id"]) # unification: id is always a string 50 | return d 51 | 52 | class Meta: 53 | database = database 54 | 55 | 56 | def get_object_or_404(model, **kwargs): 57 | """Retrieve a single object or abort with 404.""" 58 | 59 | try: 60 | return model.get(**kwargs) 61 | except model.DoesNotExist: 62 | log.warning("NO OBJECT {} {}".format(model, kwargs)) 63 | abort(404) 64 | 65 | def get_object_or_none(model, **kwargs): 66 | """Retrieve a single object or return None.""" 67 | 68 | try: 69 | return model.get(**kwargs) 70 | except model.DoesNotExist: 71 | return None 72 | 73 | 74 | # -------------------------------------------------------------------------- 75 | # USER 76 | 77 | class User(BaseModel): 78 | 79 | # Should user.id be an integer or uuid? Both have pros and cons. 80 | # Since user.id is sensitive data, I selected uuid here. 81 | if not config.IS_SQLITE: 82 | id = UUIDField(primary_key=True) 83 | id.auto_increment = True # is auto generated by server 84 | 85 | email = TextField() 86 | password = TextField() 87 | first_name = TextField() 88 | last_name = TextField() 89 | role = TextField() 90 | if not config.IS_SQLITE: 91 | tags = ArrayField(TextField) 92 | else: 93 | tags = TextField() 94 | 95 | created = DateTimeField() 96 | modified = DateTimeField() 97 | 98 | EXCLUDE_FIELDS = [password] # never expose password 99 | 100 | 101 | def is_superuser(self): 102 | return self.role == "superuser" 103 | 104 | def full_name(self): 105 | return "{} {}".format(self.first_name, self.last_name or '') 106 | 107 | def serialize(self): 108 | """Serialize this object to dict/json.""" 109 | 110 | d = super(User, self).serialize() 111 | 112 | # add extra data 113 | d["fullname"] = self.full_name() 114 | d["tags"] = self.tags or [] # never None 115 | return d 116 | 117 | def __str__(self): 118 | return "".format(self.id, 119 | self.email, self.role) 120 | 121 | class Meta: 122 | db_table = 'users' 123 | 124 | 125 | def get_user(uid): 126 | """Return user object or throw.""" 127 | return get_object_or_404(User, id=uid) 128 | 129 | 130 | def get_user_by_email(email): 131 | """Return user object or None""" 132 | 133 | if not email: 134 | return None 135 | 136 | try: 137 | # return User.select().where(User.email == email).get() 138 | # case insensitive query 139 | if config.IS_SQLITE: 140 | sql = "SELECT * FROM users where email = ? LIMIT 1" 141 | args = email.lower() 142 | else: 143 | sql = "SELECT * FROM users where LOWER(email) = LOWER(%s) LIMIT 1" 144 | args = (email,) 145 | return list(User.raw(sql, args))[0] 146 | 147 | except IndexError: 148 | return None 149 | 150 | 151 | def query_users(page=0, limit=1000, search=None): 152 | """Return list of users. Desc order""" 153 | 154 | page = int(page or 0) 155 | limit = int(limit or 1000) 156 | 157 | q = User.select() 158 | if search: 159 | search = "%"+search+"%" 160 | q = q.where(User.first_name ** search | User.last_name ** search | 161 | User.email ** search) 162 | q = q.paginate(page, limit).order_by(User.id.desc()) 163 | return q 164 | 165 | 166 | # -------------------------------------------------------------------------- 167 | # MOVIE - just an example for CRUD API... 168 | 169 | class Movie(BaseModel): 170 | 171 | #id - automatic 172 | 173 | title = TextField() 174 | director = TextField() 175 | 176 | created = DateTimeField() 177 | modified = DateTimeField() 178 | 179 | creator = ForeignKeyField(db_column='creator', null=True, 180 | model=User, to_field='id') 181 | 182 | class Meta: 183 | db_table = 'movies' 184 | 185 | 186 | def get_movie(id): 187 | """Return Movie or throw.""" 188 | return get_object_or_404(Movie, id=id) 189 | 190 | 191 | def query_movies(page=None, limit=None, search='', creator=None): 192 | """Return list of movies which match given filters.""" 193 | 194 | page = page or 0 195 | limit = limit or 1000 196 | 197 | q = Movie.select() 198 | 199 | if search: 200 | search = "%"+search+"%" 201 | q = q.where(Movie.title ** search | Movie.director ** search) 202 | 203 | if creator: 204 | q = q.where(Movie.creator == creator) 205 | 206 | q = q.paginate(page, limit).order_by(Movie.id) 207 | return q 208 | 209 | 210 | def query_unique_directors(): 211 | """Return list of unique directors. An example of a raw SQL query.""" 212 | 213 | sql = "SELECT DISTINCT(director) FROM movies" 214 | rq = database.execute_sql(sql) 215 | return [x[0] for x in rq] 216 | 217 | 218 | # -------------------------------------------------------------------------- 219 | 220 | if __name__ == '__main__': 221 | 222 | # quick adhoc tests 223 | logging.basicConfig(level=logging.DEBUG) 224 | 225 | u = User(first_name="tomi") 226 | u.email = "myemail@example.org" 227 | u.save(force_insert=True) 228 | print(u) 229 | 230 | print(list(query_users(0, "10", ".com"))) 231 | 232 | print(list(query_movies())) 233 | print(query_unique_directors()) 234 | 235 | -------------------------------------------------------------------------------- /py/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # main.py: server main script 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | # register REST API endpoints 9 | import api_account 10 | import api_dev 11 | import api_movies 12 | 13 | import ui_auth # quick login/signup pages 14 | 15 | import cron 16 | 17 | import logging 18 | log = logging.getLogger("main") 19 | 20 | log.info("Running! http://localhost:8100") 21 | 22 | from webutil import app 23 | if app.testing: 24 | import werkzeug.debug 25 | app.wsgi_app = werkzeug.debug.DebuggedApplication(app.wsgi_app, True) 26 | # uwsgi-daemon takes over the app... 27 | 28 | -------------------------------------------------------------------------------- /py/mule1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # mule1.py: independent worker process 5 | # - a TCP server as an example 6 | 7 | import time 8 | import socketserver 9 | 10 | import logging 11 | log = logging.getLogger("mule") 12 | 13 | 14 | # example from https://docs.python.org/3/library/socketserver.html 15 | class MyTCPServer(socketserver.BaseRequestHandler): 16 | """Simplest TCP server that echoes back the message.""" 17 | 18 | def handle(self): 19 | self.data = self.request.recv(1024).strip() 20 | log.info(f"got data: {self.data}") 21 | self.request.sendall(self.data.upper()+b"\n") 22 | 23 | 24 | def main(): 25 | """Mule main function""" 26 | 27 | HOST = "0.0.0.0" 28 | PORT = 9999 29 | 30 | log.info(f"example mule worker started, listening on TCP port {PORT}") 31 | 32 | # create a TCP server 33 | with socketserver.TCPServer((HOST, PORT), MyTCPServer) as server: 34 | server.serve_forever() 35 | 36 | # never reached in this example 37 | # if you return from a worker, uwsgi will automatically restart 38 | log.info("mule end") 39 | 40 | 41 | if __name__ == '__main__': 42 | main() 43 | 44 | -------------------------------------------------------------------------------- /py/red.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # red.py: read/write data in Redis 5 | # - get/set key values with expiration time 6 | # - simple list operations 7 | # - atomic increment, getset 8 | # 9 | # Note: stores pickled data in Redis. If you want to interoperate with the 10 | # data with other tools, you'd better change pickle to json (a bit slower 11 | # but interoperates better). 12 | # 13 | # https://redis.io/commands 14 | # https://github.com/andymccurdy/redis-py 15 | # 16 | # Author: Tomi.Mickelsson@iki.fi 17 | 18 | import redis 19 | import pickle 20 | 21 | import config 22 | 23 | import logging 24 | log = logging.getLogger("cache") 25 | 26 | 27 | #rdb = redis.StrictRedis(host=config.redishost) 28 | rdb = redis.from_url('redis://{}'.format(config.redishost)) 29 | 30 | # -------------------------------------------------------------------------- 31 | # key values 32 | 33 | def set_keyval(key, val, expiration_secs=0): 34 | """Sets key value. Value can be any object. Key is optionally discarded 35 | after given seconds.""" 36 | 37 | try: 38 | s = pickle.dumps(val) 39 | rdb.set(key, s, expiration_secs or None) 40 | except: 41 | log.error("redis set_keyval %s", key) 42 | 43 | def get_keyval(key, default=None): 44 | """Returns key value or default if key is missing.""" 45 | 46 | try: 47 | v = rdb.get(key) 48 | return pickle.loads(v) if v else default 49 | except: 50 | log.error("redis get_keyval %s", key) 51 | return default 52 | 53 | def delete_key(key): 54 | """Deletes key. Can be a list too.""" 55 | 56 | try: 57 | rdb.delete(key) 58 | except: 59 | log.error("redis del %s", key) 60 | 61 | 62 | # -------------------------------------------------------------------------- 63 | # list operations 64 | 65 | def list_append(name, item, max_size=None): 66 | """Inserts item at the end of the list. If max_size is given, truncates 67 | the list into max size, discarding the oldest items.""" 68 | try: 69 | s = pickle.dumps(item) 70 | rdb.rpush(name, s) 71 | 72 | if max_size: 73 | rdb.ltrim(name, -int(max_size), -1) 74 | except: 75 | log.error("redis list_append") 76 | 77 | def list_pop(name, timeout=None): 78 | """Returns first item in the list. If timeout is given, wait that many 79 | seconds.""" 80 | 81 | if timeout != None: 82 | s = rdb.blpop(name, timeout=timeout) 83 | if s: 84 | s = s[1] # with timeout, value is the 2nd item 85 | else: 86 | s = rdb.lpop(name) 87 | return pickle.loads(s) if s else None 88 | 89 | def list_peek(name): 90 | """Returns first item in queue or None. Does not remove the item.""" 91 | s = rdb.lrange(name, 0, 0) 92 | return pickle.loads(s[0]) if s else None 93 | 94 | def list_fetch(name): 95 | """Returns all items in queue or None. Does not remove the items.""" 96 | slist = rdb.lrange(name, 0, -1) 97 | if slist: 98 | return [pickle.loads(s) for s in slist] 99 | 100 | def list_length(name): 101 | """Returns the length of the list.""" 102 | return rdb.llen(name) 103 | 104 | 105 | # -------------------------------------------------------------------------- 106 | # atomic operations 107 | 108 | def incr(name, num=1): 109 | """Increments a count by num. Returns value after increment.""" 110 | return rdb.incrby(name, num) 111 | 112 | def get_set(key, val): 113 | """Sets key value atomically. Returns the previous value.""" 114 | s = pickle.dumps(val) 115 | oldval = rdb.getset(key, s) 116 | return pickle.loads(oldval) if oldval else None 117 | 118 | -------------------------------------------------------------------------------- /py/ui_auth.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # ui_auth.py: quick HTML pages for login/signup/forgot/reset password 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | from flask import redirect, request, render_template 9 | 10 | import db 11 | import account 12 | import util 13 | import red 14 | from webutil import app, get_ip 15 | 16 | import logging 17 | log = logging.getLogger("uiauth") 18 | 19 | 20 | @app.route('/auth/login', methods = ['GET']) 21 | def ui_login(): 22 | """Login page""" 23 | return render_template('auth.html', mode="login") 24 | 25 | 26 | @app.route('/auth/signup', methods = ['GET']) 27 | def ui_signup(): 28 | """Signup page""" 29 | return render_template('auth.html', mode="signup") 30 | 31 | 32 | @app.route('/auth/forgot', methods = ['GET']) 33 | def ui_forgot(): 34 | """Forgot password page""" 35 | return render_template('auth.html', mode="forgot") 36 | 37 | 38 | @app.route('/auth/reset', methods = ['GET']) 39 | def ui_reset(): 40 | """Reset password page""" 41 | 42 | # token must point to a user 43 | token = request.args.get("token") or "-" 44 | data = red.get_keyval(token) 45 | errmsg = "Token is missing or expired" if not data else "" 46 | 47 | # show email 48 | email = "?" 49 | try: 50 | log.info(f"reset token={token} data={data}") 51 | u = db.get_user(data["uid"]) 52 | email = u.email 53 | except: 54 | log.error(f"no user {data}") 55 | 56 | return render_template('auth.html', mode="reset", 57 | email=email, err=errmsg, token=token) 58 | 59 | 60 | @app.route('/auth/postform', methods = ['POST']) 61 | def postform(): 62 | """Form POST endpoint for all form variations.""" 63 | 64 | input = request.form 65 | mode = input["mode"] 66 | email = input["email"] 67 | passwd = input.get("passwd") 68 | token = input.get("token") 69 | 70 | u = db.get_user_by_email(email) 71 | 72 | errmsg = "" 73 | if not email: 74 | errmsg = "Email is missing" 75 | 76 | elif mode == "login": 77 | if not u or not account.check_password(u.password, passwd): 78 | errmsg = "Invalid login credentials" 79 | else: 80 | account.build_session(u, is_permanent=True) 81 | 82 | log.info(f"LOGIN OK {email}") 83 | 84 | # you should redirect to real ui... 85 | return redirect("/api/me") 86 | 87 | elif mode == "signup": 88 | if u: 89 | errmsg = f"Account exists already {email}" 90 | elif passwd != input.get("passwd2"): 91 | errmsg = f"Passwords differ" 92 | else: 93 | errmsg = account.check_password_validity(passwd) 94 | if not errmsg: 95 | # create new user 96 | u = db.User() 97 | u.email = email 98 | u.first_name = input["firstname"] 99 | u.last_name = input["lastname"] 100 | u.password = account.hash_password(passwd) 101 | u.role = 'editor' # set default to what makes sense to your app 102 | u.save(force_insert=True) 103 | 104 | account.new_signup_steps(u) 105 | account.build_session(u, is_permanent=True) 106 | 107 | log.info(f"SIGNUP OK {email}") 108 | 109 | # you should redirect to real ui... 110 | return redirect("/api/me") 111 | 112 | elif mode == "forgot": 113 | # request a new password 114 | if u: 115 | # generate an expiring token and store in redis 116 | token = str(util.generate_token()) 117 | data = {"uid":f"{u.id}", "ip":get_ip()} 118 | expire_secs = 60*60 # 1h 119 | red.set_keyval(token, data, expire_secs) 120 | 121 | # email the link to the user 122 | link = f"DOMAIN/auth/reset?token={token}" 123 | errmsg = f"Server should now send a reset email to {email}..." 124 | log.info(f"password reset link = {link}") 125 | 126 | else: 127 | errmsg = f"Unknown account {email}" 128 | 129 | elif mode == "reset": 130 | # reset a password 131 | data = red.get_keyval(token) 132 | if data: 133 | try: 134 | u = db.get_user(data["uid"]) 135 | 136 | # extra security: make sure ip addresses match, only the 137 | # requester can use the link 138 | if get_ip() != data["ip"]: 139 | errmsg = "Invalid IP" 140 | 141 | elif passwd != input.get("passwd2"): 142 | errmsg = "Passwords differ" 143 | 144 | else: 145 | # ok, reset the password 146 | u.password = account.hash_password(passwd) 147 | u.save() 148 | account.build_session(u, is_permanent=True) 149 | 150 | # security: disable link from further use 151 | red.delete_key(token) 152 | 153 | log.info(f"PASSWD RESET OK {email}") 154 | return redirect("/api/me") 155 | 156 | except: 157 | log.error(f"no user {value}") 158 | errmsg = "Invalid token" 159 | else: 160 | errmsg = "Invalid token" 161 | 162 | if errmsg: 163 | log.warn(errmsg) 164 | 165 | return render_template('auth.html', mode=mode, email=email, 166 | err=errmsg, token=token) 167 | 168 | -------------------------------------------------------------------------------- /py/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # util.py: utility functions 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | import pytz 9 | import datetime 10 | import time 11 | import uuid 12 | import functools 13 | 14 | import logging 15 | log = logging.getLogger("util") 16 | 17 | 18 | # -------------------------------------------------------------------------- 19 | # date related common methods 20 | 21 | tz_hki = pytz.timezone("Europe/Helsinki") 22 | tz_utc = pytz.utc 23 | 24 | def utc2local(utc_dt, tz=tz_hki): 25 | """Convert UTC into local time, given tz.""" 26 | 27 | if not utc_dt: 28 | return utc_dt 29 | 30 | d = utc_dt.replace(tzinfo=tz_utc) 31 | return d.astimezone(tz) 32 | 33 | def local2utc(local_dt, tz=tz_hki): 34 | """Convert local time into UTC.""" 35 | 36 | if not local_dt: 37 | return local_dt 38 | 39 | d = local_dt.replace(tzinfo=tz) 40 | return d.astimezone(tz_utc) 41 | 42 | def utcnow(): 43 | """Return UTC now.""" 44 | return datetime.datetime.utcnow() 45 | 46 | def generate_token(): 47 | """Generate a random token 48 | (an uuid like 8491997531e44d37ac3105b300774e08)""" 49 | return uuid.uuid4().hex 50 | 51 | def timeit(f): 52 | """Decorator to measure function execution time.""" 53 | @functools.wraps(f) 54 | def wrap(*args, **kw): 55 | t1 = time.time() 56 | result = f(*args, **kw) 57 | t2 = time.time() 58 | log.info("%r args:[%r, %r] took: %2.4f sec" % \ 59 | (f.__name__, args, kw, t2-t1)) 60 | return result 61 | return wrap 62 | 63 | 64 | if __name__ == '__main__': 65 | 66 | # quick adhoc tests 67 | logging.basicConfig(level=logging.DEBUG) 68 | 69 | @timeit 70 | def myfunc(): 71 | now = utcnow() 72 | print(now) 73 | print(utc2local(now)) 74 | time.sleep(1.0) 75 | myfunc() 76 | 77 | -------------------------------------------------------------------------------- /py/webutil.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # webutil.py: low level page request related methods, decorators, Flask app 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | import time 9 | import peewee 10 | import functools 11 | from flask import Flask, request, session, g, redirect, abort, jsonify 12 | from flask_session import Session 13 | from flask.json.provider import DefaultJSONProvider 14 | 15 | import db 16 | import config 17 | import datetime 18 | 19 | import logging 20 | log = logging.getLogger("webutil") 21 | 22 | 23 | # create and configure the Flask app 24 | app = Flask(__name__, static_folder=None, template_folder="../templates") 25 | app.config.update(config.flask_config) 26 | Session(app) 27 | 28 | 29 | # -------------------------------------------------------------------------- 30 | # API decorator 31 | 32 | def login_required(func=None, role=None): 33 | """Decorator: must be logged on, and optionally must have the given role. 34 | Insert after app.route like this: 35 | @app.route('/api/users') 36 | @login_required(role='superuser')""" 37 | 38 | # yes, this is python magic, see https://blogs.it.ox.ac.uk/inapickle/2012/01/05/python-decorators-with-optional-arguments/ 39 | if not func: 40 | return functools.partial(login_required, role=role) 41 | @functools.wraps(func) 42 | def inner(*args, **kwargs): 43 | return _check_user_role(role) or func(*args, **kwargs) 44 | return inner 45 | 46 | 47 | # -------------------------------------------------------------------------- 48 | # get data about me, return error replys 49 | 50 | def get_myself(): 51 | """Return the user object of the caller or None if he is a visitor. 52 | Loads the user from the database, then caches it during request.""" 53 | 54 | if not "userid" in session: 55 | return None 56 | 57 | if hasattr(g, "MYSELF"): 58 | return g.MYSELF # use cache 59 | else: 60 | g.MYSELF = db.get_user(session["userid"]) 61 | return g.MYSELF 62 | 63 | def error_reply(errmsg, httpcode=400): 64 | """Logs an error and returns error code to the caller.""" 65 | log.error(errmsg) 66 | return jsonify({"err":"{}: {}".format(httpcode, errmsg)}), httpcode 67 | 68 | def warn_reply(errmsg, httpcode=400): 69 | """Logs a warning and returns error code to the caller.""" 70 | log.warning(errmsg) 71 | return jsonify({"err":"{}: {}".format(httpcode, errmsg)}), httpcode 72 | 73 | def get_agent(): 74 | """Returns browser of caller.""" 75 | return request.headers.get('User-Agent', '') 76 | 77 | def get_ip(): 78 | """Returns IP address of caller.""" 79 | return request.headers.get('X-Real-IP') or request.remote_addr 80 | 81 | 82 | # -------------------------------------------------------------------------- 83 | # before/after/error request handlers 84 | 85 | @app.before_request 86 | def before_request(): 87 | """Executed always before a request. Connects to db, logs the request, 88 | prepares global data, loads current user.""" 89 | 90 | # log request path+input, but not secrets 91 | try: 92 | params = request.json or request.args or request.form 93 | except: 94 | params = None 95 | if params: 96 | cloned = None 97 | secret_keys = ["password", "passwd", "pwd"] 98 | for k in secret_keys: 99 | if k in params: 100 | if not cloned: 101 | cloned = params.copy() 102 | cloned[k] = 'X' 103 | if cloned: 104 | params = cloned 105 | 106 | params = str(params or '')[:1000] 107 | method = request.method[:2] 108 | log.info("{} {} {}".format(method, request.path, params)) 109 | 110 | # connect to db 111 | g.db = db.database 112 | g.db.connection() 113 | 114 | # have common data available in global g 115 | # but do not pollute g, store only the most relevant data 116 | g.HOST = request.headers.get('X-Real-Host', '') 117 | g.ISLOGGED = "userid" in session 118 | myrole = session.get("role") or "" 119 | g.IS_SUPER_USER = myrole == "superuser" 120 | 121 | if myrole == "disabled": 122 | err = "account disabled" 123 | log.warn(err) 124 | return jsonify({"err":err}), 400 125 | 126 | # time the request 127 | g.t1 = time.time() 128 | 129 | # where did we link from? (but filter our internal links) 130 | # if request.referrer: 131 | # log.info("linked from "+request.referrer) 132 | 133 | 134 | @app.after_request 135 | def after_request(response): 136 | """Executed after a request, unless a request occurred.""" 137 | 138 | # log about error 139 | logmethod = None 140 | if 400 <= response.status_code <= 599: 141 | logmethod = log.error 142 | elif not 200 <= response.status_code < 399: 143 | logmethod = log.warn 144 | if logmethod: 145 | logmethod(" {} {} {}".format(response.status_code, 146 | request.method, request.url)) 147 | 148 | # set CORS headers 149 | response.headers['Access-Control-Allow-Origin'] = config.CORS_ALLOW_ORIGIN 150 | response.headers['Access-Control-Allow-Methods'] = 'GET,POST,PUT,DELETE,OPTIONS' 151 | response.headers['Access-Control-Allow-Headers'] = 'Content-Type' 152 | response.headers['Access-Control-Allow-Credentials'] = 'true' 153 | # response.headers['Access-Control-Expose-Headers'] = 'Access-Control-Allow-Origin' 154 | 155 | return response 156 | 157 | @app.teardown_request 158 | def teardown(error): 159 | """Always executed after a request.""" 160 | 161 | if hasattr(g, "db"): 162 | g.db.close() 163 | 164 | # log warning when a request takes >1.0sec 165 | # (put long-running tasks into background) 166 | if hasattr(g, "t1"): 167 | delta = time.time()-g.t1 168 | if delta > 1.0: 169 | log.warn("SLOW! {} time={}".format(request.path, delta)) 170 | 171 | 172 | @app.errorhandler(404) 173 | def page_not_found(error): 174 | err = "404: " + request.path 175 | return jsonify({"err":err}), 404 176 | 177 | 178 | # -------------------------------------------------------------------------- 179 | # logging (is in this module because binds to session) 180 | 181 | class ColorFormatter(logging.Formatter): 182 | """Colorize warnings and errors""" 183 | 184 | def format(self, rec): 185 | if rec.levelno == logging.WARNING: 186 | rec.msg = "\033[93m{}\033[0m".format(rec.msg) 187 | elif rec.levelno in (logging.ERROR, logging.CRITICAL): 188 | rec.msg = "\033[91m{}\033[0m".format(rec.msg) 189 | return logging.Formatter.format(self, rec) 190 | 191 | 192 | class MyLogContextFilter(logging.Filter): 193 | """Injects contextual info, ip+userid, into the log.""" 194 | 195 | def filter(self, record): 196 | if request: 197 | # take ip from a header or actual 198 | ip = get_ip() 199 | # take userid from the session 200 | uid = session.get("userid", "anon") 201 | else: 202 | ip = "" 203 | uid = " -WORKER" # background worker 204 | 205 | record.ip = "local" if config.IS_LOCAL_DEV else ip 206 | record.uid = uid 207 | return True 208 | 209 | 210 | def init_logging(): 211 | """Initialize logging system.""" 212 | 213 | prefix = "PROD " if config.IS_PRODUCTION else "" 214 | format = prefix+"%(levelname)3.3s %(uid)s@%(ip)s %(asctime)s %(filename)s %(message)s" 215 | dfmt = "%d%m%y-%H:%M:%S" 216 | logging.basicConfig(level=logging.INFO, format=format, datefmt=dfmt) 217 | 218 | formatter = ColorFormatter(format, datefmt=dfmt) 219 | 220 | # custom log data: userid + ip addr 221 | f = MyLogContextFilter() 222 | for handler in logging.root.handlers: 223 | handler.addFilter(f) 224 | handler.setFormatter(formatter) # remove if coloring not wanted 225 | 226 | if config.PYSRV_LOG_SQL: 227 | logging.getLogger('peewee').setLevel(logging.DEBUG) 228 | 229 | 230 | # -------------------------------------------------------------------------- 231 | # internal methods, serializing models 232 | 233 | def _check_user_role(rolebase): 234 | """Check that my role is atleast the given role. If not, log and return 235 | an error.""" 236 | 237 | myrole = session.get("role") or "" 238 | 239 | if not _is_role_atleast(myrole, rolebase): 240 | uid = session.get("userid") or "" 241 | err = "Unauthorized! {} {} user={}".format( 242 | request.method, request.path, uid) 243 | return warn_reply(err, 401) 244 | 245 | def _is_role_atleast(myrole, rolebase): 246 | """Checks that myrole is same or above rolebase. Assumes a 247 | simple role model where roles can be arranged from lowest 248 | access to highest access level.""" 249 | 250 | if not rolebase: 251 | # no role required, but I need to be logged-on 252 | return "userid" in session 253 | 254 | levels = {"readonly":1, "editor":2, "admin":3, "superuser":4} 255 | try: 256 | return levels[myrole] >= levels[rolebase] 257 | except: 258 | return False 259 | 260 | 261 | class MyJSONEncoder(DefaultJSONProvider): 262 | def default(self, obj): 263 | # print(type(obj)) 264 | 265 | if isinstance(obj, peewee.SelectBase): 266 | return list(obj) 267 | elif isinstance(obj, db.BaseModel): 268 | return obj.serialize() 269 | elif isinstance(obj, datetime.datetime): 270 | return obj.isoformat() if obj else None 271 | #elif isinstance(obj, sqlite3.Cursor): 272 | #return list(obj) 273 | #if isinstance(obj, psycopg2.extensions.cursor): 274 | #return list(obj) 275 | return DefaultJSONProvider.default(obj) 276 | 277 | app.json = MyJSONEncoder(app) 278 | 279 | init_logging() 280 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==2.3.2 2 | Flask-Session2==1.3.1 3 | passlib==1.7.4 4 | peewee==3.16.2 5 | peewee-migrate==1.7.1 6 | psycopg2-binary==2.9.6 7 | pytz==2022.7.1 8 | redis==4.5.4 9 | requests==2.32.0 10 | uwsgidecorators==1.1.0 11 | -------------------------------------------------------------------------------- /rsync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # rsync files to server and reload 3 | 4 | # HOST: replace with your real data 5 | HOST='pi@192.168.100.10' 6 | 7 | echo "RSYNCING in 3secs..." 8 | sleep 3 9 | 10 | rsync -av --exclude '.git' --exclude '__pycache__' --exclude '*.pyc' --exclude '*.sqlite' * $HOST:/app/ 11 | 12 | # ask python server to reload sources 13 | ssh $HOST touch /app/RESTART 14 | 15 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # run in dev mode 3 | 4 | docker run -it --rm --name restpie-dev -p 8100:80 -v `pwd`/:/app/ restpie-dev-image 5 | 6 | -------------------------------------------------------------------------------- /scripts/dbmigrate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # dbmigrate.py: migrate the local database 5 | # - run either on dev machine or at server 6 | # 7 | # Author: Tomi.Mickelsson@iki.fi 8 | 9 | import sys 10 | import os 11 | import config 12 | 13 | if config.DATABASE_HOST.startswith("/"): 14 | # sqlite 15 | # note: can't use full path here! 16 | # db will appear in "/app/data/mydb.sqlite" (mapped volume locally) 17 | cmd = "pw_migrate migrate --directory=/app/migrations_sqlite --database=sqlite:/data/mydb.sqlite" 18 | else: 19 | # postgresql 20 | cmd = "pw_migrate migrate --database=postgresql://{}:{}@{}:{}/{}".format( 21 | config.DATABASE_USER, 22 | config.DATABASE_PASSWORD, 23 | config.DATABASE_HOST, 24 | config.DATABASE_PORT, 25 | config.DATABASE_NAME) 26 | 27 | print(cmd) 28 | 29 | ret = os.system(cmd) 30 | if ret: 31 | print("migrate ERROR", ret) 32 | else: 33 | print("migrate OK") 34 | 35 | sys.exit(ret) 36 | -------------------------------------------------------------------------------- /shell.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # run interactive shell inside docker instance 3 | 4 | docker exec -it restpie-dev bash -l 5 | 6 | -------------------------------------------------------------------------------- /templates/auth.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | RESTPie3 Auth 6 | 7 | 8 | 9 | {# logo image #} 10 | 11 | 12 | {% if err %} 13 |

{{err}}

14 | {% endif %} 15 | 16 | {% if mode == "login" %} 17 |

Login

18 | {% elif mode == "signup" %} 19 |

Signup

20 | {% elif mode == "forgot" %} 21 |

Forgot password

22 | {% elif mode == "reset" %} 23 |

Reset password

24 | {% endif %} 25 | 26 |
27 | 28 | 31 | 32 | {% if mode == "signup" %} 33 | 34 | 36 | 37 | 39 | {% endif %} 40 | 41 | {% if mode != "forgot" %} 42 | 43 | 44 | 45 | {% endif %} 46 | {% if mode == "signup" or mode == "reset" %} 47 | 48 | 49 | {% endif %} 50 | 51 |

52 | 53 | {% if token %} 54 | 55 | {% endif %} 56 | 57 |

58 | 59 |
60 | 61 |

62 | {% if mode != "login" %}Login{% endif %} 63 | {% if mode != "signup" %}Signup{% endif %} 64 | {% if mode != "forgot" %}Forgot password{% endif %} 65 |

66 | 67 |

68 | This is a sample login/signup/forgot/reset page to get you quickly started 69 | with the regular auth boilerplate of an app. Just plain HTML and CSS 70 | with zero lines of JS. 71 |

72 | 73 |

Back to REST API

74 | 75 | 129 | 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /templates/example.html: -------------------------------------------------------------------------------- 1 | 2 |

An example HTML page

3 | 4 |

This is primarily a REST API server, but if you want to create a 5 | front-end with traditional templating, here you go.

6 | 7 |

By default, Flask uses 8 | Jinja2 templates.

9 | 10 |

Sample data from the code: {{clock}}

11 | 12 | 13 | -------------------------------------------------------------------------------- /test/api-list.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomimick/restpie3/890c5bdb15ce2ebbcc357797a0a4e8eb7f1ffc92/test/api-list.jpg -------------------------------------------------------------------------------- /test/auth.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomimick/restpie3/890c5bdb15ce2ebbcc357797a0a4e8eb7f1ffc92/test/auth.jpg -------------------------------------------------------------------------------- /test/quick.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # a quick test for the API: do signup and create a new movie 4 | 5 | curl --cookie-jar ./.mycookies -d '{"fname":"tester", "lname":"boy", "email":"testing@localhost.org", "password":"aA12345678"}' -H "Content-Type: application/json" -X POST http://localhost:8100/api/signup 6 | 7 | curl --cookie ./.mycookies -d '{"title":"My great movie"}' -H "Content-Type: application/json" -X POST http://localhost:8100/api/movies/ 8 | 9 | -------------------------------------------------------------------------------- /test/sample.log.txt: -------------------------------------------------------------------------------- 1 | INF anon@local 270918-09:06:59 webutil.py PO /apitest/dbtruncate 2 | INF anon@local 270918-09:06:59 webutil.py GE /api/login 3 | ERR anon@local 270918-09:06:59 webutil.py  405 GET http://localhost:8100/api/login 4 | INF anon@local 270918-09:06:59 webutil.py GE /api/signup 5 | ERR anon@local 270918-09:06:59 webutil.py  405 GET http://localhost:8100/api/signup 6 | INF anon@local 270918-09:06:59 webutil.py GE /api/logout 7 | ERR anon@local 270918-09:06:59 webutil.py  405 GET http://localhost:8100/api/logout 8 | INF anon@local 270918-09:06:59 webutil.py PO /api/login {'email': 'a@example.com', 'password': 'X'} 9 | WAR anon@local 270918-09:06:59 webutil.py Invalid login credentials 10 | ERR anon@local 270918-09:06:59 webutil.py  400 POST http://localhost:8100/api/login 11 | INF anon@local 270918-09:06:59 webutil.py PO /api/signup {'email': 'a@example.com', 'password': 'X'} 12 | WAR anon@local 270918-09:06:59 webutil.py Invalid signup input 13 | ERR anon@local 270918-09:06:59 webutil.py  400 POST http://localhost:8100/api/signup 14 | INF anon@local 270918-09:06:59 webutil.py PO /api/signup {'email': 'tomi@example.com', 'password': 'X', 'fname': 'Tomi', 'lname': 'Mickelsson'} 15 | ERR anon@local 270918-09:06:59 account.py password validity: Password must be atleast 6 characters 16 | ERR anon@local 270918-09:06:59 webutil.py  400 POST http://localhost:8100/api/signup 17 | INF anon@local 270918-09:06:59 webutil.py PO /api/signup {'email': 'tomi@example.com', 'password': 'X', 'fname': 'Tomi', 'lname': 'Mickelsson'} 18 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 api_account.py SIGNUP OK agent=Python API Test 19 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/me 20 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/me 21 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py PO /api/logout 22 | INF anon@local 270918-09:06:59 webutil.py PO /api/me 23 | ERR anon@local 270918-09:06:59 webutil.py  405 POST http://localhost:8100/api/me 24 | INF anon@local 270918-09:06:59 webutil.py GE /api/me 25 | WAR anon@local 270918-09:06:59 webutil.py Unauthorized! GET /api/me user= 26 | ERR anon@local 270918-09:06:59 webutil.py  401 GET http://localhost:8100/api/me 27 | INF anon@local 270918-09:06:59 webutil.py PO /api/signup {'email': 'tomi@example.com', 'password': 'X', 'fname': 'Tomi', 'lname': 'Mickelsson'} 28 | WAR anon@local 270918-09:06:59 webutil.py Signup email taken: tomi@example.com 29 | ERR anon@local 270918-09:06:59 webutil.py  400 POST http://localhost:8100/api/signup 30 | INF anon@local 270918-09:06:59 webutil.py PO /api/login {'email': 'tomi@example.com', 'password': 'X'} 31 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 api_account.py LOGIN OK agent=Python API Test 32 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/users 33 | WAR 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py Unauthorized! GET /api/users user=2770060c-ebea-495f-b93f-509196e4bc73 34 | ERR 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py  401 GET http://localhost:8100/api/users 35 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py PO /api/login {'email': 'tomi@example.com', 'password': 'X'} 36 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 api_account.py LOGIN OK agent=Python API Test 37 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/movies/ 38 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py PO /api/movies/ {'title': 'Forrest Gump', 'director': 'Robert Zemeckis'} 39 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py PO /api/movies/ {'title': 'Matrix', 'director': 'Lana Wachowsk'} 40 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/movies/ 41 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/movies/306 42 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py PU /api/movies/306 {'director': 'Lana Wachowski'} 43 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:06:59 webutil.py GE /api/movies/306 44 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ 45 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ ImmutableMultiDict([('search', 'matrix')]) 46 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ ImmutableMultiDict([('search', 's')]) 47 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ ImmutableMultiDict([('creator', '2770060c-ebea-495f-b93f-509196e4bc73')]) 48 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py DE /api/movies/306 49 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ 50 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py DE /api/movies/305 51 | INF 2770060c-ebea-495f-b93f-509196e4bc73@local 270918-09:07:00 webutil.py GE /api/movies/ 52 | INF -WORKER@local 270918-09:07:22 cron.py every_minute 53 | INF -WORKER@local 270918-09:08:22 cron.py every_minute 54 | SIGINT/SIGQUIT received...killing workers... 55 | killing the spooler with pid 55972 56 | gateway "uWSGI http 1" has been buried (pid: 55974) 57 | spooler (pid: 55972) annihilated 58 | worker 1 buried after 1 seconds 59 | goodbye to uWSGI. 60 | 61 | -------------------------------------------------------------------------------- /test/test_api.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # api_test.py: automated tests for HTTP API 5 | # - not unit tests but ordered set of automated tests 6 | # - the database tables are cleaned on start 7 | # 8 | # Author: Tomi.Mickelsson@iki.fi 9 | 10 | import unittest 11 | import requests 12 | import json 13 | 14 | # URL_BASE = "http://localhost:8100/api/" 15 | URL_BASE = "http://localhost:80/api/" # port is 80 inside docker 16 | 17 | URL_SIGNUP = 'signup' 18 | URL_LOGIN = 'login' 19 | URL_LOGOUT = 'logout' 20 | URL_ME = 'me' 21 | URL_USERS = 'users' 22 | URL_TRUNCATE = "../apitest/dbtruncate" 23 | 24 | URL_MOVIES = 'movies/' 25 | 26 | 27 | s = requests.Session() # remember session 28 | 29 | headers = {'content-type': 'application/json', 30 | 'User-Agent': 'Python API Test'} 31 | 32 | 33 | class Tests(unittest.TestCase): 34 | 35 | inited = False 36 | 37 | def setUp(self): 38 | if not Tests.inited: 39 | # empty all tables before running tests 40 | url = URL_TRUNCATE 41 | self.call(url, payload={}) 42 | Tests.inited = True 43 | 44 | def test001_login_signup_get_fail(self): 45 | self.call(URL_LOGIN, 405) 46 | self.call(URL_SIGNUP, 405) 47 | self.call(URL_LOGOUT, 405) 48 | 49 | def test002_login_failure(self): 50 | payload = {"email":"a@example.com", "password":"y"} 51 | self.call(URL_LOGIN, 400, payload) 52 | 53 | def test003_signup_failure(self): 54 | payload = {"email":"a@example.com", "password":"y"} 55 | self.call(URL_SIGNUP, 400, payload) 56 | 57 | def test004_signup_ok(self): 58 | payload = {"email":"tomi@example.com", "password":"1234", 59 | "fname":"Tomi", "lname":"Mickelsson"} 60 | self.call(URL_SIGNUP, 400, payload) # password too small 61 | payload["password"] = "123abC" 62 | self.call(URL_SIGNUP, 201, payload) 63 | 64 | reply = self.call(URL_ME, 200) 65 | 66 | reply = self.call(URL_ME)["me"] 67 | self.assertIsNotNone(reply.get("id")) 68 | self.assertEqual(reply.get("first_name"), "Tomi") 69 | 70 | self.call(URL_LOGOUT, payload={}) 71 | 72 | self.call(URL_ME, 405, {}) 73 | self.call(URL_ME, 401) 74 | 75 | self.call(URL_SIGNUP, 400, payload) # email used already 76 | 77 | def test005_login_ok(self): 78 | payload = {"email":"tomi@example.com", "password":"123abC"} 79 | self.call(URL_LOGIN, 200, payload) 80 | 81 | def test006_test_roles(self): 82 | self.call(URL_USERS, 401) 83 | 84 | def test007_movies(self): 85 | payload = {"email":"tomi@example.com", "password":"123abC"} 86 | reply = self.call(URL_LOGIN, 200, payload) 87 | my_uid = reply["id"] 88 | 89 | self.assertEqual([], self.call(URL_MOVIES)) 90 | 91 | payload = {"title":"Forrest Gump", "director":"Robert Zemeckis"} 92 | self.call(URL_MOVIES, 201, payload) 93 | payload = {"title":"Matrix", "director":"Lana Wachowsk"} 94 | self.call(URL_MOVIES, 201, payload) 95 | 96 | reply = self.call(URL_MOVIES) 97 | self.assertEqual(2, len(reply)) 98 | self.assertEqual("Forrest Gump", reply[0]["title"]) 99 | self.assertEqual("Lana Wachowsk", reply[1]["director"]) 100 | self.assertEqual(my_uid, reply[1]["creator"]) 101 | 102 | id_gump = reply[0]["id"] 103 | id_matrix = reply[1]["id"] 104 | 105 | reply = self.call(URL_MOVIES+id_matrix) 106 | self.assertEqual("Matrix", reply["title"]) 107 | 108 | payload = {"director":"Lana Wachowski"} 109 | reply = self.call(URL_MOVIES+id_matrix, 200, payload, s.put) 110 | 111 | reply = self.call(URL_MOVIES+id_matrix) 112 | self.assertEqual("Lana Wachowski", reply["director"]) 113 | self.assertEqual(2, len(self.call(URL_MOVIES))) 114 | 115 | reply = self.call(URL_MOVIES+"?search=matrix") 116 | self.assertEqual(1, len(reply)) 117 | 118 | reply = self.call(URL_MOVIES+"?search=s") 119 | self.assertEqual(2, len(reply)) 120 | 121 | reply = self.call(URL_MOVIES+"?creator="+my_uid) 122 | self.assertEqual(2, len(reply)) 123 | 124 | reply = self.call(URL_MOVIES+id_matrix, 200, None, s.delete) 125 | self.assertEqual(1, len(self.call(URL_MOVIES))) 126 | reply = self.call(URL_MOVIES+id_gump, 200, None, s.delete) 127 | self.assertEqual([], self.call(URL_MOVIES)) 128 | 129 | 130 | 131 | def call(self, url, httpcode=200, payload = None, request_method=False): 132 | """GET or POST to server REST API""" 133 | 134 | func = request_method if request_method else s.post if payload != None else s.get 135 | r = func(URL_BASE + url, data = json.dumps(payload or {}), 136 | headers = headers) 137 | 138 | self.assertEqual(r.status_code, httpcode) 139 | if not r.status_code in (401, 405): 140 | return r.json() 141 | 142 | 143 | if __name__ == '__main__': 144 | unittest.main() 145 | 146 | -------------------------------------------------------------------------------- /test/test_redis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # test_redis.py: test red.py, the Redis storage 5 | # 6 | # Author: Tomi.Mickelsson@iki.fi 7 | 8 | import unittest 9 | import time 10 | 11 | import red 12 | 13 | class Tests(unittest.TestCase): 14 | 15 | def test_keyval(self): 16 | VAL = "a1" 17 | red.set_keyval("k1", VAL) 18 | val = red.get_keyval("k1") 19 | self.assertEqual(VAL, val) 20 | 21 | VAL = {"some":1, "thing":[1,2,"3"]} 22 | red.set_keyval("k1", VAL) 23 | val = red.get_keyval("k1") 24 | self.assertEqual(VAL, val) 25 | 26 | red.delete_key("k1") 27 | val = red.get_keyval("k1") 28 | self.assertEqual(None, val) 29 | 30 | val = red.get_keyval("qwerty") 31 | self.assertEqual(None, val) 32 | 33 | val = red.get_keyval("qwerty", "mydefault") 34 | self.assertEqual("mydefault", val) 35 | 36 | def test_keyval_expir(self): 37 | red.delete_key("k1") 38 | val = red.get_keyval("k1") 39 | self.assertEqual(None, val) 40 | 41 | red.set_keyval("k1", 99, 1) 42 | val = red.get_keyval("k1") 43 | self.assertEqual(99, val) 44 | time.sleep(1.1) 45 | val = red.get_keyval("k1") 46 | self.assertEqual(None, val) 47 | 48 | def test_list(self): 49 | red.delete_key("mylist") 50 | self.assertEqual(0, red.list_length("mylist")) 51 | 52 | VAL = {"foo":1, "bar":[1,2]} 53 | red.list_append("mylist", VAL) 54 | self.assertEqual(1, red.list_length("mylist")) 55 | 56 | red.list_append("mylist", "apple") 57 | self.assertEqual(2, red.list_length("mylist")) 58 | 59 | self.assertEqual(VAL, red.list_peek("mylist")) 60 | 61 | curlist = red.list_fetch("mylist") 62 | self.assertEqual(2, len(curlist)) 63 | self.assertEqual("apple", curlist[1]) 64 | 65 | self.assertEqual(VAL, red.list_pop("mylist")) 66 | self.assertEqual(1, red.list_length("mylist")) 67 | self.assertEqual("apple", red.list_pop("mylist")) 68 | self.assertEqual(0, red.list_length("mylist")) 69 | 70 | def test_list_maxsize(self): 71 | red.delete_key("mylist") 72 | self.assertEqual(0, red.list_length("mylist")) 73 | 74 | red.list_append("mylist", "abc", 3) 75 | self.assertEqual(1, red.list_length("mylist")) 76 | self.assertEqual("abc", red.list_peek("mylist")) 77 | red.list_append("mylist", "123", 3) 78 | self.assertEqual(2, red.list_length("mylist")) 79 | self.assertEqual("abc", red.list_peek("mylist")) 80 | red.list_append("mylist", "def", 3) 81 | self.assertEqual(3, red.list_length("mylist")) 82 | self.assertEqual("abc", red.list_peek("mylist")) 83 | red.list_append("mylist", "456", 3) 84 | self.assertEqual(3, red.list_length("mylist")) 85 | self.assertEqual("123", red.list_peek("mylist")) 86 | 87 | def test_incr(self): 88 | red.delete_key("counter") 89 | 90 | self.assertEqual(1, red.incr("counter")) 91 | self.assertEqual(2, red.incr("counter")) 92 | self.assertEqual(5, red.incr("counter", 3)) 93 | 94 | def test_getset(self): 95 | red.delete_key("foo") 96 | 97 | self.assertEqual(None, red.get_set("foo", "orange")) 98 | self.assertEqual("orange", red.get_set("foo", "banana")) 99 | self.assertEqual("banana", red.get_set("foo", "blackberry")) 100 | 101 | 102 | if __name__ == '__main__': 103 | unittest.main() 104 | 105 | -------------------------------------------------------------------------------- /www/index.html: -------------------------------------------------------------------------------- 1 |

RESTPie3 setup works!

2 | 3 |

This static index.html is served by Caddy from folder /app/www/

4 | 5 |

Caddy works as a proxy for RESTPie3 running here.

6 | --------------------------------------------------------------------------------