├── .dockerignore ├── .github └── workflows │ └── push-docker-image.yaml ├── .gitignore ├── README.md ├── api └── swagger.yaml ├── build └── Dockerfile ├── database ├── migrations │ ├── down-migration-001-urls_table.sql │ ├── down-migration-002-address_column.sql │ ├── down-migration-003-enable_column.sql │ ├── down-migration-004-expire_column.sql │ ├── down-migration-005-updated_at_column.sql │ ├── up-migration-001-urls_table.sql │ ├── up-migration-002-address_column.sql │ ├── up-migration-003-enable_column.sql │ ├── up-migration-004-expire_column.sql │ └── up-migration-005-updated_at_column.sql └── model │ ├── __init__.py │ └── url.py ├── internal ├── config │ ├── __init__.py │ ├── config.py │ └── env.py ├── http │ ├── __init__.py │ ├── api.py │ ├── handler │ │ ├── __init__.py │ │ └── handler.py │ └── views.py ├── storage │ ├── __init__.py │ ├── minio.py │ └── mysql.py └── utils │ ├── __init__.py │ └── migrate.py ├── main.py ├── requirements.txt └── web ├── static ├── css │ └── main.css └── js │ └── app.js └── template ├── base.html ├── bucket.j2 ├── download.j2 ├── help.j2 └── index.j2 /.dockerignore: -------------------------------------------------------------------------------- 1 | assets/ 2 | build/ 3 | api/ 4 | README.md 5 | .gitignore 6 | .dockerignore -------------------------------------------------------------------------------- /.github/workflows/push-docker-image.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*.*.*' 7 | 8 | env: 9 | DOCKER_USER: ${{ secrets.DOCKER_USER }} 10 | DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} 11 | REPO_NAME: ${{ secrets.REPO_NAME }} 12 | 13 | jobs: 14 | push_to_registry: 15 | name: Push Docker image to Docker Hub 16 | runs-on: ubuntu-latest 17 | steps: 18 | - name: Check out the repo 19 | uses: actions/checkout@v3 20 | 21 | - name: Set output 22 | id: vars 23 | run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT 24 | 25 | - name: Log in to Docker Hub 26 | uses: docker/login-action@v2 27 | with: 28 | username: ${{ env.DOCKER_USER }} 29 | password: ${{ env.DOCKER_PASSWORD }} 30 | 31 | - name: Set up Docker Buildx 32 | uses: docker/setup-buildx-action@v2 33 | 34 | - name: Build and push Docker image 35 | uses: docker/build-push-action@v2 36 | with: 37 | context: . 38 | file: ./build/Dockerfile 39 | push: true 40 | tags: ${{ env.DOCKER_USER }}/${{ env.REPO_NAME }}:${{ steps.vars.outputs.tag }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | .idea/ 5 | .DS_Store 6 | 7 | # Object files 8 | *.o 9 | *.ko 10 | *.obj 11 | *.elf 12 | 13 | # Linker output 14 | *.ilk 15 | *.map 16 | *.exp 17 | 18 | # Precompiled Headers 19 | *.gch 20 | *.pch 21 | 22 | # Libraries 23 | *.lib 24 | *.a 25 | *.la 26 | *.lo 27 | 28 | # Shared objects (inc. Windows DLLs) 29 | *.dll 30 | *.so 31 | *.so.* 32 | *.dylib 33 | 34 | # Executables 35 | *.exe 36 | *.out 37 | *.app 38 | *.i*86 39 | *.x86_64 40 | *.hex 41 | 42 | # Debug files 43 | *.dSYM/ 44 | *.su 45 | *.idb 46 | *.pdb 47 | 48 | # Kernel Module Compile Results 49 | *.mod* 50 | *.cmd 51 | .tmp_versions/ 52 | modules.order 53 | Module.symvers 54 | Mkfile.old 55 | dkms.conf 56 | 57 | 58 | .idea/ 59 | .DS_Store 60 | .vendor 61 | __pycache__/ 62 | test.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Minio URL Operator 2 | 3 |  4 |  5 |  6 | 7 | 8 | URL operator for Minio shared objects links. Since Minio object storage provides 9 | urls for sharing objects with a maximum 7 days limit, you can use this operator 10 | in order to have live urls for every object that you want. 11 | With this operator, you can create a persistent url for your objects in Minio. 12 | 13 | ## Start 14 | 15 | You can use ```docker``` image of MUP in order to setup the operator on ```Docker``` or ```Kubernetes```. 16 | 17 | ### image 18 | 19 | ```shell 20 | docker pull amirhossein21/muo:v0.4.0 21 | ``` 22 | 23 | ### environment variables 24 | 25 | | Name | Type | Example | Description | 26 | |:--------------------:|:----------:|----------------------|-------------------------| 27 | | ```HTTP_PORT``` | ```int``` | ```8080``` | HTTP port of MUO API | 28 | | ```HTTP_DEBUG``` | ```bool``` | ```true``` | Debug flag for logging | 29 | | ```HTTP_HOST``` | ```str``` | ```127.0.0.1``` | Container host name | 30 | | ```HTTP_PRIVATE``` | ```bool``` | ```false``` | Private host or not | 31 | | ```MYSQL_HOST``` | ```str``` | ```127.0.0.1``` | MySQL cluster host | 32 | | ```MYSQL_PORT``` | ```int``` | ```3306``` | MySQL cluster port | 33 | | ```MYSQL_USER``` | ```str``` | ```root``` | MySQL user | 34 | | ```MYSQL_PASSWORD``` | ```str``` | ```pa$$word``` | MySQL pass | 35 | | ```MYSQL_DB``` | ```str``` | ```minio-db``` | MySQL database | 36 | | ```MYSQL_MIGRATE``` | ```bool``` | ```false``` | Database migration | 37 | | ```MINIO_HOST``` | ```str``` | ```localhost:9000``` | Minio cluster host | 38 | | ```MINIO_SECURE``` | ```bool``` | ```false``` | Secure Minio connection | 39 | | ```MINIO_ACCESS``` | ```str``` | - | Minio access token | 40 | | ```MINIO_SECRET``` | ```str``` | - | Minio secret token | 41 | 42 | 43 | ### start 44 | 45 | ```shell 46 | docker run -d -it \ 47 | -e HTTP_PORT=80 -e HTTP_DEBUG=0 -e MINIO_HOST=localhost:9000 \ 48 | -e MINIO_SECURE=0 -e MINIO_ACCESS=9iWKawYzq68iNMN7MsiU \ 49 | -e MINIO_SECRET=zWwZlmTX56Hr8NYBOpN4ga2zV8oO2ECIjjPHPF20 \ 50 | -e HTTP_HOST=localhost -e HTTP_PRIVATE=1 \ 51 | amirhossein21/muo:v0.4.0 52 | ``` 53 | 54 | ## API 55 | 56 | In order to use the operator APIs, you can read the ```swagger``` documents in [docs](./api/swagger.yaml). 57 | -------------------------------------------------------------------------------- /api/swagger.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.3 2 | info: 3 | title: MUO API - OpenAPI 3.0 4 | description: |- 5 | This is a document for using Minio URL operator APIs in order to 6 | manage links of the objects. 7 | version: 0.4.0 8 | servers: 9 | - url: http://localhost/api 10 | tags: 11 | - name: objects 12 | description: Manages the objects of Minio cluster 13 | paths: 14 | /objects: 15 | get: 16 | tags: 17 | - objects 18 | summary: Get a list of Minio cluster objects meta-data 19 | description: Get a list of all available Minio cluster objects meta-data 20 | parameters: 21 | - in: query 22 | name: prefix 23 | description: Objects name prefix 24 | schema: 25 | type: string 26 | - in: query 27 | name: bucket 28 | required: true 29 | description: Objects bucket in Minio 30 | schema: 31 | type: string 32 | responses: 33 | '200': 34 | description: Successful operation 35 | content: 36 | application/json: 37 | schema: 38 | type: array 39 | items: 40 | type: object 41 | properties: 42 | name: 43 | type: string 44 | example: "image.jpeg" 45 | address: 46 | type: string 47 | example: "8jr2i5mt01" 48 | status: 49 | type: integer 50 | example: 0 51 | enum: [0,1] 52 | updated_at: 53 | type: string 54 | example: "2023-06-01 09:45:46.4501" 55 | format: Date 56 | expires_at: 57 | type: string 58 | example: "2023-06-01 09:45:46.4501" 59 | format: Date 60 | created_at: 61 | type: string 62 | example: "2023-06-01 09:45:46.4501" 63 | format: Date 64 | '400': 65 | description: Empty bucket in input 66 | /objects/{address}: 67 | get: 68 | tags: 69 | - objects 70 | summary: Get object by address 71 | description: Redirect to Minio object by given address 72 | parameters: 73 | - name: address 74 | in: path 75 | description: Object address 76 | schema: 77 | type: string 78 | required: True 79 | example: 'p0mm8oee34' 80 | responses: 81 | '404': 82 | description: Address not found 83 | '303': 84 | description: Object found successfully 85 | /objects/{bucket}/{key}: 86 | get: 87 | tags: 88 | - objects 89 | summary: Get an Object address 90 | description: Return the object address in operator 91 | parameters: 92 | - name: bucket 93 | in: path 94 | description: Bucket name 95 | schema: 96 | type: string 97 | required: True 98 | example: 'snapp' 99 | - name: key 100 | in: path 101 | description: Object name 102 | schema: 103 | type: string 104 | required: True 105 | example: 'image.jpeg' 106 | responses: 107 | '200': 108 | description: Successful operation 109 | content: 110 | application/json: 111 | schema: 112 | type: object 113 | properties: 114 | address: 115 | type: string 116 | example: 'http://localhost/api/objects/p0mm8oee34' 117 | post: 118 | tags: 119 | - objects 120 | summary: Update object status 121 | description: Set new status for object in order to manage the access 122 | parameters: 123 | - name: bucket 124 | in: path 125 | description: Bucket name 126 | schema: 127 | type: string 128 | required: True 129 | example: 'snapp' 130 | - name: key 131 | in: path 132 | description: Object name 133 | schema: 134 | type: string 135 | required: True 136 | example: 'image.jpeg' 137 | requestBody: 138 | content: 139 | application/json: 140 | schema: 141 | type: object 142 | properties: 143 | status: 144 | type: integer 145 | enum: [0,1] 146 | description: "Object enable or disable" 147 | expires: 148 | type: string 149 | example: "2023-06-01 09:45:46.4501" 150 | format: Date 151 | responses: 152 | '200': 153 | description: Successful operation 154 | -------------------------------------------------------------------------------- /build/Dockerfile: -------------------------------------------------------------------------------- 1 | # from base image 2 | FROM python:3.10-slim-buster 3 | 4 | # src/app directory 5 | WORKDIR /src/app 6 | 7 | LABEL app=python 8 | LABEL function=http 9 | LABEL name=minio-operator 10 | 11 | # copy requirement file 12 | COPY requirements.txt . 13 | 14 | # install them 15 | RUN pip3 install -r requirements.txt 16 | 17 | # copy all files 18 | COPY . . 19 | 20 | # start script 21 | CMD python3 main.py 22 | -------------------------------------------------------------------------------- /database/migrations/down-migration-001-urls_table.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE urls; 2 | -------------------------------------------------------------------------------- /database/migrations/down-migration-002-address_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | DROP COLUMN address; 3 | -------------------------------------------------------------------------------- /database/migrations/down-migration-003-enable_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | DROP COLUMN status; 3 | -------------------------------------------------------------------------------- /database/migrations/down-migration-004-expire_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | DROP COLUMN expires_at; 3 | -------------------------------------------------------------------------------- /database/migrations/down-migration-005-updated_at_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | DROP COLUMN updated_at; 3 | -------------------------------------------------------------------------------- /database/migrations/up-migration-001-urls_table.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE urls ( 2 | id INTEGER PRIMARY KEY AUTO_INCREMENT, 3 | bucket VARCHAR(1024) NOT NULL, 4 | object_key VARCHAR(1024) NOT NULL, 5 | url VARCHAR(2048) NOT NULL, 6 | created_at DATETIME 7 | ); 8 | -------------------------------------------------------------------------------- /database/migrations/up-migration-002-address_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | ADD COLUMN address VARCHAR(1024) NOT NULL; 3 | -------------------------------------------------------------------------------- /database/migrations/up-migration-003-enable_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | ADD COLUMN status INTEGER; 3 | -------------------------------------------------------------------------------- /database/migrations/up-migration-004-expire_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | ADD COLUMN expires_at DATETIME; 3 | -------------------------------------------------------------------------------- /database/migrations/up-migration-005-updated_at_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE urls 2 | ADD COLUMN updated_at DATETIME; 3 | -------------------------------------------------------------------------------- /database/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Module: Model 2 | # Purpose: Database models and queries 3 | -------------------------------------------------------------------------------- /database/model/url.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | class URL(object): 5 | """URL is our url database model""" 6 | 7 | def __init__(self, bucket="", key="", url="", address=""): 8 | self.id = None 9 | self.bucket = bucket 10 | self.key = key 11 | self.url = url 12 | self.createdAt = datetime.now() 13 | self.updatedAt = self.createdAt 14 | self.address = address 15 | self.status = 1 16 | self.expiresAt = None 17 | 18 | def read(self, row: tuple): 19 | """read values from database row 20 | 21 | :param row: database row 22 | :return: None 23 | """ 24 | self.id = row[0] 25 | self.bucket = row[1] 26 | self.key = row[2] 27 | self.url = row[3] 28 | self.createdAt = row[4] 29 | self.address = row[5] 30 | self.status = row[6] 31 | self.expiresAt = row[7] 32 | self.updatedAt = row[8] 33 | 34 | def write(self) -> list: 35 | """write values into a list 36 | 37 | :return: list 38 | """ 39 | return [ 40 | self.bucket, 41 | self.key, 42 | self.url, 43 | self.createdAt, 44 | self.address, 45 | self.status, 46 | self.expiresAt, 47 | self.updatedAt 48 | ] 49 | -------------------------------------------------------------------------------- /internal/config/__init__.py: -------------------------------------------------------------------------------- 1 | # Module: config 2 | # Purpose: reading config variables from env 3 | -------------------------------------------------------------------------------- /internal/config/config.py: -------------------------------------------------------------------------------- 1 | from .env import read_value_from_env 2 | 3 | 4 | class Config(object): 5 | """Config object is used to store config variables of application""" 6 | 7 | def __init__(self): 8 | self.host = None 9 | self.private = False 10 | self.port = None 11 | self.debug = True 12 | 13 | self.minio = {} 14 | self.mysql = {} 15 | 16 | def load(self) -> (str, bool): 17 | """"load configs into class fields 18 | 19 | :returns: (error type, error flag) 20 | """ 21 | try: 22 | self.host = read_value_from_env("HTTP_HOST") 23 | 24 | self.private = read_value_from_env("HTTP_PRIVATE") 25 | self.private = True if self.private == "true" else False 26 | 27 | self.port = int(read_value_from_env("HTTP_PORT")) 28 | 29 | self.debug = read_value_from_env("HTTP_DEBUG") 30 | self.debug = True if self.debug == "true" else False 31 | 32 | self.mysql['host'] = read_value_from_env("MYSQL_HOST") 33 | self.mysql['port'] = int(read_value_from_env("MYSQL_PORT")) 34 | self.mysql['user'] = read_value_from_env("MYSQL_USER") 35 | self.mysql['pass'] = read_value_from_env("MYSQL_PASSWORD") 36 | self.mysql['name'] = read_value_from_env("MYSQL_DB") 37 | 38 | self.mysql['migrate'] = read_value_from_env("MYSQL_MIGRATE") 39 | self.mysql['migrate'] = True if self.mysql['migrate'] == "true" else False 40 | 41 | self.minio['host'] = read_value_from_env("MINIO_HOST") 42 | self.minio['access'] = read_value_from_env("MINIO_ACCESS") 43 | self.minio['secret'] = read_value_from_env("MINIO_SECRET") 44 | 45 | self.minio['secure'] = read_value_from_env("MINIO_SECURE") 46 | self.minio['secure'] = True if self.minio['secure'] == "true" else False 47 | 48 | except Exception as e: 49 | return f"[config.load] failed to read params error={e}", True 50 | 51 | return "OK", False 52 | -------------------------------------------------------------------------------- /internal/config/env.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def read_value_from_env(key: str) -> str: 5 | """Read environment variable. 6 | 7 | :param key: name of the variable 8 | :return: value of the variable 9 | """ 10 | return os.getenv(key) 11 | -------------------------------------------------------------------------------- /internal/http/__init__.py: -------------------------------------------------------------------------------- 1 | # Module: http 2 | # Purpose: Creating a http server 3 | -------------------------------------------------------------------------------- /internal/http/api.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, request, jsonify, render_template 2 | 3 | from ..storage.mysql import MySQL 4 | from ..storage.minio import MinioConnector 5 | from .handler.handler import Handler 6 | 7 | 8 | class API(object): 9 | """API manages the backend rest api""" 10 | 11 | def __init__(self, database: MySQL, minio_connection: MinioConnector, host="", private=False): 12 | # create a blueprint for application apis 13 | self.blueprint = Blueprint('api_blueprint', __name__, url_prefix="/api") 14 | 15 | # create a new handler 16 | api = Handler(database, minio_connection) 17 | 18 | @self.blueprint.route("/objects", methods=['GET']) 19 | def get_objects(): 20 | """get objects metadata 21 | 22 | :return: list of objects of a bucket with prefix 23 | """ 24 | bucket = request.args.get("bucket", "") 25 | if bucket == "": 26 | return "Bucket cannot be empty", 400 27 | 28 | return jsonify(api.get_objects_metadata(bucket, request.args.get("prefix", ""))), 200 29 | 30 | @self.blueprint.route("/objects/
", methods=['GET']) 31 | def redirect_address(address): 32 | """redirect to shared storage if enabled 33 | 34 | :param address: object address in our system 35 | """ 36 | url = api.get_object_url_by_address(address) 37 | if len(url) == 0: 38 | return "Address does not exists", 404 39 | 40 | return render_template("download.j2", address=url), 200 41 | 42 | @self.blueprint.route("/objects/19 | curl -X GET "localhost:8080/api/objects?bucket=snapp&prefix=list" 20 |21 | Response:
23 | [ 24 | { 25 | 'name': 'image.jpg', 26 | 'status': 0, 27 | 'created_at': '2023-06-01 19:00:02' 28 | }, 29 | { 30 | 'name': 'file.pdf', 31 | 'status': -1, 32 | } 33 | ] 34 |35 |
40 | curl -X GET "localhost:8080/api/objects/:bucket/:key/register" 41 |42 | Response:
44 | "OK" 45 |46 |
51 | curl -X GET "localhost:8080/api/objects/:bucket/:key" 52 |53 | Response:
55 | { 56 | 'address': 'http://localhost/api/objects/m889wierop' 57 | } 58 |59 |
66 | curl -X POST "localhost:8080/api/objects/:bucket/:key?status=0" 67 |68 | Response:
70 | "OK" 71 |72 |