├── .godir ├── pkg ├── config │ ├── testdata │ │ ├── parse.tengo │ │ ├── tengo.tengo │ │ ├── no-basic-storage.yml │ │ ├── invalid-parent-bucket.yml │ │ ├── invalid-parent-storage.yml │ │ ├── config-tengo-invalid.yml │ │ ├── config-tengo-compile-error.yml │ │ └── config.yml │ └── config_test.go ├── processor │ ├── benchmark │ │ ├── local │ │ │ ├── file.txt │ │ │ ├── small.jpg │ │ │ └── large.jpeg │ │ └── small.yml │ ├── testdata │ │ └── small.yml │ └── plugins │ │ ├── accept-webp.go │ │ ├── plugins_test.go │ │ ├── plugins.go │ │ ├── accept-webp_test.go │ │ └── compress.go ├── engine │ ├── testdata │ │ ├── small.jpg │ │ └── config.yml │ └── image_engine.go ├── storage │ ├── testdata │ │ ├── config.yml │ │ ├── config_s3.yml │ │ ├── all-storages.yml │ │ └── config2.yml │ └── archive.go ├── object │ ├── testdata │ │ ├── bucket-no-transform.yml │ │ ├── bucket-transform.yml │ │ ├── bucket-transform-hash.yml │ │ ├── bucket-transform-hashParent.yml │ │ ├── bucket-transform-parent-bucket.yml │ │ ├── bucket-transform-query-parent-storage.yml │ │ ├── bucket-transform-parent-storage.yml │ │ └── bucket-transform-preset-query.yml │ ├── preset_query.go │ ├── tengo │ │ ├── testdata │ │ │ ├── parse.tengo │ │ │ └── preset.tengo │ │ ├── tengo.go │ │ ├── regexp_test.go │ │ ├── preset_test.go │ │ ├── preset.go │ │ ├── regexp.go │ │ ├── transform.go │ │ ├── tengo_test.go │ │ ├── url.go │ │ ├── url_test.go │ │ ├── file_object.go │ │ ├── transform_test.go │ │ ├── bucket_config.go │ │ ├── file_object_test.go │ │ └── bucket_config_test.go │ ├── base64_preset.go │ └── cloudinary │ │ └── cloudinary.go ├── middleware │ └── config.yml ├── monitoring │ ├── logger_test.go │ ├── metrics_test.go │ ├── logger.go │ └── metrics.go ├── throttler │ ├── throttler_test.go │ ├── throttler.go │ └── bucket.go ├── cache │ ├── cache_test.go │ ├── cache.go │ ├── memory_test.go │ ├── redis_pool.go │ ├── generic_singleton_test.go │ ├── redis_pool_test.go │ ├── redis.go │ └── memory.go ├── helpers │ └── helpers.go ├── lock │ ├── lock_test.go │ └── lock.go └── response │ └── shared_response.go ├── favicon.png ├── gopher.png ├── scripts ├── run-tests-docker.sh ├── wget.sh ├── s3.js ├── unit-travis.sh ├── prepare-for-tests.sh ├── install-deps.sh └── run-tests.sh ├── example ├── dashboard.png ├── Dockerfile ├── deploy-mort.sh ├── README.md └── config.yml ├── charts └── mort │ ├── Chart.yaml │ ├── templates │ ├── config.yaml │ ├── serviceaccount.yaml │ ├── secret.yaml │ ├── service.yaml │ ├── hpa.yaml │ ├── ingress.yaml │ ├── NOTES.txt │ ├── _helpers.tpl │ └── deployment.yaml │ ├── .helmignore │ ├── values.yaml │ └── files │ └── config.yaml ├── etc └── mort.service ├── .releaserc ├── Dockerfile.integrations ├── .github └── workflows │ ├── update-godoc.yaml │ ├── semantic-release.yaml │ ├── goreleaser.yaml │ ├── deployment.yaml │ ├── docker-build.yaml │ ├── docker-build-base.yaml │ └── ci.yaml ├── doc ├── tengo │ ├── url.md │ ├── fileobject.md │ └── bucketconfig.md ├── UrlParsers.md └── TengoUrlParser.md ├── .gitignore ├── CONTRIBUTING.md ├── .dockerignore ├── Dockerfile.test ├── docker-compose.yaml ├── configuration ├── parse.tengo └── config-glacier-example.yml ├── tests-int ├── LargeFile.Spec.js ├── ConditionalRequest.Spec.js ├── S3Read.Spec.js ├── Compress.Spec.js ├── Range.Spec.js ├── setup-mort.js └── request-helper.js ├── package.json ├── LICENSE.md ├── .goreleaser.yaml ├── Dockerfile.base ├── Makefile └── Dockerfile /.godir: -------------------------------------------------------------------------------- 1 | github.com/aldor007/mort -------------------------------------------------------------------------------- /pkg/config/testdata/parse.tengo: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/config/testdata/tengo.tengo: -------------------------------------------------------------------------------- 1 | aaaa -------------------------------------------------------------------------------- /pkg/processor/benchmark/local/file.txt: -------------------------------------------------------------------------------- 1 | jjj -------------------------------------------------------------------------------- /favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/favicon.png -------------------------------------------------------------------------------- /gopher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/gopher.png -------------------------------------------------------------------------------- /scripts/run-tests-docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | sleep 30 4 | npm run tests -------------------------------------------------------------------------------- /example/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/example/dashboard.png -------------------------------------------------------------------------------- /example/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM aldor007/mort 2 | ADD config.yml /etc/mort/mort.yml 3 | RUN mkdir -p /var/run/mort -------------------------------------------------------------------------------- /pkg/engine/testdata/small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/pkg/engine/testdata/small.jpg -------------------------------------------------------------------------------- /pkg/processor/benchmark/local/small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/pkg/processor/benchmark/local/small.jpg -------------------------------------------------------------------------------- /pkg/processor/benchmark/local/large.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aldor007/mort/HEAD/pkg/processor/benchmark/local/large.jpeg -------------------------------------------------------------------------------- /pkg/storage/testdata/config.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | storages: 4 | basic: 5 | kind: "local-meta" 6 | rootPath: "./testdata" 7 | -------------------------------------------------------------------------------- /charts/mort/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: mort 3 | description: A Helm chart for Kubernetes 4 | type: application 5 | version: 0.1.0 6 | appVersion: latest 7 | -------------------------------------------------------------------------------- /etc/mort.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=mort - A storage and image processing server 3 | 4 | [Service] 5 | ExecStart=/usr/local/bin/mort 6 | Restart=on-abort 7 | 8 | [Install] 9 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-no-transform.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | storages: 4 | basic: 5 | kind: "local" 6 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 7 | -------------------------------------------------------------------------------- /pkg/middleware/config.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | local: 3 | keys: 4 | - accessKey: "acc" 5 | secretAccessKey: "sec" 6 | storages: 7 | basic: 8 | kind: "local-meta" 9 | rootPath: "/tmp/mort-tests/" 10 | -------------------------------------------------------------------------------- /.releaserc: -------------------------------------------------------------------------------- 1 | { 2 | "branch": "master", 3 | "tagFormat": "v${version}", 4 | "plugins": [ 5 | "@semantic-release/commit-analyzer", 6 | "@semantic-release/release-notes-generator", 7 | "@semantic-release/changelog", 8 | "@semantic-release/git" 9 | ] 10 | } -------------------------------------------------------------------------------- /charts/mort/templates/config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ include "mort.fullname" . }}-config 5 | labels: 6 | {{- include "mort.labels" . | nindent 4 }} 7 | data: 8 | config.yaml: |- 9 | {{- (.Files.Get "files/config.yaml") | toYaml | nindent 6 }} -------------------------------------------------------------------------------- /Dockerfile.integrations: -------------------------------------------------------------------------------- 1 | FROM node:17-slim 2 | 3 | WORKDIR /workdir 4 | RUN mkdir -p /workdir 5 | 6 | COPY package.json /workdir/package.json 7 | COPY package-lock.json /workdir/package-lock.json 8 | RUN npm install 9 | 10 | COPY scripts/ /workdir/scripts 11 | COPY tests-int /workdir/tests-int 12 | COPY pkg/ /workdir/pkg 13 | -------------------------------------------------------------------------------- /pkg/monitoring/logger_test.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLog(t *testing.T) { 11 | l := zap.NewNop() 12 | RegisterLogger(l) 13 | assert.Equal(t, l, Log()) 14 | 15 | s := l.Sugar() 16 | assert.Equal(t, s, Logs()) 17 | } 18 | -------------------------------------------------------------------------------- /scripts/wget.sh: -------------------------------------------------------------------------------- 1 | if [ -z "$TRANSFORM" ]; then 2 | echo "No $TRANSFORM path" 3 | exit 1 4 | fi 5 | 6 | if [ -z "$URL" ]; then 7 | echo "No $URL" 8 | exit 1 9 | fi 10 | 11 | rm -rf ${TRANSFORM} 12 | rm -rf wget 13 | mkdir wget/ 14 | for i in `seq 1 100`; do 15 | wget ${URL} -P wget -O wget/img.$i.jpg & 16 | done 17 | 18 | rm -rf ${TRANSFORM} 19 | -------------------------------------------------------------------------------- /.github/workflows/update-godoc.yaml: -------------------------------------------------------------------------------- 1 | 2 | name: update godoc 3 | on: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+' 7 | - '**/v[0-9]+.[0-9]+.[0-9]+' 8 | 9 | jobs: 10 | build: 11 | name: Renew documentation 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Pull new module version 15 | uses: andrewslotin/go-proxy-pull-action@master -------------------------------------------------------------------------------- /doc/tengo/url.md: -------------------------------------------------------------------------------- 1 | # url 2 | Object in which you can check url of request 3 | 4 | # properties 5 | 6 | * `host` - uri host 7 | * `scheme` - uri scheme 8 | * `path` - uri path 9 | * `rawquery` - uri whole query in string 10 | * `uri.query` - uri query string as map 11 | 12 | Example usage 13 | 14 | ```go 15 | text := import("text") 16 | elements := text.split_n(url.path, ".", 2) 17 | `` -------------------------------------------------------------------------------- /charts/mort/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "mort.serviceAccountName" . }} 6 | labels: 7 | {{- include "mort.labels" . | nindent 4 }} 8 | {{- with .Values.serviceAccount.annotations }} 9 | annotations: 10 | {{- toYaml . | nindent 4 }} 11 | {{- end }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /example/deploy-mort.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker pull aldor007/mort 3 | docker build . -t mort 4 | docker ps | grep mort | awk '{print $1 }' | xargs docker stop 5 | docker ps -a | grep mort | awk '{print $1 }' | xargs docker rm 6 | rm -rf /var/run/mort/mort.sock 7 | docker run --name mort -d -p "127.0.0.1:8080:8080" -p "127.0.0.1:8081:8081" -v /var/mort/data/:/data/buckets -v /var/run/mort/:/var/run/mort mort 8 | -------------------------------------------------------------------------------- /charts/mort/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.secrets.enabled }} 2 | apiVersion: v1 3 | kind: Secret 4 | metadata: 5 | name: {{ include "mort.fullname" . }}-env-secrets 6 | labels: 7 | {{- include "mort.labels" . | nindent 4 }} 8 | type: Opaque 9 | data: 10 | {{- range $key, $value := .Values.secrets.env }} 11 | {{ $key }}: {{ $value | b64enc | quote}} 12 | {{- end }} 13 | {{- end }} 14 | -------------------------------------------------------------------------------- /pkg/throttler/throttler_test.go: -------------------------------------------------------------------------------- 1 | package throttler 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestNewNopThrottler(t *testing.T) { 10 | th := NewNopThrottler() 11 | ctx := context.Background() 12 | 13 | assert.True(t, th.Take(ctx)) 14 | assert.True(t, th.Take(ctx)) 15 | 16 | th.Release() 17 | assert.True(t, th.Take(ctx)) 18 | assert.True(t, th.Take(ctx)) 19 | } 20 | -------------------------------------------------------------------------------- /charts/mort/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "mort.fullname" . }} 5 | labels: 6 | {{- include "mort.labels" . | nindent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | name: http 14 | selector: 15 | {{- include "mort.selectorLabels" . | nindent 4 }} 16 | -------------------------------------------------------------------------------- /charts/mort/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | 9 | prof.cpu 10 | prof.mem 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 16 | .glide/ 17 | 18 | .DS_Store 19 | .idea 20 | vendor/ 21 | node_modules/ 22 | pkg/processor/benchmark/local/file-test 23 | coverage.txt 24 | 25 | dist/ 26 | bench.txt 27 | -------------------------------------------------------------------------------- /pkg/object/preset_query.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/config" 5 | //"github.com/aldor007/mort/pkg/object" 6 | "net/url" 7 | ) 8 | 9 | func init() { 10 | RegisterParser("presets-query", decodePreseQuery) 11 | } 12 | 13 | func decodePreseQuery(url *url.URL, bucketConfig config.Bucket, obj *FileObject) (string, error) { 14 | parent, err := decodePreset(url, bucketConfig, obj) 15 | if parent == "" || err != nil { 16 | parent, err = decodeQuery(url, bucketConfig, obj) 17 | } 18 | 19 | return parent, err 20 | } 21 | -------------------------------------------------------------------------------- /scripts/s3.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | 3 | const s3opts = { 4 | region: 'mort', 5 | endpoint: 'localhost:8080', 6 | s3ForcePathStyle: true, 7 | sslEnabled: false, 8 | accessKeyId: 'acc', 9 | secretAccessKey: 'sec', 10 | signatureVersion: 's3', 11 | computeChecksums: true 12 | }; 13 | 14 | 15 | const s3 = new AWS.S3(s3opts); 16 | 17 | const listParams = { 18 | Bucket: 'assets', 19 | Prefix: 'css/', 20 | }; 21 | 22 | s3.listObjects(listParams, function (err, data) { 23 | console.info(err, data) 24 | }); 25 | -------------------------------------------------------------------------------- /pkg/monitoring/metrics_test.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNopReporter(t *testing.T) { 9 | 10 | nop := NopReporter{} 11 | 12 | assert.NotPanics(t, func() { 13 | nop.Counter("a", 3.) 14 | nop.Gauge("b", 4.) 15 | nop.Histogram("c", 1.0) 16 | nop.Inc("d") 17 | t := nop.Timer("e") 18 | t.Done() 19 | }) 20 | } 21 | 22 | func TestRegisterReporter(t *testing.T) { 23 | nop := NopReporter{} 24 | 25 | RegisterReporter(nop) 26 | nop2 := Report() 27 | 28 | assert.Equal(t, nop, nop2) 29 | } 30 | -------------------------------------------------------------------------------- /pkg/storage/testdata/config_s3.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | files: 3 | storages: 4 | basic: 5 | kind: "s3" 6 | pathPrefix: "files" 7 | accessKey: "${S3_ACCESS_KEY}" 8 | secretAccessKey: "${S3_SECRET_ACCESS_KEY}" 9 | region: "eu-central-1" 10 | bucket: "${S3_BUCKET}" 11 | images: 12 | storages: 13 | basic: 14 | kind: "s3" 15 | pathPrefix: "images" 16 | accessKey: "${S3_ACCESS_KEY}" 17 | secretAccessKey: "${S3_SECRET_ACCESS_KEY}" 18 | region: "eu-central-1" 19 | bucket: "${S3_BUCKET}" 20 | -------------------------------------------------------------------------------- /scripts/unit-travis.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # Set CGO flags for macOS with Homebrew (needed for brotli) 6 | if [ -d "/opt/homebrew" ]; then 7 | export CGO_CFLAGS="-I/opt/homebrew/include" 8 | export CGO_LDFLAGS="-L/opt/homebrew/lib" 9 | elif [ -d "/usr/local" ]; then 10 | export CGO_CFLAGS="-I/usr/local/include" 11 | export CGO_LDFLAGS="-L/usr/local/lib" 12 | fi 13 | 14 | echo "" > coverage.txt 15 | 16 | for d in $(go list ./... | grep -v vendor); do 17 | go test -bench=. -race -coverprofile=profile.out -covermode=atomic "$d" 18 | if [ -f profile.out ]; then 19 | cat profile.out >> coverage.txt 20 | rm profile.out 21 | fi 22 | done -------------------------------------------------------------------------------- /pkg/storage/testdata/all-storages.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | azure: 3 | storages: 4 | basic: 5 | kind: "azure" 6 | azureAccount: "account" 7 | azureKey: "key-for-azure" 8 | oracle: 9 | storages: 10 | basic: 11 | kind: "oracle" 12 | oracleUsername: "oracle" 13 | oraclePassword: "password" 14 | b2: 15 | storages: 16 | basic: 17 | kind: "b2" 18 | b2Account: "aaa" 19 | b2ApplicationKeyId: "key" 20 | google: 21 | storages: 22 | basic: 23 | kind: "google" 24 | googleConfigJson: | 25 | {"no-idea": "value"} 26 | googleProjectId: "id" 27 | googleScopes: "a, b" 28 | 29 | -------------------------------------------------------------------------------- /pkg/storage/testdata/config2.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | storages: 4 | basic: 5 | kind: "local-meta" 6 | rootPath: "./no-bucket" 7 | buckets3: 8 | storages: 9 | basic: 10 | kind: "s3" 11 | accessKey: "a" 12 | secretAccessKey: "b" 13 | endpoint: "s3.amazonaws.com" 14 | region: "eu-west-1" 15 | bucket: "mybucket" # optional 16 | buckethttp: 17 | storages: 18 | basic: 19 | kind: "http" 20 | url: "http://remote//" 21 | headers: 22 | "x-mort": 1 23 | bucketlocal: 24 | storages: 25 | basic: 26 | kind: "local" 27 | rootPath: "./no-bucket" 28 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | So you want to contribute? Awesome! Welcome aboard! 2 | 3 | ## Steps 4 | 5 | 1. Fork and clone 6 | 2. Install dependencies 7 | 3. Hack, in no particular order: 8 | - Write enough code 9 | - Write tests for that code 10 | - Check that other tests pass 11 | - Repeat until you're satisfied 12 | 4. Submit a pull request 13 | 14 | ## For and clone 15 | 16 | First step is to fork it at http://help.github.com/fork-a-repo/ and create your own clone of mort. 17 | 18 | ## Running the Tests 19 | 20 | Running the tests is as easy as: 21 | 22 | make tests 23 | 24 | You should see the results of running your tests after an instant. 25 | 26 | 27 | Format code 28 | 29 | make format 30 | 31 | -------------------------------------------------------------------------------- /pkg/config/testdata/no-basic-storage.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/([a-z0-9_]+)\\/thumb_(.*)" 5 | kind: "presets" 6 | order: 7 | presetName: 0 8 | parent: 1 9 | presets: 10 | blog_small: 11 | quality: 75 12 | filters: 13 | thumbnail: { size: [100, 100], mode: outbound } 14 | storages: 15 | basicss: 16 | kind: "local" 17 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 18 | transform: 19 | kind: "local" 20 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 21 | -------------------------------------------------------------------------------- /pkg/cache/cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | "github.com/alicebob/miniredis/v2" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestCreateDefault(t *testing.T) { 13 | cfg := config.CacheCfg{} 14 | instance := Create(cfg) 15 | 16 | assert.Equal(t, reflect.TypeOf(instance).String(), reflect.TypeOf(&MemoryCache{}).String()) 17 | } 18 | 19 | func TestCreatRedis(t *testing.T) { 20 | s := miniredis.RunT(t) 21 | 22 | cfg := config.CacheCfg{} 23 | cfg.Type = "redis" 24 | cfg.Address = []string{s.Addr()} 25 | instance := Create(cfg) 26 | 27 | assert.Equal(t, reflect.TypeOf(instance).String(), reflect.TypeOf(&RedisCache{}).String()) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/monitoring/logger.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import "go.uber.org/zap" 4 | 5 | // logger is single singleton instance of logger 6 | // default logger do nothing 7 | var logger *zap.Logger = zap.NewNop() 8 | 9 | // sugaredLogger extend version on zap.Logger that allow 10 | // using sting format functions 11 | var sugaredLogger *zap.SugaredLogger = logger.Sugar() 12 | 13 | // RegisterLogger new logger as main logger for service 14 | // RegisterLogger is NOT THREAD SAFE 15 | func RegisterLogger(l *zap.Logger) { 16 | logger = l 17 | } 18 | 19 | // Log returns correct registered logger 20 | func Log() *zap.Logger { 21 | return logger 22 | } 23 | 24 | // Logs return sugared zap logger 25 | func Logs() *zap.SugaredLogger { 26 | return sugaredLogger 27 | } 28 | -------------------------------------------------------------------------------- /pkg/engine/testdata/config.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | local: 3 | transform: 4 | path: "\\/(?P[a-zA-Z0-9\\.\\/]+)\\-(?P[a-z]+)" 5 | kind: "presets-query" 6 | parentBucket: "local" 7 | presets: 8 | small: 9 | quality: 75 10 | filters: 11 | thumbnail: 12 | width: 100 13 | height: 70 14 | mode: outbound 15 | interlace: true 16 | storages: 17 | basic: 18 | kind: "local-meta" 19 | rootPath: "../tests/benchmark" 20 | transform: 21 | kind: "noop" 22 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Build artifacts 2 | *.o 3 | *.so 4 | *.a 5 | mort 6 | dist/ 7 | build/ 8 | 9 | # Dependencies 10 | vendor/ 11 | node_modules/ 12 | 13 | # Test files 14 | *_test.go 15 | **/*_test.go 16 | *.test 17 | coverage.out 18 | coverage.html 19 | TEST_COVERAGE_REPORT.md 20 | 21 | # Git 22 | .git/ 23 | .gitignore 24 | .gitattributes 25 | 26 | # CI/CD 27 | .github/ 28 | .gitlab-ci.yml 29 | .travis.yml 30 | .circleci/ 31 | 32 | # Documentation 33 | *.md 34 | docs/ 35 | example/ 36 | 37 | # IDE and editor files 38 | .vscode/ 39 | .idea/ 40 | *.swp 41 | *.swo 42 | *~ 43 | .DS_Store 44 | 45 | # Docker files (to avoid recursion) 46 | Dockerfile* 47 | .dockerignore 48 | docker-compose*.yml 49 | 50 | # Temporary files 51 | tmp/ 52 | temp/ 53 | *.tmp 54 | *.log 55 | .cache/ 56 | 57 | # Source maps and debug files 58 | src/* 59 | -------------------------------------------------------------------------------- /Dockerfile.test: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/aldor007/mort-base 2 | 3 | ENV GOLANG_VERSION 1.25.4 4 | ENV TARGETARCH amd64 5 | ARG TAG 'dev' 6 | ARG COMMIT "master" 7 | ARG DATE "now" 8 | 9 | 10 | ENV WORKDIR /workspace 11 | ENV PATH /usr/local/go/bin:$PATH 12 | RUN rm -rf /usr/local/go && curl -fsSL --insecure "https://go.dev/dl/go$GOLANG_VERSION.linux-$TARGETARCH.tar.gz" -o golang.tar.gz \ 13 | && tar -C /usr/local -xzf golang.tar.gz \ 14 | && rm golang.tar.gz 15 | 16 | WORKDIR $WORKDIR 17 | COPY go.mod ./ 18 | COPY go.sum ./ 19 | 20 | COPY cmd/ $WORKDIR/cmd 21 | COPY .godir ${WORKDIR}/.godir 22 | COPY configuration/ ${WORKDIR}/configuration 23 | COPY etc/ ${WORKDIR}/etc 24 | COPY pkg/ ${WORKDIR}/pkg 25 | COPY scripts/ ${WORKDIR}/scripts 26 | COPY Makefile ${WORKDIR}/Makefile 27 | 28 | RUN go build -o /go/mort ./cmd/mort/mort.go 29 | -------------------------------------------------------------------------------- /scripts/prepare-for-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p /tmp/mort-tests/local/dir/ 4 | mkdir -p /tmp/mort-tests/local/dir/a/b/c 5 | mkdir -p /tmp/mort-tests/local/dir2/a/b/c 6 | mkdir -p /tmp/mort-tests/remote/dir 7 | 8 | echo "test" > /tmp/mort-tests/local/file 9 | echo "test" > /tmp/mort-tests/remote/file 10 | 11 | # Create 1GB file - cross-platform approach 12 | if command -v fallocate >/dev/null 2>&1; then 13 | # Linux 14 | fallocate -l 1G /tmp/mort-tests/local/big.img 15 | elif command -v mkfile >/dev/null 2>&1; then 16 | # macOS 17 | mkfile 1g /tmp/mort-tests/local/big.img 18 | else 19 | # Fallback using dd (slower but universal) 20 | dd if=/dev/zero of=/tmp/mort-tests/local/big.img bs=1m count=1024 2>/dev/null 21 | fi 22 | 23 | cp -r pkg/processor/benchmark/local/* /tmp/mort-tests/local/ -------------------------------------------------------------------------------- /scripts/install-deps.sh: -------------------------------------------------------------------------------- 1 | LIBVIPS_VERSION=8.11.2 2 | 3 | apt-get install ca-certificates \ 4 | automake build-essential curl \ 5 | gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev \ 6 | libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev \ 7 | swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev \ 8 | libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev libimagequant-dev libaom-dev libbrotli-dev && \ 9 | cd /tmp && \ 10 | curl -OL https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.gz && \ 11 | tar zvxf vips-${LIBVIPS_VERSION}.tar.gz && \ 12 | cd /tmp/vips-${LIBVIPS_VERSION} && \ 13 | ./configure --enable-debug=no --without-python $1 && \ 14 | make -j 4 && \ 15 | make install && \ 16 | ldconfig -------------------------------------------------------------------------------- /pkg/processor/testdata/small.yml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | - "unix:/tmp/mort.sock" 5 | monitoring: "prometheus" 6 | placeholder: "./benchmark/localsmall.jpg" 7 | 8 | buckets: 9 | local: 10 | transform: 11 | path: "\\/(?P[a-zA-Z0-9\\.\\/]+)\\-(?P[a-z]+)" 12 | kind: "presets" 13 | parentBucket: "local" 14 | presets: 15 | small: 16 | quality: 75 17 | filters: 18 | thumbnail: 19 | width: 100 20 | height: 70 21 | mode: outbound 22 | interlace: true 23 | storages: 24 | basic: 25 | kind: "local-meta" 26 | rootPath: "./benchmark" 27 | transform: 28 | kind: "noop" 29 | -------------------------------------------------------------------------------- /pkg/throttler/throttler.go: -------------------------------------------------------------------------------- 1 | package throttler 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // defaultBacklogTimeout set to 60s 9 | var defaultBacklogTimeout = time.Second * 60 10 | 11 | // Throttler is rate limiter 12 | type Throttler interface { 13 | Take(ctx context.Context) (taken bool) // Take tries acquire token when its true its mean you can process when false have been throttled 14 | Release() // Release returns token to pool 15 | } 16 | 17 | // NopThrottler is always return that you can perform given operation 18 | type NopThrottler struct { 19 | } 20 | 21 | // NewNopThrottler create instance of NopThrottler 22 | func NewNopThrottler(_ ...interface{}) *NopThrottler { 23 | return &NopThrottler{} 24 | } 25 | 26 | // Take return always true 27 | func (*NopThrottler) Take(_ context.Context) bool { 28 | return true 29 | } 30 | 31 | // Release do nothing 32 | func (*NopThrottler) Release() { 33 | 34 | } 35 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | redis: 5 | image: redis 6 | mort: 7 | build: 8 | context: . 9 | dockerfile: Dockerfile 10 | args: 11 | - TAG=tests 12 | - COMMIT=master 13 | - DATE=now 14 | restart: always 15 | ports: 16 | - 8091:8091 17 | volumes: 18 | - /tmp/mort-tests:/tmp/mort-tests 19 | - ./tests-int/:/etc/mort/ 20 | tests: 21 | build: 22 | context: . 23 | dockerfile: Dockerfile.test 24 | args: 25 | - TAG=dev 26 | - COMMIT=master 27 | - DATE=now 28 | command: ["make", "unit"] 29 | integrations: 30 | build: 31 | context: . 32 | dockerfile: Dockerfile.integrations 33 | command: ["sh", "./scripts/run-tests-docker.sh"] 34 | environment: 35 | - MORT_PORT=8091 36 | - MORT_HOST=mort 37 | volumes: 38 | - /tmp/mort-tests:/tmp/mort-tests:ro 39 | depends_on: 40 | - mort 41 | 42 | 43 | -------------------------------------------------------------------------------- /pkg/processor/plugins/accept-webp.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/aldor007/mort/pkg/object" 8 | "github.com/aldor007/mort/pkg/response" 9 | ) 10 | 11 | func init() { 12 | RegisterPlugin("webp", WebpPlugin{}) 13 | } 14 | 15 | // WebpPlugin plugins that transform image to webp if web browser can handle that format 16 | type WebpPlugin struct { 17 | } 18 | 19 | func (WebpPlugin) configure(_ interface{}) { 20 | 21 | } 22 | 23 | // PreProcess add webp transform to object 24 | func (WebpPlugin) preProcess(obj *object.FileObject, req *http.Request) { 25 | if strings.Contains(req.Header.Get("Accept"), "image/webp") && obj.HasTransform() { 26 | obj.Transforms.Format("webp") 27 | obj.AppendToKey("webp") 28 | } 29 | } 30 | 31 | // PostProcess update vary header 32 | func (WebpPlugin) postProcess(obj *object.FileObject, req *http.Request, res *response.Response) { 33 | if res.IsImage() && obj.HasTransform() { 34 | res.Headers.Add("Vary", "Accept") 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /configuration/parse.tengo: -------------------------------------------------------------------------------- 1 | fmt := import("fmt") 2 | text := import("text") 3 | 4 | parse := func(reqUrl, bucketConfigF, obj) { 5 | // split by "." to remove object extension 6 | elements := text.split_n(reqUrl.path, ".", 2) 7 | ext := elements[1] 8 | if len(elements) == 1 { 9 | return "" 10 | } 11 | // split by "," to find resize parameters 12 | elements = text.split(elements[0], ",") 13 | 14 | // url has no transform 15 | if len(elements) == 1 { 16 | return "" 17 | } 18 | 19 | // apply parameters 20 | width := 0 21 | height := 0 22 | parent := elements[0] +"." + ext 23 | trans := elements[1:] 24 | for tran in trans { 25 | if tran[0] == 'w' { 26 | width = tran[1:] 27 | } 28 | 29 | if tran[0] == 'h' { 30 | height = tran[1:] 31 | } 32 | } 33 | 34 | obj.transforms.resize(int(width), int(height), false, false, false) 35 | return parent 36 | } 37 | 38 | parent := parse(url, bucketConfig, obj) 39 | -------------------------------------------------------------------------------- /charts/mort/templates/hpa.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.autoscaling.enabled }} 2 | apiVersion: autoscaling/v2beta1 3 | kind: HorizontalPodAutoscaler 4 | metadata: 5 | name: {{ include "mort.fullname" . }} 6 | labels: 7 | {{- include "mort.labels" . | nindent 4 }} 8 | spec: 9 | scaleTargetRef: 10 | apiVersion: apps/v1 11 | kind: Deployment 12 | name: {{ include "mort.fullname" . }} 13 | minReplicas: {{ .Values.autoscaling.minReplicas }} 14 | maxReplicas: {{ .Values.autoscaling.maxReplicas }} 15 | metrics: 16 | {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} 17 | - type: Resource 18 | resource: 19 | name: cpu 20 | targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} 21 | {{- end }} 22 | {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} 23 | - type: Resource 24 | resource: 25 | name: memory 26 | targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} 27 | {{- end }} 28 | {{- end }} 29 | -------------------------------------------------------------------------------- /pkg/config/testdata/invalid-parent-bucket.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/thumb_(?P.*)" 5 | kind: "presets" 6 | parentBucket: "niema" 7 | presets: 8 | blog_small: 9 | quality: 75 10 | filters: 11 | thumbnail: { size: [100, 100], mode: outbound } 12 | width: 13 | quality: 75 14 | filters: 15 | thumbnail: { size: [100], mode: outbound } 16 | height: 17 | quality: 75 18 | filters: 19 | thumbnail: { size: [0, 100], mode: outbound } 20 | storages: 21 | basic: 22 | kind: "local" 23 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 24 | transform: 25 | kind: "local" 26 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 27 | -------------------------------------------------------------------------------- /pkg/object/tengo/testdata/parse.tengo: -------------------------------------------------------------------------------- 1 | fmt := import("fmt") 2 | text := import("text") 3 | 4 | parse := func(reqUrl, bucketConfigF, obj) { 5 | // split by "." to remove object extension 6 | elements := text.split_n(reqUrl.path, ".", 2) 7 | ext := elements[1] 8 | if len(elements) == 1 { 9 | return "" 10 | } 11 | // split by "," to find resize parameters 12 | elements = text.split(elements[0], ",") 13 | 14 | // url has no transform 15 | if len(elements) == 1 { 16 | return "" 17 | } 18 | 19 | // apply parameters 20 | width := 0 21 | height := 0 22 | parent := elements[0] +"." + ext 23 | trans := elements[1:] 24 | for tran in trans { 25 | if tran[0] == 'w' { 26 | width = tran[1:] 27 | } 28 | 29 | if tran[0] == 'h' { 30 | height = tran[1:] 31 | } 32 | } 33 | 34 | obj.checkParent = true 35 | obj.transforms.resize(int(width), int(height), false, false, false) 36 | return parent 37 | } 38 | 39 | parent := parse(url, bucketConfig, obj) 40 | -------------------------------------------------------------------------------- /tests-int/LargeFile.Spec.js: -------------------------------------------------------------------------------- 1 | 2 | const request = require('superagent'); 3 | const chai = require('chai'); 4 | const expect = chai.expect; 5 | const fs = require('fs'); 6 | const crypto = require('crypto'); 7 | 8 | 9 | const url = 'http://' + process.env.MORT_HOST + ':' + process.env.MORT_PORT; 10 | 11 | 12 | const hashFile = async (input) => { 13 | let hash = crypto.createHash('sha256'); 14 | hash.setEncoding('hex'); 15 | 16 | return new Promise((resolve, reject) => { 17 | input.on('end', () => { 18 | hash.end(); 19 | let hashHex = hash.read(); 20 | resolve(hashHex); 21 | }); 22 | input.pipe(hash); 23 | }); 24 | }; 25 | 26 | 27 | describe('Large file', async () => { 28 | it('should download 1GB file', async () => { 29 | const reqPath = '/local/big.img' 30 | const expectHash = await hashFile(fs.createReadStream('/tmp/mort-tests/local/big.img')) 31 | const res = request.get(url + reqPath) 32 | const hashReq = await hashFile(res); 33 | expect(hashReq).to.eql(expectHash) 34 | }).timeout(60000) 35 | }) -------------------------------------------------------------------------------- /.github/workflows/semantic-release.yaml: -------------------------------------------------------------------------------- 1 | name: Semantic release 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | concurrency: 9 | group: release 10 | cancel-in-progress: true 11 | 12 | jobs: 13 | release: 14 | name: Release 15 | container: 16 | image: ghcr.io/aldor007/mort-base:latest 17 | credentials: 18 | username: ${{ github.actor }} 19 | password: ${{ secrets.GHR_TOKEN }} 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v4 23 | - name: Set up Go 24 | uses: actions/setup-go@v5 25 | with: 26 | go-version: '1.25' 27 | - name: Install 28 | run: go mod download && git config --global --add safe.directory '*' 29 | 30 | - name: Build 31 | run: go build -v ./... 32 | 33 | - name: Test 34 | run: ./scripts/unit-travis.sh 35 | - uses: actions/setup-node@v4 36 | with: 37 | node-version: "18" 38 | - run: npm install && npx semantic-release 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /pkg/config/testdata/invalid-parent-storage.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/([a-z0-9_]+)\\/thumb_(.*)" 5 | kind: "presets" 6 | order: 7 | presetName: 0 8 | parent: 1 9 | parentStorage: "niema" 10 | presets: 11 | blog_small: 12 | quality: 75 13 | filters: 14 | thumbnail: { size: [100, 100], mode: outbound } 15 | width: 16 | quality: 75 17 | filters: 18 | thumbnail: { size: [100], mode: outbound } 19 | height: 20 | quality: 75 21 | filters: 22 | thumbnail: { size: [0, 100], mode: outbound } 23 | storages: 24 | basic: 25 | kind: "local" 26 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 27 | transform: 28 | kind: "local" 29 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 30 | -------------------------------------------------------------------------------- /tests-int/ConditionalRequest.Spec.js: -------------------------------------------------------------------------------- 1 | const supertest = require('supertest'); 2 | const moment = require('moment'); 3 | 4 | const host = process.env.MORT_HOST + ':' + + process.env.MORT_PORT; 5 | const request = supertest(`http://${host}`); 6 | const filePath = '/local/large.jpeg'; 7 | 8 | describe('HTTP conditional requests', function () { 9 | let etag = null; 10 | let lastMod = null; 11 | 12 | before(function (done) { 13 | request.get(filePath) 14 | .end(function (_, res) { 15 | etag = res.headers['etag']; 16 | lastMod = res.headers['last-modified']; 17 | done(); 18 | }) 19 | }); 20 | 21 | it('should return 304 for if-none-match', function (done) { 22 | request.get(filePath) 23 | .set('if-none-match', etag) 24 | .expect(304) 25 | .end(done); 26 | }); 27 | 28 | it('should return 304 for if-modified-since', function (done) { 29 | request.get(filePath) 30 | .set('If-modified-Since', lastMod) 31 | .expect(304) 32 | .end(done); 33 | }); 34 | }); 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mort-integrations-tests", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "@semantic-release/changelog": "^6.0.3", 6 | "@semantic-release/commit-analyzer": "^11.1.0", 7 | "@semantic-release/git": "^10.0.1", 8 | "@semantic-release/release-notes-generator": "^12.1.0", 9 | "async": "^3.2.5", 10 | "aws-sdk": "^2.1563.0", 11 | "axios": "^1.6.5", 12 | "axios-retry": "^4.0.0", 13 | "chai": "^4.4.1", 14 | "chai-like": "^1.1.1", 15 | "chai-things": "^0.2.0", 16 | "mocha": "^10.2.0", 17 | "moment": "^2.30.1", 18 | "superagent": "^9.0.1", 19 | "superagent-binary-parser": "^1.0.1", 20 | "supertest": "^6.3.4" 21 | }, 22 | "scripts": { 23 | "test": "mocha --file ./tests-int/setup-mort.js tests-int/*Spec.js", 24 | "tests": "mocha --file ./tests-int/setup-mort.js tests-int/*Spec.js" 25 | }, 26 | "repository": { 27 | "type": "git" 28 | }, 29 | "keywords": [ 30 | "s3", 31 | "images", 32 | "microservice", 33 | "processing", 34 | "thumbnails", 35 | "processing" 36 | ], 37 | "author": "", 38 | "license": "ISC" 39 | } 40 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Marcin Kaciuba 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Mort behind nginx 2 | 3 | ## Scheme 4 | 5 | ```apple js 6 | |==============| 7 | | Dark World | 8 | |==============| 9 | |||| 10 | |==============| 11 | | nginx 12 | |==============| 13 | | 14 | | 15 | /-----------\ 16 | | mort | 17 | \-----------/ 18 | 19 | ``` 20 | ## Assumptions 21 | * Object will be stored in /var/mort/data 22 | * You will use nginx with slice module 23 | * nginx will terminate SSL 24 | * prometheus job for mort has name "mort" 25 | 26 | ## Files structure 27 | 28 | * config.yml - mort configuration file (copy of demo config) 29 | * deploy-mort.sh - bash script for building and running mort instance 30 | * Dockerfile - simple docker file with configuration for mort 31 | * mort-nginx.config - full nginx configuration that can be easy extanded for any use case 32 | * monitoring.json - example monitoring dashboard 33 | 34 | ## Grafana dashboard 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /pkg/object/tengo/tengo.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | 9 | "github.com/aldor007/mort/pkg/object" 10 | tengoLib "github.com/d5/tengo/v2" 11 | ) 12 | 13 | func init() { 14 | object.RegisterParser("tengo", decodeUsingTengo) 15 | } 16 | 17 | // decodeUsingTengo parse given url by executing tengo script 18 | func decodeUsingTengo(url *url.URL, bucketConfig config.Bucket, obj *object.FileObject) (string, error) { 19 | t := bucketConfig.Transform.TengoScript.Clone() 20 | err := t.Set("url", &URL{Value: url}) 21 | if err != nil { 22 | return "", err 23 | } 24 | err = t.Set("bucketConfig", &BucketConfig{Value: bucketConfig}) 25 | if err != nil { 26 | return "", err 27 | } 28 | err = t.Set("obj", &FileObject{Value: obj}) 29 | if err != nil { 30 | return "", err 31 | } 32 | 33 | err = t.Run() 34 | if err != nil { 35 | return "", err 36 | } 37 | 38 | parentTengo := t.Get("parent") 39 | errorTengo := t.Get("err") 40 | if errorTengo.Object() != tengoLib.UndefinedValue { 41 | return "", errors.New(errorTengo.String()) 42 | } 43 | parent := parentTengo.String() 44 | 45 | return parent, err 46 | } 47 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 5 | kind: "presets" 6 | presets: 7 | blog_small: 8 | quality: 75 9 | filters: 10 | thumbnail: 11 | width: 100 12 | height: 100 13 | mode: outbound 14 | width: 15 | quality: 75 16 | filters: 17 | thumbnail: 18 | width: 100 19 | mode: outbound 20 | height: 21 | quality: 75 22 | filters: 23 | thumbnail: 24 | height: 100 25 | mode: outbound 26 | storages: 27 | basic: 28 | kind: "local" 29 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 30 | transform: 31 | kind: "local" 32 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 33 | -------------------------------------------------------------------------------- /charts/mort/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "mort.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 5 | apiVersion: networking.k8s.io/v1beta1 6 | {{- else -}} 7 | apiVersion: extensions/v1beta1 8 | {{- end }} 9 | kind: Ingress 10 | metadata: 11 | name: {{ $fullName }} 12 | labels: 13 | {{- include "mort.labels" . | nindent 4 }} 14 | {{- with .Values.ingress.annotations }} 15 | annotations: 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | spec: 19 | {{- if .Values.ingress.tls }} 20 | tls: 21 | {{- range .Values.ingress.tls }} 22 | - hosts: 23 | {{- range .hosts }} 24 | - {{ . | quote }} 25 | {{- end }} 26 | secretName: {{ .secretName }} 27 | {{- end }} 28 | {{- end }} 29 | rules: 30 | {{- range .Values.ingress.hosts }} 31 | - host: {{ .host | quote }} 32 | http: 33 | paths: 34 | {{- range .paths }} 35 | - path: {{ .path }} 36 | backend: 37 | serviceName: {{ $fullName }} 38 | servicePort: {{ $svcPort }} 39 | {{- end }} 40 | {{- end }} 41 | {{- end }} 42 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-hash.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 5 | kind: "presets" 6 | resultKey: "hash" 7 | presets: 8 | blog_small: 9 | quality: 75 10 | filters: 11 | thumbnail: 12 | width: 100 13 | height: 100 14 | mode: outbound 15 | width: 16 | quality: 75 17 | filters: 18 | thumbnail: 19 | width: 100 20 | mode: outbound 21 | height: 22 | quality: 75 23 | filters: 24 | thumbnail: 25 | height: 100 26 | mode: outbound 27 | storages: 28 | basic: 29 | kind: "local" 30 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 31 | transform: 32 | kind: "local" 33 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 34 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-hashParent.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 5 | kind: "presets" 6 | resultKey: "hashParent" 7 | presets: 8 | blog_small: 9 | quality: 75 10 | filters: 11 | thumbnail: 12 | width: 100 13 | height: 100 14 | mode: outbound 15 | width: 16 | quality: 75 17 | filters: 18 | thumbnail: 19 | width: 100 20 | mode: outbound 21 | height: 22 | quality: 75 23 | filters: 24 | thumbnail: 25 | height: 100 26 | mode: outbound 27 | storages: 28 | basic: 29 | kind: "local" 30 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 31 | transform: 32 | kind: "local" 33 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 34 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-parent-bucket.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/thumb_(?P.*)" 5 | kind: "presets" 6 | parentBucket: "bucket" 7 | presets: 8 | blog_small: 9 | quality: 75 10 | filters: 11 | thumbnail: 12 | width: 100 13 | height: 100 14 | mode: outbound 15 | width: 16 | quality: 75 17 | filters: 18 | thumbnail: 19 | width: 100 20 | mode: outbound 21 | height: 22 | quality: 75 23 | filters: 24 | thumbnail: 25 | height: 100 26 | mode: outbound 27 | storages: 28 | basic: 29 | kind: "local" 30 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 31 | transform: 32 | kind: "local" 33 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 34 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yaml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Semantic release"] 6 | types: 7 | - completed 8 | branches: 9 | - 'master' 10 | 11 | permissions: 12 | # deployments permission to deploy GitHub pages website 13 | deployments: write 14 | # contents permission to update benchmark contents in gh-pages branch 15 | contents: write 16 | 17 | jobs: 18 | goreleaser: 19 | container: 20 | image: ghcr.io/aldor007/mort-base:latest 21 | credentials: 22 | username: ${{ github.actor }} 23 | password: ${{ secrets.GHR_TOKEN }} 24 | runs-on: ubuntu-latest 25 | steps: 26 | - 27 | name: Checkout 28 | uses: actions/checkout@v4 29 | with: 30 | fetch-depth: 0 31 | - 32 | name: Set up Go 33 | uses: actions/setup-go@v5 34 | with: 35 | go-version: '1.25' 36 | - name: Fix git config 37 | run: git config --global --add safe.directory '*' 38 | - 39 | name: Run GoReleaser 40 | uses: goreleaser/goreleaser-action@v6 41 | with: 42 | distribution: goreleaser 43 | version: '~> v2' 44 | args: release --clean 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | -------------------------------------------------------------------------------- /pkg/object/base64_preset.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "encoding/base64" 5 | "github.com/aldor007/mort/pkg/config" 6 | "path" 7 | "strings" 8 | 9 | "net/url" 10 | ) 11 | 12 | func init() { 13 | RegisterParser("base64_presets", decodeBase64Preset) 14 | } 15 | 16 | // decodePreset parse given url by matching user defined regexp with request path 17 | func decodeBase64Preset(u *url.URL, bucketConfig config.Bucket, obj *FileObject) (string, error) { 18 | trans := bucketConfig.Transform 19 | 20 | matches := trans.PathRegexp.FindStringSubmatch(obj.Key) 21 | if matches == nil { 22 | return "", nil 23 | } 24 | 25 | subMatchMap := make(map[string]string, 2) 26 | _, err := decodePreset(u, bucketConfig, obj) 27 | if err != nil { 28 | return "", err 29 | } 30 | for i, name := range trans.PathRegexp.SubexpNames() { 31 | if i != 0 && name != "" { 32 | subMatchMap[name] = matches[i] 33 | } 34 | } 35 | 36 | parent := subMatchMap["parent"] 37 | 38 | decoded, err := base64.RawStdEncoding.DecodeString(parent) 39 | if err != nil { 40 | return "", err 41 | } 42 | parent = string(decoded) 43 | 44 | if trans.ParentBucket != "" { 45 | parent = "/" + path.Join(trans.ParentBucket, parent) 46 | } else if !strings.HasPrefix(parent, "/") { 47 | parent = "/" + parent 48 | } 49 | 50 | return parent, err 51 | } 52 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-query-parent-storage.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | kind: "query" 5 | parentStorage: "other" 6 | parentBucket: "bucket" 7 | presets: 8 | blog_small: 9 | quality: 75 10 | filters: 11 | thumbnail: 12 | width: 100 13 | height: 100 14 | mode: outbound 15 | width: 16 | quality: 75 17 | filters: 18 | thumbnail: 19 | width: 100 20 | mode: outbound 21 | height: 22 | quality: 75 23 | filters: 24 | thumbnail: 25 | height: 100 26 | mode: outbound 27 | storages: 28 | basic: 29 | kind: "local" 30 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 31 | transform: 32 | kind: "local" 33 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 34 | other: 35 | kind: "http" 36 | url: "https://domain.pl" 37 | -------------------------------------------------------------------------------- /pkg/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/config" 5 | "github.com/aldor007/mort/pkg/monitoring" 6 | "github.com/aldor007/mort/pkg/object" 7 | "github.com/aldor007/mort/pkg/response" 8 | "go.uber.org/zap" 9 | ) 10 | 11 | // ResponseCache is interface for caching of mort responses 12 | type ResponseCache interface { 13 | Set(obj *object.FileObject, res *response.Response) error 14 | Get(obj *object.FileObject) (*response.Response, error) 15 | Delete(obj *object.FileObject) error 16 | } 17 | 18 | // Create returns instance of Response cache for caching HTTP responses 19 | func Create(cacheCfg config.CacheCfg) ResponseCache { 20 | switch cacheCfg.Type { 21 | case "redis": 22 | monitoring.Log().Info("Creating redis response cache", zap.Strings("addr", cacheCfg.Address)) 23 | return NewRedis(cacheCfg.Address, cacheCfg.ClientConfig, CacheCfg{MaxItemSize: cacheCfg.MaxCacheItemSize, MinUseCount: cacheCfg.MinUseCount}) 24 | case "redis-cluster": 25 | monitoring.Log().Info("Creating redis-cluster response cache", zap.Strings("addr", cacheCfg.Address)) 26 | return NewRedisCluster(cacheCfg.Address, cacheCfg.ClientConfig, CacheCfg{ 27 | MaxItemSize: cacheCfg.MaxCacheItemSize, 28 | MinUseCount: cacheCfg.MinUseCount, 29 | }) 30 | default: 31 | monitoring.Log().Info("Creating memory response cache") 32 | return NewMemoryCache(cacheCfg.CacheSize) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: mort 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod tidy 7 | # you may remove this if you don't need go generate 8 | - go mod vendor 9 | builds: 10 | - env: 11 | - GOFLAGS=-mod=vendor 12 | goos: 13 | - linux 14 | goarch: 15 | - amd64 16 | binary: mort 17 | main: "./cmd/mort/mort.go" 18 | archives: 19 | - format: tar.gz 20 | name_template: >- 21 | {{ .ProjectName }}_ 22 | {{- title .Os }}_ 23 | {{- if eq .Arch "amd64" }}x86_64 24 | {{- else if eq .Arch "386" }}i386 25 | {{- else }}{{ .Arch }}{{ end }} 26 | {{- if .Arm }}v{{ .Arm }}{{ end }} 27 | checksum: 28 | name_template: 'checksums.txt' 29 | snapshot: 30 | name_template: "{{ incpatch .Version }}-next" 31 | changelog: 32 | sort: asc 33 | use: github 34 | filters: 35 | exclude: 36 | - '^docs:' 37 | - '^test:' 38 | - '^chore' 39 | - Merge pull request 40 | - Merge remote-tracking branch 41 | - Merge branch 42 | - go mod tidy 43 | groups: 44 | - title: 'New Features' 45 | regexp: "^.*feat[(\\w)]*:+.*$" 46 | order: 0 47 | - title: 'Bug fixes' 48 | regexp: "^.*fix[(\\w)]*:+.*$" 49 | order: 10 50 | - title: Other work 51 | order: 999 52 | release: 53 | github: 54 | name: mort 55 | owner: aldor007 56 | name_template: "{{ .Tag }}" 57 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-parent-storage.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | path: "\\/(?P[a-z0-9_]+)\\/thumb_(?P.*)" 5 | kind: "presets" 6 | parentStorage: "other" 7 | parentBucket: "bucket" 8 | resultKey: "hash" 9 | presets: 10 | blog_small: 11 | quality: 75 12 | filters: 13 | thumbnail: 14 | width: 100 15 | height: 100 16 | mode: outbound 17 | width: 18 | quality: 75 19 | filters: 20 | thumbnail: 21 | width: 100 22 | mode: outbound 23 | height: 24 | quality: 75 25 | filters: 26 | thumbnail: 27 | height: 100 28 | mode: outbound 29 | storages: 30 | basic: 31 | kind: "local" 32 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 33 | transform: 34 | kind: "local" 35 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 36 | other: 37 | kind: "http" 38 | url: "https://domain.pl" 39 | -------------------------------------------------------------------------------- /pkg/object/tengo/regexp_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/object/tengo" 8 | tengoLib "github.com/d5/tengo/v2" 9 | "github.com/d5/tengo/v2/token" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestRegexpTengo(t *testing.T) { 14 | c := regexp.MustCompile("[a-z]+") 15 | 16 | tengoObject := tengo.Regexp{Value: c} 17 | 18 | assert.Equal(t, tengoObject.String(), "[a-z]+") 19 | assert.True(t, tengoObject.Equals(tengoObject.Copy())) 20 | o, err := tengoObject.BinaryOp(token.Add, tengoObject.Copy()) 21 | assert.Nil(t, o) 22 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 23 | assert.False(t, tengoObject.IsFalsy()) 24 | assert.Equal(t, tengoObject.TypeName(), "Regexp-object") 25 | } 26 | 27 | func TestRegexpTengoCall(t *testing.T) { 28 | c := regexp.MustCompile("(?P[a-z]+)") 29 | 30 | tengoObject := tengo.Regexp{Value: c} 31 | 32 | urlTengo := tengoLib.String{Value: "somethingAAA"} 33 | _, err := tengoObject.Call() 34 | assert.Equal(t, err, tengoLib.ErrWrongNumArguments) 35 | 36 | r, err := tengoObject.Call(&urlTengo) 37 | assert.Nil(t, err) 38 | assert.Equal(t, r.TypeName(), "immutable-map") 39 | 40 | rMap := r.(*tengoLib.ImmutableMap) 41 | 42 | presetName, err := rMap.IndexGet(&tengoLib.String{Value: "presetName"}) 43 | assert.Nil(t, err) 44 | 45 | presetNameStr, _ := tengoLib.ToString(presetName) 46 | assert.Equal(t, presetNameStr, "something") 47 | 48 | } 49 | -------------------------------------------------------------------------------- /pkg/object/tengo/preset_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | "github.com/aldor007/mort/pkg/object/tengo" 9 | tengoLib "github.com/d5/tengo/v2" 10 | "github.com/d5/tengo/v2/token" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestPresetTengo(t *testing.T) { 15 | c := config.Preset{} 16 | 17 | tengoObject := tengo.Preset{Value: c} 18 | 19 | assert.True(t, strings.Contains(tengoObject.String(), "format")) 20 | assert.False(t, tengoObject.Equals(tengoObject.Copy())) 21 | o, err := tengoObject.BinaryOp(token.Add, tengoObject.Copy()) 22 | assert.Nil(t, o) 23 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 24 | assert.True(t, !tengoObject.IsFalsy()) 25 | assert.Equal(t, tengoObject.TypeName(), "Preset-object") 26 | } 27 | 28 | func TestPresetTengoGet(t *testing.T) { 29 | c := config.Preset{ 30 | Quality: 100, 31 | Format: "png", 32 | Filters: config.Filters{}, 33 | } 34 | 35 | tengoObject := tengo.Preset{Value: c} 36 | 37 | // get quality 38 | v, err := tengoObject.IndexGet(&tengoLib.String{Value: "quality"}) 39 | assert.Nil(t, err) 40 | assert.Equal(t, v.TypeName(), "int") 41 | qInt, _ := tengoLib.ToInt(v) 42 | assert.Equal(t, qInt, 100) 43 | 44 | // get format 45 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "format"}) 46 | assert.Nil(t, err) 47 | assert.Equal(t, v.TypeName(), "string") 48 | fString, _ := tengoLib.ToString(v) 49 | assert.Equal(t, fString, "png") 50 | 51 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "filters"}) 52 | assert.Nil(t, err) 53 | assert.Equal(t, v.TypeName(), "Filters-object") 54 | } 55 | -------------------------------------------------------------------------------- /pkg/object/tengo/preset.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | // "encoding/json" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | tengoLib "github.com/d5/tengo/v2" 8 | "github.com/d5/tengo/v2/token" 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | // Preset struct create Preset struct inside of tengo VM 13 | type Preset struct { 14 | tengoLib.ObjectImpl 15 | Value config.Preset 16 | } 17 | 18 | // String return empty string 19 | func (o *Preset) String() string { 20 | buf, _ := yaml.Marshal(&o.Value) 21 | return string(buf) 22 | } 23 | 24 | // BinaryOp not implemented 25 | func (o *Preset) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 26 | return nil, tengoLib.ErrInvalidOperator 27 | } 28 | 29 | // IsFalsy return always true 30 | func (o *Preset) IsFalsy() bool { 31 | return false 32 | } 33 | 34 | // Equals returns false 35 | func (o *Preset) Equals(_ tengoLib.Object) bool { 36 | return false 37 | } 38 | 39 | func (o *Preset) Copy() tengoLib.Object { 40 | return &Preset{ 41 | Value: o.Value, 42 | } 43 | } 44 | 45 | func (o *Preset) TypeName() string { 46 | return "Preset-object" 47 | } 48 | 49 | // IndexGet returns the value for the given key. 50 | func (o *Preset) IndexGet(index tengoLib.Object) (val tengoLib.Object, err error) { 51 | strIdx, ok := tengoLib.ToString(index) 52 | if !ok { 53 | err = tengoLib.ErrInvalidIndexType 54 | return 55 | } 56 | 57 | val = tengoLib.UndefinedValue 58 | switch strIdx { 59 | case "quality": 60 | val = &tengoLib.Int{Value: int64(o.Value.Quality)} 61 | case "format": 62 | val = &tengoLib.String{Value: o.Value.Format} 63 | case "filters": 64 | val = &Filters{Value: o.Value.Filters} 65 | } 66 | 67 | return val, nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/processor/benchmark/small.yml: -------------------------------------------------------------------------------- 1 | server: 2 | requestTimeout: 10 3 | headers: 4 | - statusCodes: [200] 5 | values: 6 | "cache-control": "max-age=84000, public" 7 | - statusCodes: [404, 400] 8 | values: 9 | "cache-control": "max-age=60, public" 10 | - statusCodes: [500, 503] 11 | values: 12 | "cache-control": "max-age=10, public" 13 | 14 | buckets: 15 | local: 16 | transform: 17 | path: "\\/(?P[a-zA-Z0-9\\.\\/]+)\\-(?P[a-z]+)" 18 | kind: "presets-query" 19 | parentBucket: "local" 20 | presets: 21 | small: 22 | quality: 75 23 | filters: 24 | crop: 25 | width: 100 26 | height: 70 27 | mode: outbound 28 | interlace: true 29 | m: 30 | quality: 75 31 | filters: 32 | crop: 33 | width: 100 34 | height: 100 35 | mode: outbound 36 | interlace: true 37 | mm: 38 | quality: 79 39 | filters: 40 | crop: 41 | width: 10 42 | height: 100 43 | mode: outbound 44 | interlace: true 45 | storages: 46 | basic: 47 | kind: "local-meta" 48 | rootPath: "./benchmark" 49 | transform: 50 | kind: "noop" 51 | -------------------------------------------------------------------------------- /pkg/throttler/bucket.go: -------------------------------------------------------------------------------- 1 | package throttler 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | // BucketThrottler is implementation of token-bucket algorithm for rate-limiting 9 | type BucketThrottler struct { 10 | tokens chan struct{} 11 | backlogTokens chan struct{} 12 | backlogTimeout time.Duration 13 | } 14 | 15 | // NewBucketThrottler create a new instance of BucketThrottler which limit 16 | func NewBucketThrottler(limit int) *BucketThrottler { 17 | return NewBucketThrottlerBacklog(limit, 0, defaultBacklogTimeout) 18 | } 19 | 20 | // NewBucketThrottlerBacklog crete a new instance of Throttler which more configuration options 21 | func NewBucketThrottlerBacklog(limit int, backlog int, timeout time.Duration) *BucketThrottler { 22 | max := limit + backlog 23 | t := &BucketThrottler{ 24 | tokens: make(chan struct{}, limit), 25 | backlogTokens: make(chan struct{}, max), 26 | backlogTimeout: timeout, 27 | } 28 | 29 | for i := 0; i < max; i++ { 30 | if i < limit { 31 | t.tokens <- struct{}{} 32 | } 33 | t.backlogTokens <- struct{}{} 34 | 35 | } 36 | return t 37 | } 38 | 39 | // Take retrieve a token from bucket 40 | func (t *BucketThrottler) Take(ctx context.Context) bool { 41 | select { 42 | case <-ctx.Done(): 43 | return false 44 | case btok := <-t.backlogTokens: 45 | timer := time.NewTimer(t.backlogTimeout) 46 | 47 | defer func() { 48 | t.backlogTokens <- btok 49 | }() 50 | 51 | select { 52 | case <-timer.C: 53 | return false 54 | case <-t.tokens: 55 | return true 56 | } 57 | default: 58 | return false 59 | } 60 | } 61 | 62 | // Release return toke to bucket 63 | func (t *BucketThrottler) Release() { 64 | t.tokens <- struct{}{} 65 | } 66 | -------------------------------------------------------------------------------- /Dockerfile.base: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 as builder 2 | 3 | ENV LIBVIPS_VERSION=8.11.2 4 | ENV GOLANG_VERSION=1.25.4 5 | ARG TARGETARCH=amd64 6 | ARG TAG='dev' 7 | ARG COMMIT="master" 8 | ARG DATE="now" 9 | 10 | # Install build dependencies, build libvips, and install Go in optimized layers 11 | RUN apt-get update && \ 12 | DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ 13 | ca-certificates \ 14 | automake build-essential curl gcc git libc6-dev make \ 15 | gobject-introspection gtk-doc-tools libglib2.0-dev libjpeg-turbo8-dev libpng-dev \ 16 | libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev \ 17 | swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev \ 18 | libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev libimagequant-dev libaom-dev libbrotli-dev && \ 19 | cd /tmp && \ 20 | curl -fsSL https://github.com/libvips/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.gz | tar xz && \ 21 | cd vips-${LIBVIPS_VERSION} && \ 22 | ./configure --enable-debug=no --without-python --disable-static && \ 23 | make -j$(nproc) && \ 24 | make install && \ 25 | ldconfig && \ 26 | cd / && \ 27 | rm -rf /tmp/vips-${LIBVIPS_VERSION} && \ 28 | apt-get autoremove -y && \ 29 | apt-get autoclean && \ 30 | apt-get clean && \ 31 | rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* 32 | 33 | # Install Go 34 | ENV GOLANG_DOWNLOAD_URL=https://golang.org/dl/go${GOLANG_VERSION}.linux-${TARGETARCH}.tar.gz 35 | RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" | tar -C /usr/local -xz 36 | 37 | ENV WORKDIR=/workspace 38 | ENV PATH=/usr/local/go/bin:$PATH 39 | ENV CGO_CFLAGS_ALLOW="-Xpreprocessor" 40 | 41 | WORKDIR $WORKDIR 42 | 43 | -------------------------------------------------------------------------------- /pkg/helpers/helpers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "net/http" 7 | "os" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | var client = &http.Client{ 13 | Transport: &http.Transport{ 14 | DialContext: (&net.Dialer{ 15 | Timeout: 30 * time.Second, 16 | KeepAlive: 30 * time.Second, 17 | }).DialContext, 18 | TLSHandshakeTimeout: 10 * time.Second, 19 | ResponseHeaderTimeout: 10 * time.Second, 20 | ExpectContinueTimeout: 1 * time.Second, 21 | }, 22 | } 23 | 24 | // FetchObject download data from given URI 25 | func FetchObject(uri string) ([]byte, error) { 26 | if strings.HasPrefix(uri, "http") { 27 | req, err := http.NewRequest("GET", uri, nil) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | response, err := client.Do(req) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | defer response.Body.Close() 38 | buf, err := ioutil.ReadAll(response.Body) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return buf, nil 44 | } 45 | 46 | f, err := os.Open(uri) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | defer f.Close() 52 | 53 | buf, err := ioutil.ReadAll(f) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | return buf, nil 59 | } 60 | 61 | // IsRangeOrCondition check if request is range or condition 62 | func IsRangeOrCondition(req *http.Request) bool { 63 | if req.Header.Get("Range") != "" || req.Header.Get("If-Range") != "" { 64 | return true 65 | } 66 | 67 | if req.Header.Get("If-Match") != "" || req.Header.Get("If-None-Match") != "" { 68 | return true 69 | } 70 | 71 | if req.Header.Get("If-Unmodified-Since") != "" || req.Header.Get("If-Modified-Since") != "" { 72 | return true 73 | } 74 | 75 | return false 76 | } 77 | -------------------------------------------------------------------------------- /charts/mort/values.yaml: -------------------------------------------------------------------------------- 1 | 2 | replicaCount: 2 3 | image: 4 | repository: aldor007/mort 5 | pullPolicy: IfNotPresent 6 | tag: "latest" 7 | 8 | imagePullSecrets: [] 9 | nameOverride: "" 10 | fullnameOverride: "" 11 | 12 | serviceAccount: 13 | create: true 14 | annotations: {} 15 | name: "" 16 | 17 | podAnnotations: {} 18 | 19 | podSecurityContext: {} 20 | # fsGroup: 2000 21 | 22 | securityContext: {} 23 | # capabilities: 24 | # drop: 25 | # - ALL 26 | # readOnlyRootFilesystem: true 27 | # runAsNonRoot: true 28 | # runAsUser: 1000 29 | 30 | service: 31 | type: ClusterIP 32 | port: 80 33 | 34 | ingress: 35 | enabled: true 36 | annotations: {} 37 | # kubernetes.io/ingress.class: nginx 38 | # kubernetes.io/tls-acme: "true" 39 | hosts: 40 | - host: mort.ingress 41 | paths: [] 42 | tls: [] 43 | # - secretName: chart-example-tls 44 | # hosts: 45 | # - mort.ingress 46 | 47 | resources: {} 48 | # We usually recommend not to specify default resources and to leave this as a conscious 49 | # choice for the user. This also increases chances charts run on environments with little 50 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 51 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 52 | # limits: 53 | # cpu: 100m 54 | # memory: 128Mi 55 | # requests: 56 | # cpu: 100m 57 | # memory: 128Mi 58 | 59 | autoscaling: 60 | enabled: false 61 | minReplicas: 2 62 | maxReplicas: 100 63 | targetCPUUtilizationPercentage: 80 64 | # targetMemoryUtilizationPercentage: 80 65 | 66 | secrets: 67 | enabled: false 68 | env: 69 | MEDIA_ACCESS_KEY: acc 70 | 71 | nodeSelector: {} 72 | 73 | tolerations: [] 74 | 75 | affinity: {} 76 | -------------------------------------------------------------------------------- /charts/mort/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if .Values.ingress.enabled }} 3 | {{- range $host := .Values.ingress.hosts }} 4 | {{- range .paths }} 5 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} 6 | {{- end }} 7 | {{- end }} 8 | {{- else if contains "NodePort" .Values.service.type }} 9 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mort.fullname" . }}) 10 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 11 | echo http://$NODE_IP:$NODE_PORT 12 | {{- else if contains "LoadBalancer" .Values.service.type }} 13 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 14 | You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mort.fullname" . }}' 15 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mort.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") 16 | echo http://$SERVICE_IP:{{ .Values.service.port }} 17 | {{- else if contains "ClusterIP" .Values.service.type }} 18 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mort.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 19 | export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") 20 | echo "Visit http://127.0.0.1:8080 to use your application" 21 | kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT 22 | {{- end }} 23 | -------------------------------------------------------------------------------- /pkg/monitoring/metrics.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | import "time" 4 | 5 | // Reporter is a interface for gather information and send 6 | // external monitoring tool 7 | type Reporter interface { 8 | Counter(label string, val float64) 9 | Inc(label string) 10 | Histogram(label string, val float64) 11 | Gauge(label string, val float64) 12 | Timer(label string) Timer 13 | } 14 | 15 | // Timer is used for time measurement 16 | type Timer struct { 17 | start time.Time 18 | metric string 19 | done func(start time.Time, metric string) 20 | } 21 | 22 | // Done end time measurement 23 | func (t Timer) Done() { 24 | t.done(t.start, t.metric) 25 | } 26 | 27 | // NopReporter is reporter that does nothing 28 | type NopReporter struct { 29 | } 30 | 31 | // Counter does nothing 32 | func (n NopReporter) Counter(_ string, _ float64) { 33 | 34 | } 35 | 36 | // Inc does nothing 37 | func (n NopReporter) Inc(_ string) { 38 | 39 | } 40 | 41 | // Histogram does nothing 42 | func (n NopReporter) Histogram(_ string, _ float64) { 43 | 44 | } 45 | 46 | // Gauge does nothing 47 | func (n NopReporter) Gauge(_ string, _ float64) { 48 | 49 | } 50 | 51 | // Timer returns Timer object that measure time between its creation and calling Done Function 52 | func (n NopReporter) Timer(_ string) Timer { 53 | t := Timer{} 54 | t.done = func(_ time.Time, _ string) { 55 | 56 | } 57 | return t 58 | } 59 | 60 | // reporter instance for use as singleton 61 | var reporter Reporter = &NopReporter{} 62 | 63 | // Report is a function that returns reporter instance 64 | // It allows you to report stats to external monitoring 65 | func Report() Reporter { 66 | return reporter 67 | } 68 | 69 | // RegisterReporter change current used reporter with provider 70 | // default reporter is NopReporter that do nothing 71 | func RegisterReporter(r Reporter) { 72 | reporter = r 73 | } 74 | -------------------------------------------------------------------------------- /pkg/processor/plugins/plugins_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/response" 5 | "github.com/stretchr/testify/assert" 6 | "gopkg.in/yaml.v2" 7 | "net/http" 8 | "testing" 9 | ) 10 | 11 | func TestNewPluginsManager(t *testing.T) { 12 | configStr := ` 13 | compress: 14 | gzip: 15 | level: 5 16 | ` 17 | 18 | var config map[string]interface{} 19 | err := yaml.Unmarshal([]byte(configStr), &config) 20 | if err != nil { 21 | panic(err) 22 | } 23 | 24 | pm := NewPluginsManager(config) 25 | 26 | assert.Equal(t, len(pm.list), 1) 27 | } 28 | 29 | func TestNewPluginsManagerPanic(t *testing.T) { 30 | configStr := ` 31 | compress1: 32 | gzip: 33 | level: 5 34 | ` 35 | 36 | var config map[string]interface{} 37 | err := yaml.Unmarshal([]byte(configStr), &config) 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | assert.Panics(t, func() { 43 | NewPluginsManager(config) 44 | }) 45 | } 46 | 47 | func TestPluginsManager_PreProcess(t *testing.T) { 48 | configStr := ` 49 | compress: 50 | gzip: 51 | level: 5 52 | ` 53 | 54 | var config map[string]interface{} 55 | err := yaml.Unmarshal([]byte(configStr), &config) 56 | if err != nil { 57 | panic(err) 58 | } 59 | 60 | pm := NewPluginsManager(config) 61 | req, _ := http.NewRequest("GET", "http://mort/local/small.jpg-m", nil) 62 | req.Header.Add("Accept-Encoding", "gzip") 63 | body := make([]byte, 1200) 64 | body[33] = 'a' 65 | body[324] = 'c' 66 | res := response.NewBuf(200, body) 67 | res.Headers.Add("Content-Type", "text/html") 68 | 69 | pm.PreProcess(nil, req) 70 | pm.PostProcess(nil, req, res) 71 | 72 | assert.Equal(t, len(res.Headers), 3) 73 | assert.Equal(t, res.Headers.Get("Content-Encoding"), "gzip") 74 | assert.Equal(t, res.Headers.Get("Vary"), "Accept-Encoding") 75 | } 76 | -------------------------------------------------------------------------------- /.github/workflows/deployment.yaml: -------------------------------------------------------------------------------- 1 | name: Deployment 2 | 3 | concurrency: 4 | group: production 5 | cancel-in-progress: true 6 | 7 | on: 8 | workflow_run: 9 | workflows: ["Docker build"] 10 | types: 11 | - completed 12 | branches: 13 | - 'master' 14 | 15 | jobs: 16 | deployment: 17 | runs-on: ubuntu-latest 18 | environment: m39 19 | steps: 20 | - 21 | name: Checkout 22 | uses: actions/checkout@v4 23 | - name: git tag 24 | run: git fetch --tags; git fetch --prune --unshallow || true 25 | 26 | - name: Extract tag name 27 | id: dockerTag 28 | uses: actions/github-script@v6 29 | with: 30 | github-token: ${{ secrets.GITHUB_TOKEN }} 31 | result-encoding: string 32 | script: | 33 | const tagExec = await exec.getExecOutput('git', ['describe', '--tags', '--abbrev=0']); 34 | let tag = tagExec.stdout || tagExec.stderr; 35 | tag = tag.trim().replace('v', ''); 36 | return tag; 37 | - 38 | name: Checkout 39 | uses: actions/checkout@v4 40 | with: 41 | repository: ${{ secrets.REPO_NAME }} 42 | path: infra 43 | token: ${{ secrets.GHR_TOKEN }} 44 | - name: Update Image Version in the related HelmChart values.yaml 45 | uses: fjogeleit/yaml-update-action@v0.10.0 46 | with: 47 | valueFile: 'mort/values.yaml' 48 | propertyPath: 'image.tag' 49 | value: ${{ steps.dockerTag.outputs.result }} 50 | repository: ${{ secrets.REPO_NAME }} 51 | branch: master 52 | # targetBranch: master 53 | createPR: false 54 | message: 'Update mort image to ${{ steps.dockerTag.outputs.result}}' 55 | token: ${{ secrets.GHR_TOKEN }} 56 | workDir: infra 57 | -------------------------------------------------------------------------------- /pkg/object/tengo/regexp.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "regexp" 5 | 6 | tengoLib "github.com/d5/tengo/v2" 7 | "github.com/d5/tengo/v2/token" 8 | ) 9 | 10 | // Regexp struct create regexp in tengo VM 11 | type Regexp struct { 12 | tengoLib.ObjectImpl 13 | Value *regexp.Regexp 14 | } 15 | 16 | // String return regexp in string 17 | func (o *Regexp) String() string { 18 | return o.Value.String() 19 | } 20 | 21 | // BinaryOp not implemented 22 | func (o *Regexp) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 23 | return nil, tengoLib.ErrInvalidOperator 24 | } 25 | 26 | // IsFalsy return true if regexp is nil 27 | func (o *Regexp) IsFalsy() bool { 28 | return o.Value == nil 29 | } 30 | 31 | // Equals return true if regexp string are equal 32 | func (o *Regexp) Equals(x tengoLib.Object) bool { 33 | return o.Value.String() == o.Value.String() 34 | } 35 | 36 | func (o *Regexp) Copy() tengoLib.Object { 37 | return &Regexp{ 38 | Value: o.Value, 39 | } 40 | } 41 | 42 | func (o *Regexp) TypeName() string { 43 | return "Regexp-object" 44 | } 45 | 46 | func (o *Regexp) CanCall() bool { 47 | return true 48 | } 49 | 50 | // Call can executed regexp on given value and return immutable map with matches 51 | func (o *Regexp) Call(args ...tengoLib.Object) (ret tengoLib.Object, err error) { 52 | if len(args) != 1 { 53 | err = tengoLib.ErrWrongNumArguments 54 | return 55 | } 56 | val := args[0].(*tengoLib.String) 57 | matches := o.Value.FindStringSubmatch(val.Value) 58 | if matches == nil { 59 | return &tengoLib.Map{}, nil 60 | } 61 | subMatchMap := make(map[string]tengoLib.Object) 62 | 63 | for i, name := range o.Value.SubexpNames() { 64 | if i != 0 && name != "" { 65 | subMatchMap[name] = &tengoLib.String{Value: matches[i]} 66 | } 67 | } 68 | 69 | ret = &tengoLib.ImmutableMap{Value: subMatchMap} 70 | return 71 | } 72 | -------------------------------------------------------------------------------- /pkg/object/tengo/transform.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/config" 5 | tengoLib "github.com/d5/tengo/v2" 6 | "github.com/d5/tengo/v2/token" 7 | ) 8 | 9 | type Transform struct { 10 | tengoLib.ObjectImpl 11 | Value *config.Transform 12 | } 13 | 14 | func (o *Transform) String() string { 15 | return o.Value.Path 16 | } 17 | 18 | func (o *Transform) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 19 | return nil, tengoLib.ErrInvalidOperator 20 | } 21 | 22 | func (o *Transform) IsFalsy() bool { 23 | return o.Value.Path == "" 24 | } 25 | 26 | func (o *Transform) Equals(x tengoLib.Object) bool { 27 | return o.String() == x.String() 28 | } 29 | 30 | func (o *Transform) Copy() tengoLib.Object { 31 | return &Transform{ 32 | Value: o.Value, 33 | } 34 | } 35 | 36 | func (o *Transform) TypeName() string { 37 | return "Transform-object" 38 | } 39 | 40 | // IndexGet returns the value for the given key. 41 | func (o *Transform) IndexGet(index tengoLib.Object) (val tengoLib.Object, err error) { 42 | strIdx, ok := tengoLib.ToString(index) 43 | if !ok { 44 | err = tengoLib.ErrInvalidIndexType 45 | return 46 | } 47 | 48 | val = tengoLib.UndefinedValue 49 | switch strIdx { 50 | case "path": 51 | val = &tengoLib.String{Value: o.Value.Path} 52 | case "parentStorage": 53 | val = &tengoLib.String{Value: o.Value.ParentStorage} 54 | case "parentBucket": 55 | val = &tengoLib.String{Value: o.Value.ParentBucket} 56 | case "pathRegexp": 57 | val = &Regexp{Value: o.Value.PathRegexp} 58 | case "kind": 59 | val = &tengoLib.String{Value: o.Value.Kind} 60 | case "presets": 61 | internalMap := make(map[string]tengoLib.Object) 62 | for k, v := range o.Value.Presets { 63 | internalMap[k] = &Preset{Value: v} 64 | } 65 | val = &tengoLib.ImmutableMap{Value: internalMap} 66 | 67 | } 68 | 69 | return val, nil 70 | } 71 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # CGO flags for brotli and libvips 2 | export CGO_CFLAGS := $(shell pkg-config --cflags libbrotlienc libbrotlidec 2>/dev/null || echo "-I/opt/homebrew/include") 3 | export CGO_LDFLAGS := $(shell pkg-config --libs libbrotlienc libbrotlidec 2>/dev/null || echo "-L/opt/homebrew/lib -lbrotlienc -lbrotlidec") 4 | export CGO_CFLAGS_ALLOW := -Xpreprocessor 5 | 6 | install: 7 | dep ensure 8 | 9 | unit: format 10 | @(go list ./... | grep -v "vendor/" | xargs -n1 go test -race -cover) 11 | 12 | unit-bench: 13 | ./scripts/unit-travis.sh 14 | 15 | coverage: 16 | go test github.com/aldor007/mort/... -race -coverprofile=coverage.txt -covermode=atomic 17 | 18 | integrations: 19 | npm install 20 | ./scripts/run-tests.sh 21 | 22 | format: 23 | @(go fmt ./...) 24 | @(go vet ./...) 25 | 26 | tests: unit integrations 27 | 28 | docker-push: 29 | docker buildx build --platform linux/amd64,linux/arm64 -t aldor007/mort -f Dockerfile . -t aldor007/mort:latest --push; docker push aldor007/mort:latest 30 | run-server: 31 | mkdir -p /tmp/mort 32 | go run cmd/mort/mort.go -config configuration/config.yml 33 | 34 | run-test-server: 35 | scripts/prepare-for-tests.sh 36 | go run cmd/mort/mort.go -config ./tests-int/mort.yml 37 | 38 | run-test-server-redis: 39 | scripts/prepare-for-tests.sh 40 | go run cmd/mort/mort.go -config ./tests-int/mort-redis.yml 41 | 42 | clean-prof: 43 | find . -name ".test" -depth -exec rm {} \; 44 | find . -name ".cpu" -depth -exec rm {} \; 45 | find . -name ".mem" -depth -exec rm {} \; 46 | 47 | release: 48 | docker build . -f Dockerfile.build --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} 49 | 50 | docker-tests: 51 | scripts/prepare-for-tests.sh 52 | docker-compose up --build 53 | 54 | base-docker-push: 55 | docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/aldor007/mort-base:1.2.0-8.11.2 -f Dockerfile.base . -t ghcr.io/aldor007/mort-base:latest --push -------------------------------------------------------------------------------- /doc/tengo/fileobject.md: -------------------------------------------------------------------------------- 1 | 2 | # file object 3 | This object is injected into script and available via `obj` value 4 | 5 | ## read only properties 6 | 7 | * `uri` - net/url object which contains whole url 8 | ** `uri.host` - uri host 9 | ** `uri.scheme` - uri scheme 10 | ** `uri.path` - uri path 11 | ** `uri.rawquery` - uri whole query in string 12 | ** `uri.query` - uri query string as map 13 | * `bucket` - string name of bucket for object 14 | * `key` - string, storage path for object 15 | * `transforms` - object on which you can add image manipulations. For more see [Transforms](#transform)s 16 | 17 | ## properties that can be changed 18 | 19 | * `allowChangeKey` - bool (default: true) if storage path for object can be changed 20 | * `checkParent` - bool (default: false) if mort should make check if parent exist before generating image 21 | * `debug` - bool (default: false) add debug headers to response 22 | 23 | Example usage 24 | 25 | ```go 26 | fmt := import("fmt") 27 | fmt.println(obj.key) 28 | fmt.println(obj.uri) 29 | 30 | ``` 31 | 32 | # transforms 33 | 34 | Object on which you can execute image manipulation described in [Image-Operations](doc/Image-Operations.md) 35 | 36 | # properties 37 | 38 | * `resize(width int, height, int, enlarge bool, preverseAspectRatio bool, fill bool)` - resize image 39 | * `extract(top, left, width, height int)` - crop image 40 | * `crop(width int, height int, gravity string, enlarge bool, embed bool)` - crop image 41 | * `resizeCropAuto(width int, height int)` - crop image 42 | * `interlace()` 43 | * `quality(quality int)` - image quality 44 | * `stripMetadata()` - remove metadata 45 | * `blur(sigma float, mingAmpl float)` - blur image 46 | * `format(format string)` - change image format 47 | * `watermark(image string, position string, opacity float)` - add watermark to image 48 | * `grayscale()` - image in grayscale 49 | * `rotate(angle int)` - rotate image 50 | 51 | -------------------------------------------------------------------------------- /scripts/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to run all Mort tests with proper CGO configuration 4 | 5 | set -e 6 | 7 | echo "=========================================" 8 | echo "Running Mort Test Suite" 9 | echo "=========================================" 10 | echo "" 11 | 12 | # Set CGO flags for macOS with Homebrew 13 | if [ -d "/opt/homebrew" ]; then 14 | echo "Detected Homebrew installation, setting CGO flags..." 15 | export CGO_CFLAGS="-I/opt/homebrew/include" 16 | export CGO_LDFLAGS="-L/opt/homebrew/lib" 17 | elif [ -d "/usr/local" ]; then 18 | echo "Detected /usr/local installation, setting CGO flags..." 19 | export CGO_CFLAGS="-I/usr/local/include" 20 | export CGO_LDFLAGS="-L/usr/local/lib" 21 | fi 22 | 23 | echo "" 24 | echo "=========================================" 25 | echo "1. Throttler Tests (Concurrent Limiting)" 26 | echo "=========================================" 27 | go test -race -v ./pkg/throttler/... 28 | 29 | echo "" 30 | echo "=========================================" 31 | echo "2. Config Tests" 32 | echo "=========================================" 33 | go test -race -v ./pkg/config/... 34 | 35 | echo "" 36 | echo "=========================================" 37 | echo "3. Lock Tests" 38 | echo "=========================================" 39 | go test -race -v ./pkg/lock/... || echo "Note: Goroutine leak tests may show expected go-redis goroutines" 40 | 41 | echo "" 42 | echo "=========================================" 43 | echo "4. Processor Tests" 44 | echo "=========================================" 45 | go test -race -v ./pkg/processor/... 46 | 47 | echo "" 48 | echo "=========================================" 49 | echo "Test Suite Complete!" 50 | echo "=========================================" 51 | echo "" 52 | echo "Note: Goroutine leak tests for Redis lock are skipped" 53 | echo " (they detect expected go-redis connection pool goroutines)" 54 | -------------------------------------------------------------------------------- /charts/mort/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "mort.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "mort.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "mort.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "mort.labels" -}} 37 | helm.sh/chart: {{ include "mort.chart" . }} 38 | {{ include "mort.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "mort.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "mort.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "mort.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "mort.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /pkg/processor/plugins/plugins.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/aldor007/mort/pkg/monitoring" 8 | "github.com/aldor007/mort/pkg/object" 9 | "github.com/aldor007/mort/pkg/response" 10 | "go.uber.org/zap" 11 | ) 12 | 13 | // pluginsList list of plugins 14 | var pluginsList = make(map[string]Plugin) 15 | 16 | // Plugin interface for Plugins 17 | type Plugin interface { 18 | preProcess(obj *object.FileObject, req *http.Request) // PreProcess is used before start of processing object 19 | postProcess(obj *object.FileObject, req *http.Request, res *response.Response) // PostProcess is used after end of processing object 20 | configure(config interface{}) 21 | } 22 | 23 | // PluginsManager process plugins 24 | type PluginsManager struct { 25 | list []string 26 | } 27 | 28 | // NewPluginsManager create new instance of plugins manager 29 | func NewPluginsManager(plugins map[string]interface{}) PluginsManager { 30 | pm := PluginsManager{} 31 | pm.list = make([]string, 0) 32 | for pName, pConfig := range plugins { 33 | if _, ok := pluginsList[pName]; !ok { 34 | panic(fmt.Errorf("unknown plugin %s", pName)) 35 | } 36 | 37 | monitoring.Log().Info("Plugin manager configuring", zap.String("pluginName", pName)) 38 | pluginsList[pName].configure(pConfig) 39 | pm.list = append(pm.list, pName) 40 | } 41 | return pm 42 | } 43 | 44 | // PreProcess run PreProcess functions of plugins 45 | func (h PluginsManager) PreProcess(obj *object.FileObject, req *http.Request) { 46 | for _, hook := range h.list { 47 | pluginsList[hook].preProcess(obj, req) 48 | } 49 | } 50 | 51 | // PostProcess run PostProcess functions of plugins 52 | func (h PluginsManager) PostProcess(obj *object.FileObject, req *http.Request, res *response.Response) { 53 | for _, hook := range h.list { 54 | pluginsList[hook].postProcess(obj, req, res) 55 | } 56 | } 57 | 58 | // RegisterPlugin register plugin 59 | func RegisterPlugin(name string, fnc Plugin) { 60 | pluginsList[name] = fnc 61 | } 62 | -------------------------------------------------------------------------------- /pkg/object/testdata/bucket-transform-preset-query.yml: -------------------------------------------------------------------------------- 1 | buckets: 2 | bucket: 3 | transform: 4 | kind: "presets-query" 5 | path: "\\/(?P[a-z]+)\\/(?P.*)" 6 | parentStorage: "other" 7 | parentBucket: "bucket" 8 | presets: 9 | blog: 10 | quality: 75 11 | filters: 12 | thumbnail: 13 | width: 100 14 | height: 100 15 | mode: outbound 16 | crop: 17 | width: 10 18 | interlace: yes 19 | strip: yes 20 | format: png 21 | blur: 22 | sigma: 2 23 | minAmpl: 3 24 | watermark: 25 | image: "../processor/benchmark/local/small.jpg" 26 | position: "top-center" 27 | opacity: 0.8 28 | grayscale: yes 29 | rotate: 30 | angle: 90 31 | width: 32 | quality: 75 33 | filters: 34 | thumbnail: 35 | width: 100 36 | mode: outbound 37 | height: 38 | quality: 75 39 | filters: 40 | thumbnail: 41 | height: 100 42 | mode: outbound 43 | storages: 44 | basic: 45 | kind: "local" 46 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 47 | transform: 48 | kind: "local" 49 | rootPath: "/Users/aldor/workspace/mkaciubacom/web" 50 | other: 51 | kind: "http" 52 | url: "https://domain.pl" 53 | -------------------------------------------------------------------------------- /pkg/object/tengo/tengo_test.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | "github.com/aldor007/mort/pkg/object" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParseUsingTengo(t *testing.T) { 13 | mortConfig := config.Config{} 14 | mortConfig.BaseConfigPath = "./testdata" 15 | mortConfig.Load("./testdata/config.yml") 16 | 17 | fObj := object.FileObject{} 18 | objUrl, _ := url.Parse("https://mort.com/bucket/image,w100,h100.png") 19 | fObj.Uri = objUrl 20 | fObj.Key = objUrl.Path 21 | 22 | parent, err := decodeUsingTengo(objUrl, mortConfig.Buckets["tengo"], &fObj) 23 | 24 | assert.Nil(t, err) 25 | assert.Equal(t, "/bucket/image.png", parent) 26 | assert.True(t, fObj.CheckParent) 27 | assert.Equal(t, fObj.Transforms.HashStr(), "8bb55054d70af2be") 28 | 29 | } 30 | 31 | func TestParseUsingTengoPreset(t *testing.T) { 32 | mortConfig := config.Config{} 33 | mortConfig.BaseConfigPath = "./testdata" 34 | mortConfig.Load("./testdata/config.yml") 35 | 36 | fObj := object.FileObject{} 37 | objUrl, _ := url.Parse("https://mort.com/preset-tengo/watermark/image.png") 38 | fObj.Uri = objUrl 39 | fObj.Key = objUrl.Path 40 | 41 | parent, err := decodeUsingTengo(objUrl, mortConfig.Buckets["preset-tengo"], &fObj) 42 | 43 | assert.Nil(t, err) 44 | assert.Equal(t, "/preset-tengo/image.png", parent) 45 | assert.Equal(t, fObj.Transforms.HashStr(), "50293c54e6375ab9") 46 | 47 | } 48 | 49 | func TestParseUsingTengoUnknowPreset(t *testing.T) { 50 | mortConfig := config.Config{} 51 | mortConfig.BaseConfigPath = "./testdata" 52 | mortConfig.Load("./testdata/config.yml") 53 | 54 | fObj := object.FileObject{} 55 | objUrl, _ := url.Parse("https://mort.com/preset-tengo/noname/image.png") 56 | fObj.Uri = objUrl 57 | fObj.Key = objUrl.Path 58 | 59 | parent, err := decodeUsingTengo(objUrl, mortConfig.Buckets["preset-tengo"], &fObj) 60 | 61 | assert.NotNil(t, err) 62 | assert.Equal(t, err.Error(), "error: \"unknown preset noname\"") 63 | assert.Equal(t, parent, "") 64 | 65 | } 66 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yaml: -------------------------------------------------------------------------------- 1 | name: Docker build 2 | 3 | on: 4 | workflow_run: 5 | workflows: ["Semantic release"] 6 | types: 7 | - completed 8 | branches: 9 | - 'master' 10 | 11 | jobs: 12 | multi: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - 16 | name: Checkout 17 | uses: actions/checkout@v4 18 | 19 | - name: git tag 20 | run: git fetch --tags; git fetch --prune --unshallow || true 21 | 22 | - name: Extract tag name 23 | id: dockerTag 24 | uses: actions/github-script@v6 25 | with: 26 | github-token: ${{ secrets.GITHUB_TOKEN }} 27 | result-encoding: string 28 | script: | 29 | const tagExec = await exec.getExecOutput('git', ['describe', '--tags', '--abbrev=0']); 30 | let tag = tagExec.stdout || tagExec.stderr; 31 | tag = tag.trim().replace('v', ''); 32 | return tag; 33 | - name: Get current date 34 | id: date 35 | run: echo "::set-output name=date::$(date +'%Y-%m-%d')" 36 | - 37 | name: Set up QEMU 38 | uses: docker/setup-qemu-action@v3 39 | - 40 | name: Set up Docker Buildx 41 | uses: docker/setup-buildx-action@v3 42 | - 43 | name: Login to DockerHub 44 | uses: docker/login-action@v3 45 | with: 46 | username: ${{ github.actor }} 47 | password: ${{ secrets.GHR_TOKEN }} 48 | registry: ghcr.io 49 | - 50 | name: Build and push 51 | uses: docker/build-push-action@v6 52 | with: 53 | context: . 54 | file: ./Dockerfile 55 | platforms: linux/arm64,linux/amd64 56 | push: true 57 | tags: ghcr.io/aldor007/mort:latest, ghcr.io/aldor007/mort:${{ steps.dockerTag.outputs.result}} 58 | cache-from: type=registry,ref=ghcr.io/aldor007/mort-base:latest 59 | cache-to: type=inline 60 | build-args: | 61 | COMMIT=${{ github.sha }} 62 | DATE=${{ steps.date.outputs.date }} 63 | TAG=${{ steps.dockerTag.outputs.result }} 64 | 65 | -------------------------------------------------------------------------------- /.github/workflows/docker-build-base.yaml: -------------------------------------------------------------------------------- 1 | name: Build and Push mort-base Image 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | paths: 8 | - Dockerfile.base 9 | workflow_dispatch: 10 | inputs: 11 | tag: 12 | description: 'Additional tag for the image' 13 | required: false 14 | default: '' 15 | 16 | jobs: 17 | build-base-image: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - 21 | name: Checkout 22 | uses: actions/checkout@v4 23 | 24 | - 25 | name: Fetch git tags 26 | run: git fetch --tags; git fetch --prune --unshallow || true 27 | 28 | - 29 | name: Docker metadata 30 | id: meta 31 | uses: docker/metadata-action@v5 32 | with: 33 | images: ghcr.io/aldor007/mort-base 34 | tags: | 35 | type=raw,value=latest 36 | type=sha,prefix={{branch}}- 37 | type=ref,event=branch 38 | ${{ github.event.inputs.tag && format('type=raw,value={0}', github.event.inputs.tag) || '' }} 39 | labels: | 40 | org.opencontainers.image.title=mort-base 41 | org.opencontainers.image.description=Base image for Mort with libvips and Go 42 | org.opencontainers.image.vendor=aldor007 43 | 44 | - 45 | name: Set up QEMU 46 | uses: docker/setup-qemu-action@v3 47 | 48 | - 49 | name: Set up Docker Buildx 50 | uses: docker/setup-buildx-action@v3 51 | 52 | - 53 | name: Login to GitHub Container Registry 54 | uses: docker/login-action@v3 55 | with: 56 | registry: ghcr.io 57 | username: ${{ github.actor }} 58 | password: ${{ secrets.GHR_TOKEN }} 59 | 60 | - 61 | name: Build and push 62 | uses: docker/build-push-action@v6 63 | with: 64 | context: . 65 | file: ./Dockerfile.base 66 | platforms: linux/arm64,linux/amd64 67 | push: true 68 | tags: ${{ steps.meta.outputs.tags }} 69 | labels: ${{ steps.meta.outputs.labels }} 70 | cache-from: type=gha 71 | cache-to: type=gha,mode=max 72 | provenance: false 73 | 74 | -------------------------------------------------------------------------------- /doc/UrlParsers.md: -------------------------------------------------------------------------------- 1 | # Writing you own url decoder 2 | 3 | Using mort you can write you own request url parser. This section will guide you how to do so and use it. 4 | 5 | ## Implementation 6 | 7 | Parser should be function with look like 8 | ```go 9 | type ParseFnc func(url *url.URL, bucketConfig config.Bucket, obj *FileObject) (parent string, err error) 10 | 11 | ``` 12 | Function parameters: 13 | * url is a request URI 14 | * bucketConfig - is configuration for current bucket 15 | * obj - is result object. In which you should store operation to perform 16 | 17 | It should return 18 | * parent - string path for original image 19 | * error - if any error occurred 20 | 21 | To register parser you should call 22 | ```go 23 | object.RegisterParser(kind string, fn ParserFNc) 24 | ``` 25 | Parser have to be registered before loading configuration. 26 | It will be called on transform object and original object. 27 | 28 | # Example custom parser 29 | 30 | This parser will parse url like 31 | https://mort/bucket/parent,w100,h100.png 32 | 33 | 34 | ```go 35 | object.RegisterParser("custom", func (reqUrl *url.URL, bucketConfig config.Bucket, obj *object.FileObject) (string, error) { 36 | // split by "." to remove object extension 37 | elements := strings.SplitN(reqUrl.Path, ".", 2) 38 | if len(elements) == 1 { 39 | return "", nil 40 | } 41 | // split by "," to find resize parameters 42 | elements = strings.Split(elements[0], ",") 43 | 44 | // url has no transform 45 | if len(elements) == 1 { 46 | return "", nil 47 | } 48 | 49 | // apply parameters 50 | var width, height int64 51 | parent := elements[0] + path.Ext(reqUrl.Path) 52 | trans := elements[1:] 53 | for _, tran := range trans { 54 | if tran[0] == 'w' { 55 | width, _ = strconv.ParseInt(tran[1:], 10, 32) 56 | } 57 | 58 | if tran[0] == 'h' { 59 | height, _ = strconv.ParseInt(tran[1:], 10, 32) 60 | } 61 | } 62 | 63 | obj.Transforms.Resize(int(width), int(height), false) 64 | return parent, nil 65 | }) 66 | ``` 67 | 68 | In bucket configuration: 69 | ```yaml 70 | buckets: 71 | bucket: 72 | transform: 73 | kind: "custom" 74 | ``` 75 | 76 | -------------------------------------------------------------------------------- /pkg/lock/lock_test.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewNopLock(t *testing.T) { 12 | l := NewNopLock() 13 | 14 | ctx := context.Background() 15 | _, locked := l.Lock(ctx, "a") 16 | assert.True(t, locked) 17 | 18 | _, locked = l.Lock(ctx, "a") 19 | assert.True(t, locked) 20 | 21 | // NopLock accepts nil since it does nothing with the response 22 | l.NotifyAndRelease(ctx, "a", nil) 23 | 24 | _, locked = l.Lock(ctx, "a") 25 | assert.True(t, locked) 26 | 27 | l.Release(ctx, "a") 28 | 29 | _, locked = l.Lock(ctx, "a") 30 | assert.True(t, locked) 31 | } 32 | 33 | func TestCreate_NilConfig(t *testing.T) { 34 | t.Parallel() 35 | 36 | l := Create(nil, 30) 37 | assert.NotNil(t, l) 38 | 39 | // Should return a memory lock 40 | _, ok := l.(*MemoryLock) 41 | assert.True(t, ok, "Should create MemoryLock when config is nil") 42 | } 43 | 44 | func TestCreate_DefaultType(t *testing.T) { 45 | t.Parallel() 46 | 47 | cfg := &config.LockCfg{ 48 | Type: "unknown", 49 | } 50 | l := Create(cfg, 30) 51 | assert.NotNil(t, l) 52 | 53 | // Should return a memory lock for unknown type 54 | _, ok := l.(*MemoryLock) 55 | assert.True(t, ok, "Should create MemoryLock for unknown type") 56 | } 57 | 58 | func TestCreate_RedisType(t *testing.T) { 59 | t.Parallel() 60 | 61 | cfg := &config.LockCfg{ 62 | Type: "redis", 63 | Address: []string{"localhost:6379"}, 64 | } 65 | l := Create(cfg, 45) 66 | assert.NotNil(t, l) 67 | 68 | // Should return a redis lock 69 | rl, ok := l.(*RedisLock) 70 | assert.True(t, ok, "Should create RedisLock for redis type") 71 | assert.Equal(t, 45, rl.LockTimeout, "Should set lock timeout") 72 | } 73 | 74 | func TestCreate_RedisClusterType(t *testing.T) { 75 | t.Parallel() 76 | 77 | cfg := &config.LockCfg{ 78 | Type: "redis-cluster", 79 | Address: []string{"localhost:7000", "localhost:7001"}, 80 | } 81 | l := Create(cfg, 60) 82 | assert.NotNil(t, l) 83 | 84 | // Should return a redis lock 85 | rl, ok := l.(*RedisLock) 86 | assert.True(t, ok, "Should create RedisLock for redis-cluster type") 87 | assert.Equal(t, 60, rl.LockTimeout, "Should set lock timeout") 88 | } 89 | -------------------------------------------------------------------------------- /pkg/object/tengo/url.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "net/url" 5 | 6 | tengoLib "github.com/d5/tengo/v2" 7 | "github.com/d5/tengo/v2/token" 8 | ) 9 | 10 | // URL tengo struct wrapping net/url 11 | type URL struct { 12 | tengoLib.ObjectImpl 13 | Value *url.URL 14 | } 15 | 16 | // Strings returns full url 17 | func (o *URL) String() string { 18 | return o.Value.String() 19 | } 20 | 21 | // BinaryOp not implemented 22 | func (o *URL) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 23 | return nil, tengoLib.ErrInvalidOperator 24 | } 25 | 26 | // IsFalsy returns true if url is emptu 27 | func (o *URL) IsFalsy() bool { 28 | return o.String() == "" 29 | } 30 | 31 | // Equals returns true if url are the same 32 | func (o *URL) Equals(x tengoLib.Object) bool { 33 | other := x.(*URL) 34 | return o.Value.String() == other.Value.String() 35 | } 36 | 37 | // Copy create copy of url 38 | func (o *URL) Copy() tengoLib.Object { 39 | newUrl, _ := url.Parse(o.Value.String()) 40 | 41 | return &URL{ 42 | Value: newUrl, 43 | } 44 | } 45 | 46 | // IndexGet returns the value for the given key. 47 | // Avaiable operations 48 | 49 | func (o *URL) IndexGet(index tengoLib.Object) (val tengoLib.Object, err error) { 50 | strIdx, ok := tengoLib.ToString(index) 51 | if !ok { 52 | err = tengoLib.ErrInvalidIndexType 53 | return 54 | } 55 | 56 | val = tengoLib.UndefinedValue 57 | switch strIdx { 58 | case "scheme": 59 | val = &tengoLib.String{ 60 | Value: o.Value.Scheme, 61 | } 62 | case "host": 63 | val = &tengoLib.String{ 64 | Value: o.Value.Host, 65 | } 66 | case "path": 67 | val = &tengoLib.String{ 68 | Value: o.Value.Path, 69 | } 70 | case "rawquery": 71 | val = &tengoLib.String{ 72 | Value: o.Value.RawQuery, 73 | } 74 | case "query": 75 | query := o.Value.Query() 76 | internalMap := make(map[string]tengoLib.Object) 77 | for k, v := range query { 78 | array := make([]tengoLib.Object, 0) 79 | for _, q := range v { 80 | array = append(array, &tengoLib.String{Value: q}) 81 | } 82 | internalMap[k] = &tengoLib.Array{ 83 | Value: array, 84 | } 85 | } 86 | val = &tengoLib.Map{Value: internalMap} 87 | 88 | } 89 | 90 | return val, nil 91 | } 92 | 93 | func (o *URL) TypeName() string { 94 | return "url-object" 95 | } 96 | -------------------------------------------------------------------------------- /pkg/cache/memory_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/object" 5 | "github.com/aldor007/mort/pkg/response" 6 | "github.com/stretchr/testify/assert" 7 | "testing" 8 | ) 9 | 10 | func TestMemoryCache_Set(t *testing.T) { 11 | i := NewMemoryCache(1) 12 | 13 | obj := object.FileObject{} 14 | obj.Key = "cacheKey" 15 | res := response.NewString(200, "test") 16 | 17 | i.Set(&obj, res) 18 | resCache, err := i.Get(&obj) 19 | assert.Nil(t, err) 20 | b, err := resCache.Body() 21 | assert.Nil(t, err) 22 | 23 | assert.Equal(t, resCache.StatusCode, res.StatusCode) 24 | assert.Equal(t, string(b), "test") 25 | } 26 | 27 | func TestMemoryCache_Delete(t *testing.T) { 28 | t.Parallel() 29 | 30 | i := NewMemoryCache(2) 31 | 32 | obj := object.FileObject{} 33 | obj.Key = "cacheKey" 34 | res := response.NewString(200, "test") 35 | 36 | i.Set(&obj, res) 37 | i.Delete(&obj) 38 | _, err := i.Get(&obj) 39 | assert.NotNil(t, err) 40 | } 41 | 42 | func TestMemoryCache_SetTooLarge(t *testing.T) { 43 | t.Parallel() 44 | 45 | i := NewMemoryCache(10) // 10 bytes max 46 | 47 | obj := object.FileObject{} 48 | obj.Key = "large" 49 | res := response.NewString(200, "this is a very long response that exceeds the cache size limit by a lot") 50 | 51 | err := i.Set(&obj, res) 52 | assert.Nil(t, err) // Set doesn't return error, just doesn't cache 53 | 54 | // The item IS cached even if large (memory cache doesn't enforce size limit strictly) 55 | // This test just verifies Set doesn't panic 56 | } 57 | 58 | func TestMemoryCache_GetNotFound(t *testing.T) { 59 | t.Parallel() 60 | 61 | i := NewMemoryCache(100) 62 | 63 | obj := object.FileObject{} 64 | obj.Key = "notfound" 65 | 66 | _, err := i.Get(&obj) 67 | assert.NotNil(t, err) 68 | assert.Contains(t, err.Error(), "not found") 69 | } 70 | 71 | func TestMemoryCache_Concurrent(t *testing.T) { 72 | t.Parallel() 73 | 74 | i := NewMemoryCache(1000) 75 | 76 | // Test concurrent Set/Get operations 77 | done := make(chan bool, 10) 78 | for idx := 0; idx < 10; idx++ { 79 | go func(id int) { 80 | obj := object.FileObject{} 81 | obj.Key = string(rune('a' + id)) 82 | res := response.NewString(200, "test") 83 | 84 | i.Set(&obj, res) 85 | _, err := i.Get(&obj) 86 | assert.Nil(t, err) 87 | done <- true 88 | }(idx) 89 | } 90 | 91 | // Wait for all goroutines 92 | for idx := 0; idx < 10; idx++ { 93 | <-done 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /pkg/cache/redis_pool.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "context" 5 | "crypto/sha256" 6 | "fmt" 7 | "sync" 8 | 9 | goRedis "github.com/go-redis/redis/v8" 10 | ) 11 | 12 | // redisClientPool manages shared Redis client connections 13 | type redisClientPool struct { 14 | clients map[string]goRedis.UniversalClient 15 | mu sync.RWMutex 16 | } 17 | 18 | var ( 19 | pool = &redisClientPool{clients: make(map[string]goRedis.UniversalClient)} 20 | poolOnce sync.Once 21 | ) 22 | 23 | // getRedisClient returns a shared Redis client for the given configuration 24 | // Multiple Cache[T] instances with the same config will share the same connection pool 25 | func getRedisClient(addresses []string, clientConfig map[string]string, cluster bool) goRedis.UniversalClient { 26 | // Create a hash of the configuration to use as cache key 27 | configHash := hashConfig(addresses, cluster) 28 | 29 | pool.mu.RLock() 30 | client, exists := pool.clients[configHash] 31 | pool.mu.RUnlock() 32 | 33 | if exists { 34 | return client 35 | } 36 | 37 | pool.mu.Lock() 38 | defer pool.mu.Unlock() 39 | 40 | // Double-check after acquiring write lock 41 | if client, exists := pool.clients[configHash]; exists { 42 | return client 43 | } 44 | 45 | // Create new client 46 | var newClient goRedis.UniversalClient 47 | 48 | if cluster { 49 | newClient = goRedis.NewClusterClient(&goRedis.ClusterOptions{ 50 | Addrs: addresses, 51 | }) 52 | } else if len(addresses) > 1 { 53 | // Use ring for multiple addresses 54 | addrs := make(map[string]string) 55 | for i, addr := range addresses { 56 | addrs[string(rune('a'+i))] = addr 57 | } 58 | newClient = goRedis.NewRing(&goRedis.RingOptions{ 59 | Addrs: addrs, 60 | }) 61 | } else { 62 | // Single client 63 | newClient = goRedis.NewClient(&goRedis.Options{ 64 | Addr: addresses[0], 65 | }) 66 | } 67 | 68 | // Apply client configuration 69 | if clientConfig != nil { 70 | for key, value := range clientConfig { 71 | newClient.ConfigSet(context.Background(), key, value) 72 | } 73 | } 74 | 75 | pool.clients[configHash] = newClient 76 | return newClient 77 | } 78 | 79 | // hashConfig creates a unique hash for a Redis configuration 80 | func hashConfig(addresses []string, cluster bool) string { 81 | h := sha256.New() 82 | for _, addr := range addresses { 83 | h.Write([]byte(addr)) 84 | } 85 | if cluster { 86 | h.Write([]byte("cluster")) 87 | } 88 | return fmt.Sprintf("%x", h.Sum(nil)) 89 | } 90 | -------------------------------------------------------------------------------- /tests-int/S3Read.Spec.js: -------------------------------------------------------------------------------- 1 | const chai = require('chai'); 2 | const expect = chai.expect; 3 | const AWS = require('aws-sdk'); 4 | const supertest = require('supertest'); 5 | 6 | const host = process.env.MORT_HOST + ':' + + process.env.MORT_PORT; 7 | const request = supertest(`http://${host}`); 8 | 9 | describe('S3 Read features', function () { 10 | beforeEach(function () { 11 | this.s3opts = { 12 | region: 'mort', 13 | endpoint: host, 14 | s3ForcePathStyle: true, 15 | sslEnabled: false, 16 | accessKeyId: 'acc', 17 | secretAccessKey: 'sec', 18 | signatureVersion: 's3', 19 | computeChecksums: true 20 | }; 21 | 22 | this.s3opts.signatureVersion = 's3'; 23 | this.s3opts.accessKeyId = 'acc'; 24 | this.s3 = new AWS.S3(this.s3opts); 25 | }); 26 | 27 | describe('head bucket', function () { 28 | 29 | it('bucket local should exists', function (done) { 30 | this.s3.headBucket({ 31 | Bucket: 'local' 32 | }, function (err, data) { 33 | if (err) { 34 | return done(err) 35 | } 36 | 37 | expect(data).not.to.be.null; 38 | done(); 39 | }) 40 | }); 41 | 42 | it('bucket local2 shouldn\'t exists', function (done) { 43 | this.s3.headBucket({ 44 | Bucket: 'local2' 45 | }, function (err, data) { 46 | expect(err).not.to.be.null; 47 | done(); 48 | }) 49 | }); 50 | 51 | }); 52 | 53 | describe('head and create dir', function () { 54 | it('should return error when dir doesn\'t exist', function (done) { 55 | const params = { 56 | Bucket: 'local', 57 | Key: 'dir-2' 58 | }; 59 | 60 | this.s3.headObject(params, function (err) { 61 | expect(err).not.to.be.null; 62 | done(); 63 | }) 64 | }); 65 | 66 | it('should create dir', function (done) { 67 | const params = { 68 | Bucket: 'local', 69 | Key: 'dir-11/', 70 | Body: '' 71 | }; 72 | 73 | this.s3.upload(params, function (err) { 74 | expect(err).to.be.null; 75 | done(); 76 | }) 77 | }); 78 | 79 | }) 80 | }); 81 | -------------------------------------------------------------------------------- /pkg/storage/archive.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/aldor007/mort/pkg/cache" 10 | "github.com/aldor007/mort/pkg/config" 11 | ) 12 | 13 | // RestoreStatus represents the state of a GLACIER restore operation 14 | // All fields are exported for msgpack serialization 15 | type RestoreStatus struct { 16 | Key string `msgpack:"key"` 17 | RequestedAt time.Time `msgpack:"requestedAt"` 18 | ExpiresAt time.Time `msgpack:"expiresAt"` 19 | InProgress bool `msgpack:"inProgress"` 20 | } 21 | 22 | // RestoreCache handles GLACIER restore status tracking using generic cache 23 | type RestoreCache struct { 24 | cache cache.Cache[RestoreStatus] 25 | } 26 | 27 | // NewRestoreCache creates a RestoreCache using the generic cache 28 | func NewRestoreCache(cacheCfg config.CacheCfg) *RestoreCache { 29 | return &RestoreCache{ 30 | cache: cache.CreateCache[RestoreStatus](cacheCfg), 31 | } 32 | } 33 | 34 | func (r *RestoreCache) getKey(key string) string { 35 | return fmt.Sprintf("mort:glacier:restore:%s", key) 36 | } 37 | 38 | // MarkRestoreRequested records that a restore was initiated 39 | func (r *RestoreCache) MarkRestoreRequested(ctx context.Context, key string, expiresIn time.Duration) error { 40 | cacheKey := r.getKey(key) 41 | 42 | status := RestoreStatus{ 43 | Key: key, 44 | RequestedAt: time.Now(), 45 | ExpiresAt: time.Now().Add(expiresIn), 46 | InProgress: true, 47 | } 48 | 49 | // Set with TTL slightly longer than restore time to track completion 50 | return r.cache.Set(ctx, cacheKey, status, expiresIn+24*time.Hour) 51 | } 52 | 53 | // GetRestoreStatus checks if restore was already requested 54 | func (r *RestoreCache) GetRestoreStatus(ctx context.Context, key string) (*RestoreStatus, error) { 55 | cacheKey := r.getKey(key) 56 | 57 | status, found, err := r.cache.Get(ctx, cacheKey) 58 | if err != nil { 59 | return nil, err 60 | } 61 | if !found { 62 | return nil, nil 63 | } 64 | 65 | // Update InProgress based on current time 66 | status.InProgress = time.Now().Before(status.ExpiresAt) 67 | 68 | return &status, nil 69 | } 70 | 71 | // Global singleton restore cache 72 | var restoreCache *RestoreCache 73 | var restoreCacheOnce sync.Once 74 | 75 | // GetRestoreCache returns the singleton restore cache instance 76 | func GetRestoreCache(cacheCfg config.CacheCfg) *RestoreCache { 77 | restoreCacheOnce.Do(func() { 78 | restoreCache = NewRestoreCache(cacheCfg) 79 | }) 80 | return restoreCache 81 | } 82 | -------------------------------------------------------------------------------- /pkg/object/tengo/url_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aldor007/mort/pkg/object/tengo" 7 | tengoLib "github.com/d5/tengo/v2" 8 | "github.com/d5/tengo/v2/token" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestUrlTengo(t *testing.T) { 13 | objectPath := "/bucket/image.png" 14 | c := pathToURL(objectPath) 15 | 16 | tengoObject := tengo.URL{Value: c} 17 | 18 | assert.Equal(t, tengoObject.String(), objectPath) 19 | assert.True(t, tengoObject.Equals(tengoObject.Copy())) 20 | o, err := tengoObject.BinaryOp(token.Add, tengoObject.Copy()) 21 | assert.Nil(t, o) 22 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 23 | assert.False(t, tengoObject.IsFalsy()) 24 | assert.Equal(t, tengoObject.TypeName(), "url-object") 25 | } 26 | 27 | func TestUrlGetTengo(t *testing.T) { 28 | objectPath := "https://mort.com/bucket/image.png?width=100" 29 | c := pathToURL(objectPath) 30 | 31 | tengoObject := tengo.URL{Value: c} 32 | // get unknown index 33 | v, err := tengoObject.IndexGet(&tengoLib.String{Value: "no-name"}) 34 | assert.Nil(t, err) 35 | assert.Equal(t, v, tengoLib.UndefinedValue) 36 | 37 | // invalid index type 38 | v, err = tengoObject.IndexGet(tengoLib.UndefinedValue) 39 | assert.Equal(t, err, tengoLib.ErrInvalidIndexType) 40 | 41 | // get scheme 42 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "scheme"}) 43 | assert.Nil(t, err) 44 | assert.Equal(t, v.TypeName(), "string") 45 | schemaStr, _ := tengoLib.ToString(v) 46 | assert.Equal(t, schemaStr, "https") 47 | 48 | // get host 49 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "host"}) 50 | assert.Nil(t, err) 51 | assert.Equal(t, v.TypeName(), "string") 52 | hostStr, _ := tengoLib.ToString(v) 53 | assert.Equal(t, hostStr, "mort.com") 54 | 55 | // get path 56 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "path"}) 57 | assert.Nil(t, err) 58 | assert.Equal(t, v.TypeName(), "string") 59 | pathStr, _ := tengoLib.ToString(v) 60 | assert.Equal(t, pathStr, "/bucket/image.png") 61 | 62 | // get rawquery 63 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "rawquery"}) 64 | assert.Nil(t, err) 65 | assert.Equal(t, v.TypeName(), "string") 66 | qsStr, _ := tengoLib.ToString(v) 67 | assert.Equal(t, qsStr, "width=100") 68 | 69 | // get query 70 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "query"}) 71 | assert.Nil(t, err) 72 | assert.Equal(t, v.TypeName(), "map") 73 | tengoMap := v.(*tengoLib.Map) 74 | assert.Equal(t, len(tengoMap.Value), 1) 75 | } 76 | -------------------------------------------------------------------------------- /doc/tengo/bucketconfig.md: -------------------------------------------------------------------------------- 1 | # bucket config object 2 | This object is injected into script and available via `bucketConfig` value 3 | 4 | ## properties 5 | 6 | * `transform` - returns [Transform object](#transform) description for image manipulation 7 | * `keys` - array of map with S3 style `accessKey` and `secretAccessKey` 8 | * `headers` - map[string]string with header configured for bucket 9 | * `name` - string name of bucket 10 | 11 | Example usage 12 | 13 | ```go 14 | [...] 15 | parse := func(reqUrl, bucketConfig, obj) { 16 | trans := bucketConfig.transform 17 | if ok := trans.presets[presetName]; ok == undefined { 18 | return ["", error("unknown preset " + presetName)] 19 | } 20 | 21 | ``` 22 | 23 | # transform 24 | 25 | Object describing what more should do with image. It is based on mort config 26 | 27 | 28 | ## properties 29 | 30 | * `path` - string, mort regexp describing parent and transforms 31 | * `parentStorage` - string, override parent storage 32 | * `parentBucket` - string, overrride parent storage 33 | * `pathRegexp` - compiled regexp from `path` string. can be used as a function 34 | * `kind` - string, type of transformation 35 | * `presets` - map, map of object which contains description of transformations. More about [Preset](#preset) 36 | 37 | Example usage 38 | 39 | ```go 40 | presetName := "mypreset" 41 | if ok := bucketConfig.transform.presets[presetName]; ok == undefined { 42 | return ["", error("unknown preset " + presetName)] 43 | } 44 | ``` 45 | 46 | # preset 47 | 48 | It is tengo representation of mort preset configuration. Each property is lowercase and in camelCase 49 | 50 | ## properties 51 | 52 | * `quality` - int, quality of image 53 | * `format` - string, format of image 54 | * `filters` - filters used for image transformation. More about it [here](/doc/Image-Operations.md) please take a look on "preset" type 55 | 56 | Example usage 57 | 58 | ```go 59 | presetToTransform := func(obj, preset) { 60 | filters := preset.filters 61 | 62 | if filters.thumbnial { 63 | err := obj.transforms.resize(filters.thumbnial.width, filters.thumbnial.height, filters.thumbnial.mode == "outbound", filters.thumbnial.preserveAspectRatio, filters.thumbnial.fill) 64 | if err != undefined { 65 | return err 66 | } 67 | } 68 | 69 | if filters.crop { 70 | err := obj.transforms.crop(filters.crop.width, filters.crop.height, filters.crop.gravity, filters.crop.mode == "outbound", filters.crop.embed) 71 | if err != undefined { 72 | return err 73 | } 74 | } 75 | 76 | ``` 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /pkg/processor/plugins/accept-webp_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/config" 5 | "github.com/aldor007/mort/pkg/object" 6 | "github.com/aldor007/mort/pkg/response" 7 | "github.com/stretchr/testify/assert" 8 | "net/http" 9 | "net/url" 10 | "testing" 11 | ) 12 | 13 | func pathToURL(urlPath string) *url.URL { 14 | u, _ := url.Parse(urlPath) 15 | return u 16 | } 17 | 18 | func TestWebpInAccept(t *testing.T) { 19 | req, _ := http.NewRequest("GET", "http://mort/local/small.jpg-m", nil) 20 | req.Header.Add("Accept", "image/webp") 21 | 22 | mortConfig := config.Config{} 23 | err := mortConfig.Load("../benchmark/small.yml") 24 | assert.Nil(t, err) 25 | 26 | obj, err := object.NewFileObject(req.URL, &mortConfig) 27 | assert.Nil(t, err) 28 | 29 | res := response.NewNoContent(200) 30 | res.Headers.Set("content-type", "image/jpg") 31 | 32 | obj.Ctx = req.Context() 33 | w := WebpPlugin{} 34 | w.preProcess(obj, req) 35 | w.postProcess(obj, req, res) 36 | 37 | assert.Equal(t, res.Headers.Get("Vary"), "Accept") 38 | assert.Equal(t, obj.Transforms.FormatStr, "webp") 39 | } 40 | 41 | func TestDontChangeWhenNoAccept(t *testing.T) { 42 | req, _ := http.NewRequest("GET", "http://mort/local/small.jpg-m", nil) 43 | req.Header.Add("Accept", "image/*") 44 | 45 | mortConfig := config.Config{} 46 | err := mortConfig.Load("../benchmark/small.yml") 47 | assert.Nil(t, err) 48 | 49 | obj, err := object.NewFileObject(req.URL, &mortConfig) 50 | assert.Nil(t, err) 51 | 52 | res := response.NewNoContent(200) 53 | res.Headers.Set("content-type", "image/jpg") 54 | 55 | obj.Ctx = req.Context() 56 | w := WebpPlugin{} 57 | w.preProcess(obj, req) 58 | w.postProcess(obj, req, res) 59 | 60 | assert.Equal(t, res.Headers.Get("Vary"), "Accept") 61 | assert.Equal(t, obj.Transforms.FormatStr, "") 62 | } 63 | 64 | func TestDontChangeWhenNotImage(t *testing.T) { 65 | req, _ := http.NewRequest("GET", "http://mort/local/small.jpg-m", nil) 66 | req.Header.Add("Accept", "image/*") 67 | 68 | mortConfig := config.Config{} 69 | err := mortConfig.Load("../benchmark/small.yml") 70 | assert.Nil(t, err) 71 | 72 | obj, err := object.NewFileObject(req.URL, &mortConfig) 73 | assert.Nil(t, err) 74 | 75 | res := response.NewNoContent(200) 76 | res.Headers.Set("content-type", "text/plain") 77 | 78 | obj.Ctx = req.Context() 79 | w := WebpPlugin{} 80 | w.preProcess(obj, req) 81 | w.postProcess(obj, req, res) 82 | 83 | assert.Equal(t, res.Headers.Get("Vary"), "") 84 | assert.Equal(t, obj.Transforms.FormatStr, "") 85 | } 86 | -------------------------------------------------------------------------------- /tests-int/Compress.Spec.js: -------------------------------------------------------------------------------- 1 | 2 | const supertest = require('supertest'); 3 | const crypto = require('crypto'); 4 | const { expect } = require('chai'); 5 | const fs = require('fs'); 6 | const zlib = require('zlib'); 7 | const stream = require('stream'); 8 | const binary = require('superagent-binary-parser'); 9 | 10 | const host = process.env.MORT_HOST + ':' + + process.env.MORT_PORT; 11 | const request = supertest(`http://${host}`); 12 | const filePath = '/local/main.css'; 13 | 14 | const hashFile = async (input) => { 15 | let hash = crypto.createHash('sha256'); 16 | hash.setEncoding('hex'); 17 | 18 | return new Promise((resolve, reject) => { 19 | input.on('end', () => { 20 | hash.end(); 21 | let hashHex = hash.read(); 22 | resolve(hashHex); 23 | }); 24 | input.pipe(hash); 25 | }); 26 | }; 27 | 28 | const hashBuffer = (data) => { 29 | return crypto.createHash('sha256').update(data).digest('hex') 30 | } 31 | 32 | describe('Compression', function () { 33 | describe('gzip', function () { 34 | it('should return gzip response', function (done) { 35 | request.get(filePath) 36 | .set('Accept-Encoding', 'gzip') 37 | .expect(200) 38 | .expect('Content-Encoding', 'gzip') 39 | .end(async (err, res) => { 40 | if(err) done(err) 41 | const expectedHash = await hashFile(fs.createReadStream('/tmp/mort-tests/local/main.css')) 42 | expect(hashBuffer(res.text)).to.be.eql(expectedHash) 43 | done() 44 | }) 45 | }); 46 | }); 47 | 48 | describe('br', function () { 49 | it('should return br response', function (done) { 50 | request.get(filePath) 51 | .set('accept-encoding', 'br, gzip') 52 | .parse(binary) 53 | .expect(200) 54 | .expect('content-encoding', 'br') 55 | .end(async (err, res) => { 56 | if(err) done(err) 57 | const expectedHash = await hashFile(fs.createReadStream('/tmp/mort-tests/local/main.css')) 58 | zlib.brotliDecompress(res.body, async (err, body) => { 59 | if (err) done(err) 60 | const hash = await hashFile(stream.Readable.from(body)) 61 | expect(hash).to.be.eql(expectedHash) 62 | done() 63 | 64 | }) 65 | 66 | }) 67 | }) 68 | }); 69 | }) -------------------------------------------------------------------------------- /doc/TengoUrlParser.md: -------------------------------------------------------------------------------- 1 | # Tengo URL parser 2 | 3 | With mort you can use your own url parser for extracting operation for images 4 | 5 | ## Configuration 6 | 7 | To enable tengo decoder you need to have configuration like below 8 | 9 | ```yaml 10 | # config.yaml 11 | buckets: 12 | tengo: 13 | keys: 14 | - accessKey: "acc" 15 | secretAccessKey: "sec" 16 | transform: 17 | kind: "tengo" # enable tengo decoder/parser 18 | tengoPath: 'parse.tengo' # path to tengo script 19 | storages: 20 | basic: 21 | kind: "http" 22 | url: "https://i.imgur.com/" 23 | headers: 24 | "x--key": "sec" 25 | transform: 26 | kind: "local-meta" 27 | rootPath: "/tmp/mort/" 28 | pathPrefix: "transforms" 29 | ``` 30 | 31 | Tengo script 32 | 33 | ```go 34 | fmt := import("fmt") 35 | text := import("text") 36 | 37 | parse := func(reqUrl, bucketConfigF, obj) { 38 | // split by "." to remove object extension 39 | elements := text.split_n(reqUrl.path, ".", 2) 40 | ext := elements[1] 41 | if len(elements) == 1 { 42 | return "" 43 | } 44 | // split by "," to find resize parameters 45 | elements = text.split(elements[0], ",") 46 | 47 | // url has no transform 48 | if len(elements) == 1 { 49 | return "" 50 | } 51 | 52 | // apply parameters 53 | width := 0 54 | height := 0 55 | parent := elements[0] +"." + ext 56 | trans := elements[1:] 57 | for tran in trans { 58 | if tran[0] == 'w' { 59 | width = tran[1:] 60 | } 61 | 62 | if tran[0] == 'h' { 63 | height = tran[1:] 64 | } 65 | } 66 | 67 | obj.transforms.resize(int(width), int(height), false, false, false) 68 | return parent 69 | } 70 | 71 | parent := parse(url, bucketConfig, obj) 72 | err := undefined 73 | ``` 74 | 75 | Above script will work for URL http://localhost:8084/tengo/udXmD2T,w100,h100.jpeg 76 | 77 | 78 | Mort is injecting variables inside of tengo script 79 | * `url` golang net.URL struct [url](tengo/url.md) 80 | * `bucketConfig` - mort bucket configuration [bucketConfig](tengo/bucketconfig.md) 81 | * `obj` - mort object.FileObject [fileobject](tengo/fileobject.md) 82 | 83 | Output variables 84 | 85 | * `parent` - path for parent object, string 86 | * `err` - error if occurred 87 | 88 | 89 | More advance example of tengo script https://github.com/aldor007/mort/blob/master/pkg/object/tengo/testdata/preset.tengo 90 | -------------------------------------------------------------------------------- /pkg/cache/generic_singleton_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | "github.com/alicebob/miniredis/v2" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | // TestCreateCache_MultipleCallsShareRedisClient verifies connection pool sharing 12 | func TestCreateCache_MultipleCallsShareRedisClient(t *testing.T) { 13 | t.Parallel() 14 | 15 | s := miniredis.RunT(t) 16 | 17 | cfg := config.CacheCfg{ 18 | Type: "redis", 19 | Address: []string{s.Addr()}, 20 | } 21 | 22 | // Create first cache 23 | cache1 := CreateCache[testStruct](cfg) 24 | redisCache1, ok1 := cache1.(*GenericRedisCache[testStruct]) 25 | assert.True(t, ok1, "should be GenericRedisCache") 26 | 27 | // Create second cache with same config 28 | cache2 := CreateCache[testStruct](cfg) 29 | redisCache2, ok2 := cache2.(*GenericRedisCache[testStruct]) 30 | assert.True(t, ok2, "should be GenericRedisCache") 31 | 32 | // Different Cache instances 33 | assert.NotSame(t, cache1, cache2, "Cache instances are different (OK)") 34 | 35 | // But same underlying Redis client (shared pool) 36 | assert.Same(t, redisCache1.client, redisCache2.client, 37 | "Redis clients should be the same (shared pool)") 38 | } 39 | 40 | // TestCreateCache_DifferentTypesShareClient verifies pool sharing across types 41 | func TestCreateCache_DifferentTypesShareClient(t *testing.T) { 42 | t.Parallel() 43 | 44 | s := miniredis.RunT(t) 45 | 46 | cfg := config.CacheCfg{ 47 | Type: "redis", 48 | Address: []string{s.Addr()}, 49 | } 50 | 51 | // Create cache for different types 52 | cache1 := CreateCache[testStruct](cfg) 53 | cache2 := CreateCache[string](cfg) 54 | 55 | redisCache1 := cache1.(*GenericRedisCache[testStruct]) 56 | redisCache2 := cache2.(*GenericRedisCache[string]) 57 | 58 | // Should share the same Redis client despite different types 59 | assert.Same(t, redisCache1.client, redisCache2.client, 60 | "Different Cache[T] types should share Redis client") 61 | } 62 | 63 | // TestCreateCache_MemoryVsRedis verifies backend selection 64 | func TestCreateCache_MemoryVsRedis(t *testing.T) { 65 | t.Parallel() 66 | 67 | s := miniredis.RunT(t) 68 | 69 | memCfg := config.CacheCfg{Type: "memory"} 70 | redisCfg := config.CacheCfg{Type: "redis", Address: []string{s.Addr()}} 71 | 72 | memCache := CreateCache[testStruct](memCfg) 73 | redisCache := CreateCache[testStruct](redisCfg) 74 | 75 | _, isMemory := memCache.(*GenericMemoryCache[testStruct]) 76 | _, isRedis := redisCache.(*GenericRedisCache[testStruct]) 77 | 78 | assert.True(t, isMemory, "should create memory cache for type=memory") 79 | assert.True(t, isRedis, "should create redis cache for type=redis") 80 | } 81 | -------------------------------------------------------------------------------- /pkg/object/tengo/file_object.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "github.com/aldor007/mort/pkg/object" 5 | tengoLib "github.com/d5/tengo/v2" 6 | "github.com/d5/tengo/v2/token" 7 | ) 8 | 9 | // FileObject struct wraping objectFileObject 10 | type FileObject struct { 11 | tengoLib.ObjectImpl 12 | Value *object.FileObject 13 | } 14 | 15 | // String returns object uri 16 | func (o *FileObject) String() string { 17 | return o.Value.Uri.String() 18 | } 19 | 20 | // BinaryOp not implemented 21 | func (o *FileObject) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 22 | return nil, tengoLib.ErrInvalidOperator 23 | } 24 | 25 | // IsFalsy returns false if uri is empty 26 | func (o *FileObject) IsFalsy() bool { 27 | return o.Value.Uri.String() == "" 28 | } 29 | 30 | // Equals returns true if objects url are the same 31 | func (o *FileObject) Equals(x tengoLib.Object) bool { 32 | return o.String() == x.String() 33 | } 34 | 35 | // Copy create copy using object.FileObject.Copy 36 | func (o *FileObject) Copy() tengoLib.Object { 37 | return &FileObject{ 38 | Value: o.Value.Copy(), 39 | } 40 | } 41 | 42 | func (o *FileObject) TypeName() string { 43 | return "FileObject-object" 44 | } 45 | 46 | // IndexGet returns the value for the given key. 47 | // * `uri` return object Url 48 | // * `bucket` return bucket name string 49 | // * `key` return object storage path 50 | // * `transforms` return Transforms object on which you can execute image manipulations 51 | // Usage in tengo 52 | // 53 | // obj.key // access to object key 54 | func (o *FileObject) IndexGet(index tengoLib.Object) (val tengoLib.Object, err error) { 55 | strIdx, ok := tengoLib.ToString(index) 56 | if !ok { 57 | err = tengoLib.ErrInvalidIndexType 58 | return 59 | } 60 | 61 | val = tengoLib.UndefinedValue 62 | switch strIdx { 63 | case "uri": 64 | val = &URL{Value: o.Value.Uri} 65 | case "bucket": 66 | val = &tengoLib.String{Value: o.Value.Bucket} 67 | case "key": 68 | val = &tengoLib.String{Value: o.Value.Key} 69 | case "transforms": 70 | val = &Transforms{Value: &o.Value.Transforms} 71 | } 72 | 73 | return val, nil 74 | } 75 | 76 | // IndexSet allow to change value on FileObject 77 | // * `allowChangeKey` 78 | // * `checkParent` 79 | // * `debug` 80 | func (o *FileObject) IndexSet(index, value tengoLib.Object) (err error) { 81 | strIdx, ok := tengoLib.ToString(index) 82 | if !ok { 83 | err = tengoLib.ErrInvalidIndexType 84 | return 85 | } 86 | 87 | switch strIdx { 88 | case "allowChangeKey": 89 | o.Value.AllowChangeKey, _ = tengoLib.ToBool(value) 90 | case "checkParent": 91 | o.Value.CheckParent, _ = tengoLib.ToBool(value) 92 | case "debug": 93 | o.Value.Debug, _ = tengoLib.ToBool(value) 94 | } 95 | 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /tests-int/Range.Spec.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const binary = require('superagent-binary-parser'); 3 | 4 | const file = fs.readFileSync('./pkg/processor/benchmark/local/large.jpeg'); 5 | const supertest = require('supertest'); 6 | 7 | const host = process.env.MORT_HOST + ':' + + process.env.MORT_PORT; 8 | const request = supertest(`http://${host}`); 9 | 10 | function getRange(range) { 11 | range = range.split('=')[1].split('-'); 12 | range = range.map(function (r) { 13 | return parseInt(r, 10); 14 | }); 15 | if (range[1] >= file.length) { 16 | range[1] = file.length - 1; 17 | } 18 | return range; 19 | } 20 | 21 | describe('Range', function () { 22 | 23 | it('should handle single range', function (done) { 24 | request.get('/local/large.jpeg') 25 | .set('Range', 'bytes=0-1000') 26 | .parse(binary) 27 | .buffer() 28 | .expect('Content-Range', 'bytes 0-1000/' + file.length) 29 | .expect(206, file.slice(0, 1001)) 30 | .end(done); 31 | }); 32 | 33 | it('should handle multiple ranges', function (done) { 34 | 35 | request.get('/local/large.jpeg') 36 | .set('Range', 'bytes=2000-4000,6000-6500') 37 | .parse(binary) 38 | .buffer() 39 | .expect(function (res) { 40 | var str = res.body.toString('utf-8'); 41 | if (str.indexOf('Content-Range: bytes 2000-4000/' + file.length) === -1) { 42 | throw new Error('Missing first range'); 43 | } 44 | if (str.indexOf('Content-Range: bytes 6000-6500/' + file.length) === -1) { 45 | throw new Error('Missing second range'); 46 | } 47 | if (res.body.length <= 2502) { // first range + second range (+ delimiters) 48 | throw new Error('Response too small'); 49 | } 50 | }) 51 | .expect(206) 52 | .end(done); 53 | }); 54 | 55 | it('should handle partial ranges', function (done) { 56 | request.get('/local/large.jpeg') 57 | .set('Range', 'bytes=20000-') 58 | .parse(binary) 59 | .buffer() 60 | .expect('Content-Range', 'bytes 20000-' + (file.length - 1) + '/' + file.length) 61 | .expect(206, file.slice(20000)) 62 | .end(done); 63 | }); 64 | 65 | 66 | it('should return 416 on unparsable range', function (done) { 67 | request.get('/local/large.jpeg') 68 | .set('Range', 'litres=3.14-1.68') 69 | .parse(binary) 70 | .buffer() 71 | .expect(416) 72 | .end(done); 73 | }); 74 | 75 | }); 76 | -------------------------------------------------------------------------------- /charts/mort/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- $fullName := "mort.fullname" }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "mort.fullname" . }} 6 | labels: 7 | {{- include "mort.labels" . | nindent 4 }} 8 | spec: 9 | {{- if not .Values.autoscaling.enabled }} 10 | replicas: {{ .Values.replicaCount }} 11 | {{- end }} 12 | selector: 13 | matchLabels: 14 | {{- include "mort.selectorLabels" . | nindent 6 }} 15 | template: 16 | metadata: 17 | {{- with .Values.podAnnotations }} 18 | annotations: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | labels: 22 | {{- include "mort.selectorLabels" . | nindent 8 }} 23 | spec: 24 | volumes: 25 | - name: mort-config 26 | configMap: 27 | name: "{{ $fullName }}-config" 28 | items: 29 | - key: config.yaml 30 | path: config.yaml 31 | {{- with .Values.imagePullSecrets }} 32 | imagePullSecrets: 33 | {{- toYaml . | nindent 8 }} 34 | {{- end }} 35 | serviceAccountName: {{ include "mort.serviceAccountName" . }} 36 | securityContext: 37 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 38 | containers: 39 | - name: {{ .Chart.Name }} 40 | securityContext: 41 | {{- toYaml .Values.securityContext | nindent 12 }} 42 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" 43 | imagePullPolicy: {{ .Values.image.pullPolicy }} 44 | volumeMounts: 45 | - name: mort-config 46 | mountPath: "/etc/mort/mort.yml" 47 | subPath: mort.yml 48 | {{- if .Values.secrets.enabled }} 49 | env: 50 | {{- range $key, $value := .Values.secrets.env }} 51 | - name: {{ $key }} 52 | valueFrom: 53 | secretKeyRef: 54 | name: "{{ $fullName }}-env-secrets" 55 | key: {{ $key }} 56 | {{- end }} 57 | {{- end }} 58 | ports: 59 | - name: http 60 | containerPort: 8080 61 | protocol: TCP 62 | livenessProbe: 63 | httpGet: 64 | path: / 65 | port: http 66 | readinessProbe: 67 | httpGet: 68 | path: / 69 | port: http 70 | resources: 71 | {{- toYaml .Values.resources | nindent 12 }} 72 | {{- with .Values.nodeSelector }} 73 | nodeSelector: 74 | {{- toYaml . | nindent 8 }} 75 | {{- end }} 76 | {{- with .Values.affinity }} 77 | affinity: 78 | {{- toYaml . | nindent 8 }} 79 | {{- end }} 80 | {{- with .Values.tolerations }} 81 | tolerations: 82 | {{- toYaml . | nindent 8 }} 83 | {{- end }} 84 | -------------------------------------------------------------------------------- /pkg/lock/lock.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | "github.com/aldor007/mort/pkg/monitoring" 8 | "github.com/aldor007/mort/pkg/response" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // Lock is responding for collapsing request for same object 13 | type Lock interface { 14 | // Lock try get a lock for given key 15 | Lock(ctx context.Context, key string) (observer LockResult, acquired bool) 16 | // Release remove lock for given key 17 | Release(ctx context.Context, key string) 18 | // NotifyAndRelease remove lock for given key and notify all clients waiting for result 19 | // SharedResponse allows sharing the same buffer across all waiting requests without copying 20 | NotifyAndRelease(ctx context.Context, key string, res *response.SharedResponse) 21 | } 22 | 23 | // LockResult contain struct 24 | type LockResult struct { 25 | ResponseChan chan *response.Response // channel on which you get response 26 | Cancel chan bool // channel for notify about cancel of waiting 27 | Error error // error when creating error 28 | } 29 | 30 | type lockData struct { 31 | Key string 32 | notifyQueue []LockResult 33 | } 34 | 35 | // AddWatcher add next request waiting for lock to expire or return result 36 | func (l *lockData) AddWatcher() LockResult { 37 | d := LockResult{} 38 | d.ResponseChan = make(chan *response.Response, 1) 39 | d.Cancel = make(chan bool, 1) 40 | l.notifyQueue = append(l.notifyQueue, d) 41 | return d 42 | } 43 | 44 | // NewNopLock create lock that do nothing 45 | func NewNopLock() *NopLock { 46 | return &NopLock{} 47 | } 48 | 49 | // NopLock will never collapse any request 50 | type NopLock struct { 51 | } 52 | 53 | // Lock always return that lock was acquired 54 | func (l *NopLock) Lock(_ context.Context, _ string) (LockResult, bool) { 55 | return LockResult{}, true 56 | } 57 | 58 | // Release do nothing 59 | func (l *NopLock) Release(_ context.Context, _ string) { 60 | 61 | } 62 | 63 | // NotifyAndRelease do nothing 64 | func (l *NopLock) NotifyAndRelease(_ context.Context, _ string, _ *response.SharedResponse) { 65 | 66 | } 67 | 68 | func Create(lockCfg *config.LockCfg, lockTimeout int) Lock { 69 | if lockCfg == nil { 70 | monitoring.Log().Info("Creating memory lock") 71 | return NewMemoryLock() 72 | } 73 | switch lockCfg.Type { 74 | case "redis": 75 | monitoring.Log().Info("Creating redis lock", zap.Strings("addr", lockCfg.Address), zap.Int("lockTimeout", lockTimeout)) 76 | r := NewRedisLock(lockCfg.Address, lockCfg.ClientConfig) 77 | r.LockTimeout = lockTimeout 78 | return r 79 | case "redis-cluster": 80 | monitoring.Log().Info("Creating redis-cluster lock", zap.Strings("addr", lockCfg.Address), zap.Int("lockTimeout", lockTimeout)) 81 | r := NewRedisCluster(lockCfg.Address, lockCfg.ClientConfig) 82 | r.LockTimeout = lockTimeout 83 | return r 84 | 85 | default: 86 | return NewMemoryLock() 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /pkg/object/tengo/transform_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | 7 | "github.com/aldor007/mort/pkg/config" 8 | "github.com/aldor007/mort/pkg/object/tengo" 9 | tengoLib "github.com/d5/tengo/v2" 10 | "github.com/d5/tengo/v2/token" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestTransformTengo(t *testing.T) { 15 | 16 | c := &config.Transform{ 17 | Path: "/aaa", 18 | ParentStorage: "basic", 19 | ParentBucket: "parent-bucket", 20 | PathRegexp: regexp.MustCompile("[a-z]+"), 21 | } 22 | 23 | tengoObject := tengo.Transform{Value: c} 24 | 25 | assert.Equal(t, tengoObject.String(), "/aaa") 26 | assert.True(t, tengoObject.Equals(tengoObject.Copy())) 27 | o, err := tengoObject.BinaryOp(token.Add, tengoObject.Copy()) 28 | assert.Nil(t, o) 29 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 30 | assert.False(t, tengoObject.IsFalsy()) 31 | assert.Equal(t, tengoObject.TypeName(), "Transform-object") 32 | } 33 | 34 | func TestTransformGetTengo(t *testing.T) { 35 | c := &config.Transform{ 36 | Path: "/aaa", 37 | ParentStorage: "basic", 38 | ParentBucket: "parent-bucket", 39 | PathRegexp: regexp.MustCompile("[a-z]+"), 40 | Kind: "it", 41 | } 42 | 43 | tengoObject := tengo.Transform{Value: c} 44 | // get unknown index 45 | v, err := tengoObject.IndexGet(&tengoLib.String{Value: "no-name"}) 46 | assert.Nil(t, err) 47 | assert.Equal(t, v, tengoLib.UndefinedValue) 48 | 49 | // invalid index type 50 | v, err = tengoObject.IndexGet(tengoLib.UndefinedValue) 51 | assert.Equal(t, err, tengoLib.ErrInvalidIndexType) 52 | 53 | // get path 54 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "path"}) 55 | assert.Nil(t, err) 56 | assert.Equal(t, v.TypeName(), "string") 57 | pathStr, _ := tengoLib.ToString(v) 58 | assert.Equal(t, pathStr, "/aaa") 59 | 60 | // get parentStorage 61 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "parentStorage"}) 62 | assert.Nil(t, err) 63 | assert.Equal(t, v.TypeName(), "string") 64 | pStorage, _ := tengoLib.ToString(v) 65 | assert.Equal(t, pStorage, "basic") 66 | 67 | // get parentBucket 68 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "parentBucket"}) 69 | assert.Nil(t, err) 70 | assert.Equal(t, v.TypeName(), "string") 71 | pBucket, _ := tengoLib.ToString(v) 72 | assert.Equal(t, pBucket, "parent-bucket") 73 | 74 | // get regexp 75 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "pathRegexp"}) 76 | assert.Nil(t, err) 77 | assert.Equal(t, v.TypeName(), "Regexp-object") 78 | 79 | // get kind 80 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "kind"}) 81 | assert.Nil(t, err) 82 | assert.Equal(t, v.TypeName(), "string") 83 | kind, _ := tengoLib.ToString(v) 84 | assert.Equal(t, kind, "it") 85 | 86 | // get presets 87 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "presets"}) 88 | assert.Nil(t, err) 89 | assert.Equal(t, v.TypeName(), "immutable-map") 90 | 91 | } 92 | -------------------------------------------------------------------------------- /pkg/object/tengo/bucket_config.go: -------------------------------------------------------------------------------- 1 | package tengo 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | tengoLib "github.com/d5/tengo/v2" 8 | "github.com/d5/tengo/v2/token" 9 | ) 10 | 11 | // BucketConfig struct wrapping tengo 12 | type BucketConfig struct { 13 | tengoLib.ObjectImpl 14 | Value config.Bucket 15 | } 16 | 17 | // Strings returns Bucket name 18 | func (o *BucketConfig) String() string { 19 | return o.Value.Name 20 | } 21 | 22 | // BinaryOp not implented 23 | func (o *BucketConfig) BinaryOp(op token.Token, rhs tengoLib.Object) (tengoLib.Object, error) { 24 | return nil, tengoLib.ErrInvalidOperator 25 | } 26 | 27 | // IsFalsy returns false if bucket name is empty 28 | func (o *BucketConfig) IsFalsy() bool { 29 | return o.Value.Name == "" 30 | } 31 | 32 | // Equals checks bucket name 33 | func (o *BucketConfig) Equals(x tengoLib.Object) bool { 34 | other := x.(*BucketConfig) 35 | return o.Value.Name == other.Value.Name 36 | } 37 | 38 | // Copy create shallow copy of bucket object 39 | func (o *BucketConfig) Copy() tengoLib.Object { 40 | 41 | return &BucketConfig{ 42 | Value: o.Value, 43 | } 44 | } 45 | 46 | func (o *BucketConfig) TypeName() string { 47 | return "BucketConfig-object" 48 | } 49 | 50 | // IndexGet returns the value for the given key. 51 | // for 52 | // * `transform` it will return Transform tengo object 53 | // * `keys` it will return tengo map with s3 access keys 54 | // * `headers` it will return tengo map with headers 55 | // * `name` it will return name of bucket 56 | // for others keys it will return undefine value 57 | func (o *BucketConfig) IndexGet(index tengoLib.Object) (val tengoLib.Object, err error) { 58 | strIdx, ok := tengoLib.ToString(index) 59 | if !ok { 60 | err = tengoLib.ErrInvalidIndexType 61 | return 62 | } 63 | 64 | val = tengoLib.UndefinedValue 65 | switch strIdx { 66 | case "transform": 67 | if o.Value.Transform != nil { 68 | val = &Transform{Value: o.Value.Transform.ForParser()} 69 | } else { 70 | err = fmt.Errorf("no transform for %s", o.Value.Name) 71 | return 72 | } 73 | 74 | case "keys": 75 | keys := make([]tengoLib.Object, len(o.Value.Keys)) 76 | for i, k := range o.Value.Keys { 77 | internalMap := make(map[string]tengoLib.Object) 78 | internalMap["accessKey"] = &tengoLib.String{ 79 | Value: k.AccessKey, 80 | } 81 | internalMap["secretAccessKey"] = &tengoLib.String{ 82 | Value: k.SecretAccessKey, 83 | } 84 | keys[i] = &tengoLib.Map{Value: internalMap} 85 | } 86 | val = &tengoLib.Array{ 87 | Value: keys, 88 | } 89 | case "headers": 90 | internalMap := make(map[string]tengoLib.Object) 91 | for k, v := range o.Value.Headers { 92 | internalMap[k] = &tengoLib.String{Value: v} 93 | } 94 | val = &tengoLib.Map{ 95 | Value: internalMap, 96 | } 97 | case "name": 98 | val = &tengoLib.String{Value: o.Value.Name} 99 | 100 | } 101 | 102 | return 103 | } 104 | -------------------------------------------------------------------------------- /pkg/config/testdata/config-tengo-invalid.yml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | - "unix:/tmp/mort.sock" 5 | monitoring: "prometheus" 6 | placeholder: "./testdata/config.yml" 7 | plugins: 8 | webp: ~ 9 | compress: 10 | gzip: 11 | types: 12 | - text/plain 13 | - text/css 14 | - application/json 15 | - application/javascript 16 | - text/xml 17 | - application/xml 18 | - application/xml+rss 19 | - text/javascript 20 | - text/html; 21 | level: 4 22 | brotli: 23 | types: 24 | - text/plain 25 | - text/css 26 | - application/json 27 | - application/javascript 28 | - text/xml 29 | - application/xml 30 | - application/xml+rss 31 | - text/javascript 32 | - text/html; 33 | level: 4 34 | 35 | 36 | headers: 37 | - statusCodes: [200] 38 | values: 39 | "cache-control": "max-age=84000, public" 40 | - statusCodes: [404, 400] 41 | values: 42 | "cache-control": "max-age=60, public" 43 | - statusCodes: [500, 503] 44 | values: 45 | "cache-control": "max-age=10, public" 46 | 47 | buckets: 48 | media: 49 | keys: 50 | - accessKey: "acc" 51 | secretAccessKey: "sec" 52 | transform: 53 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 54 | kind: "tengo" 55 | parentBucket: "media" 56 | resultKey: "hash" 57 | presets: 58 | small: 59 | quality: 75 60 | filters: 61 | thumbnail: 62 | width: 150 63 | blur: 64 | quality: 80 65 | filters: 66 | thumbnail: 67 | width: 700 68 | blur: 69 | sigma: 5.0 70 | webp: 71 | quality: 100 72 | format: webp 73 | filters: 74 | thumbnail: 75 | width: 1000 76 | watermark: 77 | quality: 100 78 | filters: 79 | thumbnail: 80 | width: 1300 81 | watermark: 82 | image: "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Imgur_logo.svg/150px-Imgur_logo.svg.png" 83 | position: "center-center" 84 | opacity: 0.5 85 | storages: 86 | basic: 87 | kind: "http" 88 | url: "https://i.imgur.com/" 89 | headers: 90 | "x--key": "sec" 91 | transform: 92 | kind: "local-meta" 93 | rootPath: "/tmp/mort" 94 | pathPrefix: "transf" -------------------------------------------------------------------------------- /pkg/object/tengo/testdata/preset.tengo: -------------------------------------------------------------------------------- 1 | fmt := import("fmt") 2 | text := import("text") 3 | presetToTransform := func(obj, preset) { 4 | filters := preset.filters 5 | 6 | if filters.thumbnial { 7 | err := obj.transforms.resize(filters.thumbnial.width, filters.thumbnial.height, filters.thumbnial.mode == "outbound", filters.thumbnial.preserveAspectRatio, filters.thumbnial.fill) 8 | if err != undefined { 9 | return err 10 | } 11 | } 12 | 13 | if filters.crop { 14 | err := obj.transforms.crop(filters.crop.width, filters.crop.height, filters.crop.gravity, filters.crop.mode == "outbound", filters.crop.embed) 15 | if err != undefined { 16 | return err 17 | } 18 | } 19 | if filters.watermark { 20 | err := obj.transforms.watermark(filters.watermark.image, filters.watermark.position, filters.watermark.opacity) 21 | if err != undefined { 22 | return err 23 | } 24 | } 25 | if filters.blur { 26 | err := obj.transforms.blur(filters.blur.sigma, filters.blur.minAmpl) 27 | if err != undefined { 28 | return err 29 | } 30 | } 31 | 32 | if filters.extract { 33 | err := obj.transforms.extract(filters.extract.top, filters.extract.left, filters.extract.width, filters.extract.height) 34 | if err != undefined { 35 | return err 36 | } 37 | } 38 | 39 | if filters.resizecropAuto != undefined { 40 | err := obj.transforms.resizeCropAuto(filters.resizecropAuto.Width, filters.ResizecropAuto.Height) 41 | if err != undefined { 42 | return err 43 | } 44 | } 45 | 46 | obj.transforms.quality(preset.quality) 47 | 48 | if filters.interlace == true { 49 | err := obj.transforms.interlace() 50 | if err != undefined { 51 | return err 52 | } 53 | } 54 | 55 | if filters.strip == true { 56 | err := obj.transforms.stripMetadata() 57 | if err != undefined { 58 | return err 59 | } 60 | } 61 | 62 | if preset.format != "" { 63 | err := obj.transforms.format(preset.format) 64 | if err != undefined { 65 | return err 66 | } 67 | } 68 | 69 | 70 | if filters.grayscale { 71 | obj.transforms.grayscale() 72 | } 73 | 74 | if filters.rotate != undefined { 75 | obj.transforms.rotate(filters.rotate.angle) 76 | } 77 | } 78 | 79 | parse := func(reqUrl, bucketConfig, obj) { 80 | trans := bucketConfig.transform 81 | matches := trans.pathRegexp(obj.key) 82 | if len(matches) == 0 { 83 | return ["", undefined] 84 | } 85 | 86 | 87 | presetName := matches["presetName"] 88 | parent := matches["parent"] 89 | 90 | if ok := trans.presets[presetName]; ok == undefined { 91 | return ["", error("unknown preset " + presetName)] 92 | } 93 | 94 | err := presetToTransform(obj, trans.presets[presetName]) 95 | 96 | if trans.parentBucket != "" { 97 | parent = "/" + trans.parentBucket + "/" + parent 98 | } else if !text.has_prefix(parent, "/") { 99 | parent = "/" + parent 100 | } 101 | 102 | return [parent, err] 103 | } 104 | 105 | result := parse(url, bucketConfig, obj) 106 | parent := result[0] 107 | err := result[1] 108 | -------------------------------------------------------------------------------- /pkg/config/testdata/config-tengo-compile-error.yml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | - "unix:/tmp/mort.sock" 5 | monitoring: "prometheus" 6 | placeholder: "./testdata/config.yml" 7 | plugins: 8 | webp: ~ 9 | compress: 10 | gzip: 11 | types: 12 | - text/plain 13 | - text/css 14 | - application/json 15 | - application/javascript 16 | - text/xml 17 | - application/xml 18 | - application/xml+rss 19 | - text/javascript 20 | - text/html; 21 | level: 4 22 | brotli: 23 | types: 24 | - text/plain 25 | - text/css 26 | - application/json 27 | - application/javascript 28 | - text/xml 29 | - application/xml 30 | - application/xml+rss 31 | - text/javascript 32 | - text/html; 33 | level: 4 34 | 35 | 36 | headers: 37 | - statusCodes: [200] 38 | values: 39 | "cache-control": "max-age=84000, public" 40 | - statusCodes: [404, 400] 41 | values: 42 | "cache-control": "max-age=60, public" 43 | - statusCodes: [500, 503] 44 | values: 45 | "cache-control": "max-age=10, public" 46 | 47 | buckets: 48 | media: 49 | keys: 50 | - accessKey: "acc" 51 | secretAccessKey: "sec" 52 | transform: 53 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 54 | kind: "tengo" 55 | tengoPath: "tengo.tengo" 56 | parentBucket: "media" 57 | resultKey: "hash" 58 | presets: 59 | small: 60 | quality: 75 61 | filters: 62 | thumbnail: 63 | width: 150 64 | blur: 65 | quality: 80 66 | filters: 67 | thumbnail: 68 | width: 700 69 | blur: 70 | sigma: 5.0 71 | webp: 72 | quality: 100 73 | format: webp 74 | filters: 75 | thumbnail: 76 | width: 1000 77 | watermark: 78 | quality: 100 79 | filters: 80 | thumbnail: 81 | width: 1300 82 | watermark: 83 | image: "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Imgur_logo.svg/150px-Imgur_logo.svg.png" 84 | position: "center-center" 85 | opacity: 0.5 86 | storages: 87 | basic: 88 | kind: "http" 89 | url: "https://i.imgur.com/" 90 | headers: 91 | "x--key": "sec" 92 | transform: 93 | kind: "local-meta" 94 | rootPath: "/tmp/mort" 95 | pathPrefix: "transf" -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=$TARGETPLATFORM ghcr.io/aldor007/mort-base:master-6feb103 AS builder 2 | 3 | ARG TARGETPLATFORM 4 | ARG BUILDPLATFORM 5 | ARG TARGETARCH 6 | ARG TAG='dev' 7 | ARG COMMIT="master" 8 | ARG DATE="now" 9 | 10 | ENV WORKDIR=/workspace 11 | ENV PATH=/usr/local/go/bin:$PATH 12 | 13 | WORKDIR $WORKDIR 14 | 15 | # Download dependencies first (better caching with cache mount) 16 | COPY go.mod go.sum ./ 17 | RUN --mount=type=cache,target=/go/pkg/mod \ 18 | go mod download 19 | 20 | # Copy source code 21 | COPY cmd/ $WORKDIR/cmd 22 | COPY .godir ${WORKDIR}/.godir 23 | COPY configuration/ ${WORKDIR}/configuration 24 | COPY etc/ ${WORKDIR}/etc 25 | COPY pkg/ ${WORKDIR}/pkg 26 | 27 | # Build binary with optimizations and build cache 28 | RUN --mount=type=cache,target=/root/.cache/go-build \ 29 | --mount=type=cache,target=/go/pkg/mod \ 30 | CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GOEXPERIMENT=greenteagc \ 31 | go build -ldflags="-s -w -X 'main.version=${TAG}' -X 'main.commit=${COMMIT}' -X 'main.date=${DATE}'" \ 32 | -trimpath \ 33 | -o /go/mort ./cmd/mort/mort.go 34 | 35 | # Download mime.types at build time for reproducibility 36 | RUN curl -fsSL -o /tmp/mime.types https://raw.githubusercontent.com/apache/httpd/refs/heads/trunk/docs/conf/mime.types 37 | 38 | 39 | # Runtime stage - use minimal ubuntu 20.04 40 | FROM --platform=$TARGETPLATFORM ubuntu:20.04 41 | 42 | # Install runtime dependencies, create user, and cleanup in single layer 43 | RUN apt-get update && \ 44 | DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ 45 | ca-certificates \ 46 | libglib2.0-0 libjpeg-turbo8 libpng16-16 libopenexr24 \ 47 | libwebp6 libwebpmux3 libwebpdemux2 libtiff5 libgif7 libexif12 libxml2 libpoppler-glib8 \ 48 | libmagickwand-6.q16-6 libpango1.0-0 libmatio9 libopenslide0 \ 49 | libgsf-1-114 fftw3 liborc-0.4-0 librsvg2-2 libcfitsio8 libimagequant0 libheif1 libbrotli1 && \ 50 | useradd -r -u 1000 -g users mort && \ 51 | mkdir -p /etc/mort && \ 52 | chown -R mort:users /etc/mort && \ 53 | apt-get autoremove -y && \ 54 | apt-get autoclean && \ 55 | apt-get clean && \ 56 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/cache/apt/archives/* 57 | 58 | # Copy libvips libraries and application files 59 | COPY --from=builder /usr/local/lib /usr/local/lib 60 | COPY --from=builder /go/mort /usr/local/bin/mort 61 | COPY --from=builder /workspace/configuration/config.yml /etc/mort/mort.yml 62 | COPY --from=builder /workspace/configuration/parse.tengo /etc/mort/parse.tengo 63 | COPY --from=builder /tmp/mime.types /etc/mime.types 64 | 65 | # Update linker cache and verify installation 66 | RUN ldconfig /usr/local/lib && \ 67 | /usr/local/bin/mort -version 68 | 69 | ENV MORT_CONFIG_DIR=/etc/mort 70 | 71 | # Switch to non-root user 72 | USER mort 73 | 74 | # Health check 75 | HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ 76 | CMD ["/usr/local/bin/mort", "-version"] 77 | 78 | # Run the server 79 | ENTRYPOINT ["/usr/local/bin/mort"] 80 | 81 | # Expose the server TCP port 82 | EXPOSE 8080 8081 83 | -------------------------------------------------------------------------------- /pkg/cache/redis_pool_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGetRedisClient_SharedConnectionPool(t *testing.T) { 10 | t.Parallel() 11 | 12 | addresses := []string{"localhost:6379"} 13 | clientConfig := map[string]string{} 14 | 15 | // Create first client 16 | client1 := getRedisClient(addresses, clientConfig, false) 17 | assert.NotNil(t, client1, "first client should be created") 18 | 19 | // Create second client with same config 20 | client2 := getRedisClient(addresses, clientConfig, false) 21 | assert.NotNil(t, client2, "second client should be created") 22 | 23 | // Should return the SAME client instance (shared pool) 24 | assert.Same(t, client1, client2, "clients with same config should be the same instance") 25 | } 26 | 27 | func TestGetRedisClient_DifferentConfigurations(t *testing.T) { 28 | t.Parallel() 29 | 30 | // Client 1: single address 31 | client1 := getRedisClient([]string{"localhost:6379"}, nil, false) 32 | 33 | // Client 2: different address 34 | client2 := getRedisClient([]string{"localhost:6380"}, nil, false) 35 | 36 | // Should be different clients (different configs) 37 | assert.NotSame(t, client1, client2, "clients with different addresses should be different instances") 38 | } 39 | 40 | func TestGetRedisClient_ClusterVsNonCluster(t *testing.T) { 41 | t.Parallel() 42 | 43 | addresses := []string{"localhost:6379"} 44 | 45 | // Non-cluster client 46 | client1 := getRedisClient(addresses, nil, false) 47 | 48 | // Cluster client with same address 49 | client2 := getRedisClient(addresses, nil, true) 50 | 51 | // Should be different clients (different cluster setting) 52 | assert.NotSame(t, client1, client2, "cluster and non-cluster clients should be different") 53 | } 54 | 55 | func TestHashConfig_Consistency(t *testing.T) { 56 | t.Parallel() 57 | 58 | tests := []struct { 59 | name string 60 | addresses []string 61 | cluster bool 62 | }{ 63 | { 64 | name: "should generate consistent hash for same config", 65 | addresses: []string{"localhost:6379"}, 66 | cluster: false, 67 | }, 68 | { 69 | name: "should generate consistent hash for multiple addresses", 70 | addresses: []string{"host1:6379", "host2:6379"}, 71 | cluster: false, 72 | }, 73 | } 74 | 75 | for _, tt := range tests { 76 | t.Run(tt.name, func(t *testing.T) { 77 | hash1 := hashConfig(tt.addresses, tt.cluster) 78 | hash2 := hashConfig(tt.addresses, tt.cluster) 79 | 80 | assert.Equal(t, hash1, hash2, "hash should be consistent for same config") 81 | assert.NotEmpty(t, hash1, "hash should not be empty") 82 | }) 83 | } 84 | } 85 | 86 | func TestHashConfig_Different(t *testing.T) { 87 | t.Parallel() 88 | 89 | hash1 := hashConfig([]string{"localhost:6379"}, false) 90 | hash2 := hashConfig([]string{"localhost:6380"}, false) 91 | hash3 := hashConfig([]string{"localhost:6379"}, true) 92 | 93 | assert.NotEqual(t, hash1, hash2, "different addresses should have different hashes") 94 | assert.NotEqual(t, hash1, hash3, "different cluster setting should have different hashes") 95 | } 96 | -------------------------------------------------------------------------------- /tests-int/setup-mort.js: -------------------------------------------------------------------------------- 1 | const axiosRetry = require('axios-retry').default; 2 | const axios = require('axios'); 3 | 4 | // Configure axios with retry for health checks 5 | axiosRetry(axios, { 6 | retries: 10, 7 | retryDelay: axiosRetry.exponentialDelay, 8 | retryCondition: (err) => { 9 | if (err.response && err.response.status > 0) { 10 | return false 11 | } 12 | return true 13 | } 14 | }); 15 | 16 | // Global retry configuration for tests 17 | global.TEST_RETRY_CONFIG = { 18 | maxRetries: 3, 19 | retryDelay: 1000, 20 | shouldRetry: function(err, res) { 21 | // Retry on connection errors 22 | if (err && ( 23 | err.code === 'ECONNREFUSED' || 24 | err.code === 'ECONNRESET' || 25 | err.code === 'ETIMEDOUT' || 26 | err.code === 'ENOTFOUND' || 27 | err.code === 'ENETUNREACH' || 28 | err.code === 'EPIPE' 29 | )) { 30 | return true; 31 | } 32 | 33 | // Retry on 5xx errors except 501 (Not Implemented) 34 | if (res && res.status >= 500 && res.status !== 501) { 35 | return true; 36 | } 37 | 38 | return false; 39 | } 40 | }; 41 | 42 | // Helper function to wrap test requests with retry logic 43 | global.withRetry = function(requestFn, options = {}) { 44 | const maxRetries = options.maxRetries || global.TEST_RETRY_CONFIG.maxRetries; 45 | const retryDelay = options.retryDelay || global.TEST_RETRY_CONFIG.retryDelay; 46 | 47 | return function(callback) { 48 | let attempt = 0; 49 | 50 | const tryRequest = () => { 51 | requestFn((err, res) => { 52 | if (err || (res && res.error)) { 53 | const shouldRetry = global.TEST_RETRY_CONFIG.shouldRetry(err, res); 54 | 55 | if (shouldRetry && attempt < maxRetries) { 56 | attempt++; 57 | const delay = retryDelay * Math.pow(2, attempt - 1); 58 | console.log(` → Retrying request (attempt ${attempt}/${maxRetries}) after ${delay}ms`); 59 | setTimeout(tryRequest, delay); 60 | return; 61 | } 62 | } 63 | 64 | callback(err, res); 65 | }); 66 | }; 67 | 68 | tryRequest(); 69 | }; 70 | }; 71 | 72 | before(async function () { 73 | this.timeout(60000); 74 | console.log('Waiting for mort server to be ready...'); 75 | 76 | // Wait for server with retries 77 | for (let i = 0; i < 30; i++) { 78 | try { 79 | await axios.get(`http://${process.env.MORT_HOST}:${process.env.MORT_PORT}`, { timeout: 2000 }); 80 | console.log('✓ Mort server is ready!'); 81 | return; 82 | } catch (e) { 83 | if (i < 29) { 84 | await new Promise(resolve => setTimeout(resolve, 1000)); 85 | } 86 | } 87 | } 88 | console.warn('⚠ Mort server may not be fully ready, proceeding anyway...'); 89 | }); 90 | 91 | after(function() { 92 | console.log('\n✓ Test suite completed'); 93 | }); 94 | -------------------------------------------------------------------------------- /pkg/engine/image_engine.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | "time" 7 | 8 | "crypto/md5" 9 | "encoding/hex" 10 | "github.com/h2non/bimg" 11 | 12 | "github.com/aldor007/mort/pkg/monitoring" 13 | "github.com/aldor007/mort/pkg/object" 14 | "github.com/aldor007/mort/pkg/response" 15 | "github.com/aldor007/mort/pkg/transforms" 16 | "go.uber.org/zap" 17 | ) 18 | 19 | // ImageEngine is main struct that is responding for image processing 20 | type ImageEngine struct { 21 | parent *response.Response // source file 22 | } 23 | 24 | // NewImageEngine create instance of ImageEngine with source file that should be processed 25 | func NewImageEngine(res *response.Response) *ImageEngine { 26 | return &ImageEngine{parent: res} 27 | } 28 | 29 | // Process main ImageEngine function that create new image (stored in response object) 30 | func (c *ImageEngine) Process(obj *object.FileObject, trans []transforms.Transforms) (*response.Response, error) { 31 | t := monitoring.Report().Timer("generation_time") 32 | defer t.Done() 33 | 34 | buf, err := c.parent.Body() 35 | 36 | if err != nil { 37 | return response.NewError(500, err), err 38 | } 39 | 40 | // Cache image type name to avoid repeated detection 41 | imageType := bimg.DetermineImageTypeName(buf) 42 | 43 | for _, tran := range trans { 44 | image := bimg.NewImage(buf) 45 | meta, err := image.Metadata() 46 | if err != nil { 47 | return response.NewError(500, err), err 48 | } 49 | 50 | // Use cached image type (updated after each transform if needed) 51 | optsArr, err := tran.BimgOptions(transforms.NewImageInfo(meta, imageType)) 52 | if err != nil { 53 | monitoring.Log().Error("ImageEngine unable to create opts array age", obj.LogData(zap.Any("transforms", trans), zap.Any("currentTrans", tran), zap.Error(err))...) 54 | return response.NewError(500, err), err 55 | } 56 | optsLen := len(optsArr) 57 | for i, opts := range optsArr { 58 | buf, err = image.Process(opts) 59 | if err != nil { 60 | monitoring.Log().Error("ImageEngine unable to process image", obj.LogData(zap.Any("optsArr", optsArr), zap.Any("opts", opts), zap.Error(err))...) 61 | return response.NewError(500, err), err 62 | } 63 | 64 | // Only create new image if not the last iteration 65 | if i < optsLen-1 { 66 | image = bimg.NewImage(buf) 67 | } 68 | } 69 | // Update image type for next transform (format may have changed) 70 | imageType = bimg.DetermineImageTypeName(buf) 71 | } 72 | 73 | bodyHash := md5.New() 74 | bodyHash.Write(buf) 75 | 76 | res := response.NewBuf(200, buf) 77 | res.SetContentType("image/" + bimg.DetermineImageTypeName(buf)) 78 | //res.Set("cache-control", "max-age=6000, public") 79 | res.Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) 80 | res.Set("ETag", hex.EncodeToString(bodyHash.Sum(nil))) 81 | meta, err := bimg.Metadata(buf) 82 | if err == nil { 83 | res.Set("x-amz-meta-public-width", strconv.Itoa(meta.Size.Width)) 84 | res.Set("x-amz-meta-public-height", strconv.Itoa(meta.Size.Height)) 85 | 86 | } else { 87 | monitoring.Log().Warn("ImageEngine/process unable to get metadata", obj.LogData(zap.Error(err))...) 88 | } 89 | 90 | return res, nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/object/tengo/file_object_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | "github.com/aldor007/mort/pkg/object" 8 | "github.com/aldor007/mort/pkg/object/tengo" 9 | tengoLib "github.com/d5/tengo/v2" 10 | "github.com/d5/tengo/v2/token" 11 | "github.com/stretchr/testify/assert" 12 | "net/url" 13 | ) 14 | 15 | func pathToURL(urlPath string) *url.URL { 16 | u, _ := url.Parse(urlPath) 17 | return u 18 | } 19 | 20 | func TestFileObjectTengo(t *testing.T) { 21 | objectPath := "/bucket/image.png" 22 | mortConfig := config.GetInstance() 23 | c, _ := object.NewFileObject(pathToURL(objectPath), mortConfig) 24 | 25 | tengoObject := tengo.FileObject{Value: c} 26 | 27 | assert.Equal(t, tengoObject.String(), objectPath) 28 | assert.True(t, tengoObject.Equals(tengoObject.Copy())) 29 | o, err := tengoObject.BinaryOp(token.Add, tengoObject.Copy()) 30 | assert.Nil(t, o) 31 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 32 | assert.False(t, tengoObject.IsFalsy()) 33 | assert.Equal(t, tengoObject.TypeName(), "FileObject-object") 34 | } 35 | 36 | func TestFileObjectGetTengo(t *testing.T) { 37 | objectPath := "/bucket/image.png" 38 | mortConfig := config.GetInstance() 39 | c, _ := object.NewFileObject(pathToURL(objectPath), mortConfig) 40 | 41 | tengoObject := tengo.FileObject{Value: c} 42 | // get unknown index 43 | v, err := tengoObject.IndexGet(&tengoLib.String{Value: "no-name"}) 44 | assert.Nil(t, err) 45 | assert.Equal(t, v, tengoLib.UndefinedValue) 46 | 47 | // invalid index type 48 | v, err = tengoObject.IndexGet(tengoLib.UndefinedValue) 49 | assert.Equal(t, err, tengoLib.ErrInvalidIndexType) 50 | 51 | // get uri 52 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "uri"}) 53 | assert.Nil(t, err) 54 | assert.Equal(t, v.TypeName(), "url-object") 55 | assert.Equal(t, v.String(), objectPath) 56 | 57 | // get bucket 58 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "bucket"}) 59 | assert.Nil(t, err) 60 | assert.Equal(t, v.TypeName(), "string") 61 | bStr, _ := tengoLib.ToString(v) 62 | assert.Equal(t, bStr, "bucket") 63 | 64 | // get transforms 65 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "transforms"}) 66 | assert.Nil(t, err) 67 | assert.Equal(t, v.TypeName(), "Transforms-object") 68 | 69 | // get key 70 | v, err = tengoObject.IndexGet(&tengoLib.String{Value: "key"}) 71 | assert.Nil(t, err) 72 | assert.Equal(t, v.TypeName(), "string") 73 | kStr, _ := tengoLib.ToString(v) 74 | assert.Equal(t, kStr, "/image.png") 75 | 76 | } 77 | 78 | func TestFileObjectSetTengo(t *testing.T) { 79 | objectPath := "/bucket/image.png" 80 | mortConfig := config.GetInstance() 81 | c, _ := object.NewFileObject(pathToURL(objectPath), mortConfig) 82 | 83 | tengoObject := tengo.FileObject{Value: c} 84 | 85 | err := tengoObject.IndexSet(&tengoLib.String{Value: "allowChangeKey"}, tengoLib.FalseValue) 86 | assert.Nil(t, err) 87 | assert.False(t, c.AllowChangeKey) 88 | 89 | err = tengoObject.IndexSet(&tengoLib.String{Value: "checkParent"}, tengoLib.TrueValue) 90 | assert.Nil(t, err) 91 | assert.True(t, c.CheckParent) 92 | 93 | err = tengoObject.IndexSet(&tengoLib.String{Value: "debug"}, tengoLib.TrueValue) 94 | assert.Nil(t, err) 95 | assert.True(t, c.Debug) 96 | } 97 | -------------------------------------------------------------------------------- /pkg/object/tengo/bucket_config_test.go: -------------------------------------------------------------------------------- 1 | package tengo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aldor007/mort/pkg/config" 7 | "github.com/aldor007/mort/pkg/object/tengo" 8 | tengoLib "github.com/d5/tengo/v2" 9 | "github.com/d5/tengo/v2/token" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestBucketConfigTengo(t *testing.T) { 14 | c := config.Bucket{ 15 | Name: "test", 16 | Transform: &config.Transform{}, 17 | Headers: nil, 18 | } 19 | tengoConfig := tengo.BucketConfig{Value: c} 20 | 21 | assert.Equal(t, tengoConfig.String(), "test") 22 | assert.True(t, tengoConfig.Equals(tengoConfig.Copy())) 23 | o, err := tengoConfig.BinaryOp(token.Add, tengoConfig.Copy()) 24 | assert.Nil(t, o) 25 | assert.Equal(t, err, tengoLib.ErrInvalidOperator) 26 | assert.False(t, tengoConfig.IsFalsy()) 27 | assert.Equal(t, tengoConfig.TypeName(), "BucketConfig-object") 28 | } 29 | 30 | func TestBucketConfigGetTengo(t *testing.T) { 31 | c := config.Bucket{ 32 | Name: "test", 33 | Transform: nil, 34 | Headers: nil, 35 | Keys: []config.S3Key{ 36 | config.S3Key{ 37 | AccessKey: "aaa", 38 | SecretAccessKey: "bbb", 39 | }, 40 | }, 41 | } 42 | tengoConfig := tengo.BucketConfig{Value: c} 43 | // get unknown index 44 | v, err := tengoConfig.IndexGet(&tengoLib.String{Value: "no-name"}) 45 | assert.Nil(t, err) 46 | assert.Equal(t, v, tengoLib.UndefinedValue) 47 | 48 | // invalid index type 49 | v, err = tengoConfig.IndexGet(tengoLib.UndefinedValue) 50 | assert.Equal(t, err, tengoLib.ErrInvalidIndexType) 51 | 52 | // get transform - nil 53 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "transform"}) 54 | assert.NotNil(t, err) 55 | 56 | // get transform 57 | tengoConfig.Value.Transform = &config.Transform{} 58 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "transform"}) 59 | assert.Nil(t, err) 60 | assert.Equal(t, v.TypeName(), "Transform-object") 61 | 62 | // get keys 63 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "keys"}) 64 | assert.Nil(t, err) 65 | assert.Equal(t, v.TypeName(), "array") 66 | vArr := v.(*tengoLib.Array) 67 | assert.Equal(t, len(vArr.Value), len(c.Keys)) 68 | k, err := vArr.IndexGet(&tengoLib.Int{Value: 0}) 69 | assert.Nil(t, err) 70 | kSecret, err := k.IndexGet(&tengoLib.String{Value: "accessKey"}) 71 | assert.Nil(t, err) 72 | kString, _ := tengoLib.ToString(kSecret) 73 | assert.Equal(t, kString, "aaa") 74 | 75 | // get headers - nil 76 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "headers"}) 77 | assert.Nil(t, err) 78 | assert.Equal(t, v.TypeName(), "map") 79 | 80 | // get headers 81 | tengoConfig.Value.Headers = make(map[string]string) 82 | tengoConfig.Value.Headers["x-test"] = "test" 83 | 84 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "headers"}) 85 | assert.Nil(t, err) 86 | assert.Equal(t, v.TypeName(), "map") 87 | hMap, err := v.IndexGet(&tengoLib.String{Value: "x-test"}) 88 | assert.Nil(t, err) 89 | hValue, _ := tengoLib.ToString(hMap) 90 | assert.Equal(t, hValue, "test") 91 | 92 | // get name 93 | v, err = tengoConfig.IndexGet(&tengoLib.String{Value: "name"}) 94 | assert.Nil(t, err) 95 | assert.Equal(t, v.TypeName(), "string") 96 | name, _ := tengoLib.ToString(v) 97 | assert.Equal(t, name, "test") 98 | 99 | } 100 | -------------------------------------------------------------------------------- /pkg/object/cloudinary/cloudinary.go: -------------------------------------------------------------------------------- 1 | package cloudinary 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "path" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/aldor007/mort/pkg/config" 11 | "github.com/aldor007/mort/pkg/monitoring" 12 | "github.com/aldor007/mort/pkg/object" 13 | "github.com/aldor007/mort/pkg/transforms" 14 | "go.uber.org/zap" 15 | "go.uber.org/zap/zapcore" 16 | ) 17 | 18 | const Kind = "cloudinary" 19 | 20 | type ( 21 | Decoder struct { 22 | cache map[string]transforms.Transforms 23 | cacheLock sync.RWMutex 24 | } 25 | ) 26 | 27 | func init() { 28 | decoder := newCloudinaryDecoder() 29 | object.RegisterParser(Kind, decoder.decode) 30 | } 31 | 32 | func newCloudinaryDecoder() *Decoder { 33 | return &Decoder{ 34 | cache: make(map[string]transforms.Transforms), 35 | } 36 | } 37 | 38 | func (c *Decoder) getCached(definition string) (transforms.Transforms, bool) { 39 | c.cacheLock.RLock() 40 | defer c.cacheLock.RUnlock() 41 | t, exists := c.cache[definition] 42 | return t, exists 43 | } 44 | 45 | func (c *Decoder) createTransformationsFromDefinition(definition string) (transforms.Transforms, error) { 46 | parser, err := newNotationParser(definition) 47 | if err != nil { 48 | return transforms.Transforms{}, err 49 | } 50 | transform, err := parser.NextTransform() 51 | if err != nil { 52 | return transforms.Transforms{}, err 53 | } 54 | if parser.HasNext() { 55 | return transforms.Transforms{}, errors.New("multiple transforms are not supported") 56 | } 57 | return transform, nil 58 | } 59 | 60 | func (c *Decoder) getTransformations(transformationsDefinition string) (transforms.Transforms, error) { 61 | var err error 62 | t, ok := c.getCached(transformationsDefinition) 63 | if !ok { 64 | t, err = c.createTransformationsFromDefinition(transformationsDefinition) 65 | if err != nil && err != errNoToken { 66 | return t, err 67 | } 68 | c.cacheLock.Lock() 69 | c.cache[transformationsDefinition] = t 70 | c.cacheLock.Unlock() 71 | } 72 | return t, nil 73 | } 74 | 75 | // decodePreset parse given url by matching user defined regexp with request path 76 | func (c *Decoder) decode(_ *url.URL, bucketConfig config.Bucket, obj *object.FileObject) (string, error) { 77 | trans := bucketConfig.Transform 78 | matches := trans.PathRegexp.FindStringSubmatch(obj.Key) 79 | if matches == nil { 80 | return "", nil 81 | } 82 | subMatchMap := make(map[string]string) 83 | 84 | for i, name := range trans.PathRegexp.SubexpNames() { 85 | if i != 0 && name != "" { 86 | subMatchMap[name] = matches[i] 87 | } 88 | } 89 | 90 | transformationsDefinition := subMatchMap["transformations"] 91 | if transformationsDefinition != "" { 92 | var err error 93 | obj.Transforms, err = c.getTransformations(transformationsDefinition) 94 | if err != nil { 95 | if errors.Is(err, notImplementedError{}) { 96 | monitoring.Log().Error("Cloudinary", append([]zapcore.Field{zap.Error(err)}, obj.LogData()...)...) 97 | return "", err 98 | } 99 | monitoring.Log().Info("Cloudinary", append([]zapcore.Field{zap.Error(err)}, obj.LogData()...)...) 100 | return "", err 101 | } 102 | } 103 | 104 | parent := subMatchMap["parent"] 105 | if trans.ParentBucket != "" { 106 | parent = "/" + path.Join(trans.ParentBucket, parent) 107 | } else if !strings.HasPrefix(parent, "/") { 108 | parent = "/" + parent 109 | } 110 | 111 | return parent, nil 112 | } 113 | -------------------------------------------------------------------------------- /pkg/config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEmptyString(t *testing.T) { 10 | c := Config{} 11 | err := c.LoadFromString("") 12 | assert.Nil(t, err) 13 | } 14 | 15 | func TestInvalidParentBucketInTransform(t *testing.T) { 16 | c := Config{} 17 | err := c.Load("testdata/invalid-parent-bucket.yml") 18 | assert.NotNil(t, err) 19 | } 20 | 21 | func TestInvalidParentStorageInTransform(t *testing.T) { 22 | c := Config{} 23 | err := c.Load("testdata/invalid-parent-storage.yml") 24 | assert.NotNil(t, err) 25 | } 26 | 27 | func TestNoBasicStorage(t *testing.T) { 28 | c := Config{} 29 | err := c.Load("testdata/no-basic-storage.yml") 30 | assert.NotNil(t, err) 31 | } 32 | 33 | func TestInvalidYaml(t *testing.T) { 34 | c := GetInstance() 35 | assert.Panics(t, func() { 36 | c.load([]byte(` 37 | server: 38 | a: [ 39 | `)) 40 | }) 41 | } 42 | 43 | func TestInvalidFile(t *testing.T) { 44 | c := GetInstance() 45 | assert.Panics(t, func() { 46 | c.Load("no-file") 47 | }) 48 | } 49 | 50 | func TestConfig_Load(t *testing.T) { 51 | c := Config{} 52 | c.BaseConfigPath = "testdata" 53 | err := c.Load("testdata/config.yml") 54 | 55 | assert.Nil(t, err) 56 | 57 | buckets := c.BucketsByAccessKey("acc") 58 | 59 | assert.Equal(t, len(buckets), 2) 60 | 61 | bucket := c.Buckets["media"] 62 | assert.Equal(t, bucket.Storages.Transform().Kind, "local-meta") 63 | 64 | bucket = c.Buckets["query"] 65 | assert.Equal(t, bucket.Transform.ResultKey, "hashParent") 66 | } 67 | 68 | func TestConfig_Load_TengoInvalid(t *testing.T) { 69 | c := Config{} 70 | err := c.Load("testdata/config-tengo-invalid.yml") 71 | 72 | assert.NotNil(t, err) 73 | assert.Equal(t, err.Error(), "unable to read tengo script file \"\", error open configuration: no such file or directory") 74 | } 75 | 76 | func TestConfig_Load_TengoCompileError(t *testing.T) { 77 | c := Config{} 78 | c.BaseConfigPath = "testdata" 79 | err := c.Load("testdata/config-tengo-compile-error.yml") 80 | 81 | assert.NotNil(t, err) 82 | assert.Equal(t, err.Error(), "unable to compile tengo script tengo.tengo error Compile Error: unresolved reference 'aaaa'\n\tat (main):1:1") 83 | } 84 | 85 | func TestConfig_Transform_ForParser(t *testing.T) { 86 | c := Config{} 87 | c.BaseConfigPath = "testdata" 88 | err := c.Load("testdata/config.yml") 89 | 90 | assert.Nil(t, err) 91 | ten := c.Buckets["tengo"].Transform.ForParser() 92 | assert.Nil(t, ten.TengoScript) 93 | } 94 | 95 | func TestConfig_ConcurrentImageProcessing(t *testing.T) { 96 | t.Parallel() 97 | 98 | t.Run("loads configured value", func(t *testing.T) { 99 | c := Config{} 100 | err := c.LoadFromString(` 101 | server: 102 | concurrentImageProcessing: 50 103 | listens: 104 | - ":8080" 105 | buckets: 106 | test: 107 | storages: 108 | basic: 109 | kind: "local-meta" 110 | rootPath: "/tmp" 111 | `) 112 | assert.Nil(t, err) 113 | assert.Equal(t, 50, c.Server.ConcurrentImageProcessing) 114 | }) 115 | 116 | t.Run("defaults to 0 when not set", func(t *testing.T) { 117 | c := Config{} 118 | err := c.LoadFromString(` 119 | server: 120 | listens: 121 | - ":8080" 122 | buckets: 123 | test: 124 | storages: 125 | basic: 126 | kind: "local-meta" 127 | rootPath: "/tmp" 128 | `) 129 | assert.Nil(t, err) 130 | assert.Equal(t, 0, c.Server.ConcurrentImageProcessing) 131 | }) 132 | } 133 | -------------------------------------------------------------------------------- /pkg/response/shared_response.go: -------------------------------------------------------------------------------- 1 | package response 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "net/http" 8 | "sync/atomic" 9 | 10 | "github.com/aldor007/mort/pkg/transforms" 11 | ) 12 | 13 | // SharedResponse wraps a Response with reference counting for safe sharing 14 | // across multiple goroutines. This eliminates the need to create full copies 15 | // of responses when distributing them to multiple consumers (e.g., in request collapsing). 16 | type SharedResponse struct { 17 | resp *Response 18 | refCount *atomic.Int32 19 | body []byte // Immutable shared buffer 20 | headers http.Header // Immutable shared headers 21 | } 22 | 23 | // NewSharedResponse creates a shareable response from a buffered response. 24 | // The response must be fully buffered (IsBuffered() == true) before creating a SharedResponse. 25 | // Returns an error if the response is not buffered or if buffering fails. 26 | func NewSharedResponse(resp *Response) (*SharedResponse, error) { 27 | if resp == nil { 28 | return nil, errors.New("response cannot be nil") 29 | } 30 | 31 | // Ensure response is buffered 32 | body, err := resp.Body() 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | sr := &SharedResponse{ 38 | resp: resp, 39 | refCount: &atomic.Int32{}, 40 | body: body, 41 | headers: resp.Headers.Clone(), 42 | } 43 | sr.refCount.Store(1) 44 | return sr, nil 45 | } 46 | 47 | // Acquire increments the reference count and returns a Response view 48 | // that shares the underlying buffer. The returned Response is safe to use 49 | // for reading but should not be modified. 50 | // 51 | // Each call to Acquire() must be matched with a corresponding Release() call. 52 | func (sr *SharedResponse) Acquire() *Response { 53 | sr.refCount.Add(1) 54 | 55 | // Create lightweight Response view that shares the buffer 56 | view := &Response{ 57 | StatusCode: sr.resp.StatusCode, 58 | Headers: sr.headers.Clone(), // Headers are small, clone for safety 59 | ContentLength: sr.resp.ContentLength, 60 | body: sr.body, // Share the body buffer (read-only) 61 | debug: sr.resp.debug, 62 | errorValue: sr.resp.errorValue, 63 | cachable: sr.resp.cachable, 64 | ttl: sr.resp.ttl, 65 | } 66 | 67 | // Set up reader for the shared body 68 | view.bodySeeker = bytes.NewReader(sr.body) 69 | view.reader = io.NopCloser(view.bodySeeker) 70 | 71 | // Copy transforms if present 72 | if len(sr.resp.trans) > 0 { 73 | view.trans = make([]transforms.Transforms, len(sr.resp.trans)) 74 | copy(view.trans, sr.resp.trans) 75 | } 76 | 77 | return view 78 | } 79 | 80 | // Release decrements the reference count. 81 | // When the reference count reaches zero, the underlying resources are released. 82 | // 83 | // This method is safe to call from multiple goroutines and should always be 84 | // called to prevent resource leaks. Use defer to ensure Release is called: 85 | // 86 | // view := sharedResp.Acquire() 87 | // defer sharedResp.Release() 88 | func (sr *SharedResponse) Release() { 89 | if sr == nil { 90 | return // Nil-safe for convenience 91 | } 92 | if sr.refCount.Add(-1) == 0 { 93 | // Last reference released, clean up resources 94 | if sr.resp != nil { 95 | sr.resp.Close() 96 | } 97 | } 98 | } 99 | 100 | // RefCount returns the current reference count. 101 | // This method is primarily useful for testing and debugging. 102 | func (sr *SharedResponse) RefCount() int32 { 103 | return sr.refCount.Load() 104 | } 105 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | branches: [ '*' ] 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | ci: 12 | container: 13 | image: ghcr.io/aldor007/mort-base:latest 14 | credentials: 15 | username: ${{ github.actor }} 16 | password: ${{ secrets.GHR_TOKEN }} 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Set up Go 22 | uses: actions/setup-go@v5 23 | with: 24 | go-version-file: 'go.mod' 25 | - name: Install 26 | run: go mod download && git config --global --add safe.directory '*' 27 | 28 | - name: Build 29 | run: go build -v ./... 30 | 31 | - name: Test 32 | run: ./scripts/unit-travis.sh 33 | 34 | redis-lock: 35 | container: 36 | image: ghcr.io/aldor007/mort-base:latest 37 | credentials: 38 | username: ${{ github.actor }} 39 | password: ${{ secrets.GHR_TOKEN }} 40 | runs-on: ubuntu-latest 41 | # Steps represent a sequence of tasks that will be executed as part of the job 42 | services: 43 | redis: 44 | image: redis 45 | options: >- 46 | --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 47 | ports: 48 | - 6379:6379 49 | steps: 50 | - name: Git checkout 51 | uses: actions/checkout@v4 52 | 53 | - name: Set up Go 54 | uses: actions/setup-go@v5 55 | with: 56 | go-version-file: 'go.mod' 57 | 58 | - name: Install Node JS 59 | uses: actions/setup-node@v4 60 | with: 61 | node-version: 18 62 | 63 | - name: Install npm deps 64 | run: npm install 65 | 66 | - name: Install go deps 67 | run: go mod download && git config --global --add safe.directory '*' 68 | 69 | - name: Run the integrations tests 70 | run: | 71 | make run-test-server-redis & 72 | pid=$! 73 | sleep 13 && curl --retry 20 --retry-delay 5 -s -o /dev/null "http://localhost:8091" || true && sleep 5 && npm run tests 74 | kill -9 $pid 75 | env: 76 | MORT_HOST: localhost 77 | MORT_PORT: 8091 78 | CI: true 79 | memory-lock: 80 | container: 81 | image: ghcr.io/aldor007/mort-base:latest 82 | credentials: 83 | username: ${{ github.actor }} 84 | password: ${{ secrets.GHR_TOKEN }} 85 | runs-on: ubuntu-latest 86 | # Steps represent a sequence of tasks that will be executed as part of the job 87 | steps: 88 | - name: Git checkout 89 | uses: actions/checkout@v4 90 | 91 | - name: Set up Go 92 | uses: actions/setup-go@v5 93 | with: 94 | go-version-file: 'go.mod' 95 | 96 | - name: Install Node JS 97 | uses: actions/setup-node@v4 98 | with: 99 | node-version: 18 100 | 101 | - name: Install npm deps 102 | run: npm install 103 | 104 | - name: Install go deps 105 | run: go mod download && git config --global --add safe.directory '*' 106 | 107 | - name: Run the integrations tests 108 | run: | 109 | make run-test-server & 110 | pid=$! 111 | sleep 13 && curl --retry 20 --retry-delay 5 -s -o /dev/null "http://localhost:8091" || true && sleep 5 && npm run tests 112 | kill -9 $pid 113 | env: 114 | MORT_HOST: localhost 115 | MORT_PORT: 8091 116 | CI: true 117 | -------------------------------------------------------------------------------- /tests-int/request-helper.js: -------------------------------------------------------------------------------- 1 | const supertest = require('supertest'); 2 | 3 | /** 4 | * Wrapper around supertest that adds retry logic for flaky tests 5 | */ 6 | class RetryableRequest { 7 | constructor(baseUrl, maxRetries = 3, retryDelay = 1000) { 8 | this.request = supertest(baseUrl); 9 | this.maxRetries = maxRetries; 10 | this.retryDelay = retryDelay; 11 | } 12 | 13 | /** 14 | * Execute a request with retry logic 15 | * @param {string} method - HTTP method (get, post, put, delete, etc.) 16 | * @param {string} path - Request path 17 | * @param {object} options - Additional options (timeout, etc.) 18 | * @returns {Promise} - Supertest Test object 19 | */ 20 | async executeWithRetry(method, path, options = {}) { 21 | let lastError; 22 | const maxRetries = options.maxRetries || this.maxRetries; 23 | const retryDelay = options.retryDelay || this.retryDelay; 24 | 25 | for (let attempt = 0; attempt <= maxRetries; attempt++) { 26 | try { 27 | const req = this.request[method](path); 28 | 29 | // Apply timeout if specified 30 | if (options.timeout) { 31 | req.timeout(options.timeout); 32 | } 33 | 34 | return req; 35 | } catch (error) { 36 | lastError = error; 37 | 38 | // Check if error is retryable 39 | if (this.isRetryableError(error) && attempt < maxRetries) { 40 | // Wait before retrying with exponential backoff 41 | const delay = retryDelay * Math.pow(2, attempt); 42 | await this.sleep(delay); 43 | continue; 44 | } 45 | 46 | // If not retryable or max retries reached, throw 47 | throw error; 48 | } 49 | } 50 | 51 | throw lastError; 52 | } 53 | 54 | /** 55 | * Determine if an error is retryable 56 | */ 57 | isRetryableError(error) { 58 | if (!error) return false; 59 | 60 | // Retry on connection errors 61 | if (error.code === 'ECONNREFUSED' || 62 | error.code === 'ECONNRESET' || 63 | error.code === 'ETIMEDOUT' || 64 | error.code === 'ENOTFOUND' || 65 | error.code === 'ENETUNREACH') { 66 | return true; 67 | } 68 | 69 | // Retry on 5xx errors except 501 (Not Implemented) 70 | if (error.status >= 500 && error.status !== 501) { 71 | return true; 72 | } 73 | 74 | return false; 75 | } 76 | 77 | sleep(ms) { 78 | return new Promise(resolve => setTimeout(resolve, ms)); 79 | } 80 | 81 | // Convenience methods 82 | get(path, options = {}) { 83 | return this.request.get(path); 84 | } 85 | 86 | post(path, options = {}) { 87 | return this.request.post(path); 88 | } 89 | 90 | put(path, options = {}) { 91 | return this.request.put(path); 92 | } 93 | 94 | delete(path, options = {}) { 95 | return this.request.delete(path); 96 | } 97 | 98 | head(path, options = {}) { 99 | return this.request.head(path); 100 | } 101 | 102 | patch(path, options = {}) { 103 | return this.request.patch(path); 104 | } 105 | } 106 | 107 | /** 108 | * Create a retryable request instance 109 | */ 110 | function createRetryableRequest(baseUrl, maxRetries = 3, retryDelay = 1000) { 111 | return new RetryableRequest(baseUrl, maxRetries, retryDelay); 112 | } 113 | 114 | module.exports = { 115 | RetryableRequest, 116 | createRetryableRequest 117 | }; 118 | -------------------------------------------------------------------------------- /charts/mort/files/config.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | monitoring: "prometheus" 5 | plugins: 6 | webp: ~ 7 | compress: 8 | gzip: 9 | types: 10 | - text/plain 11 | - text/css 12 | - application/json 13 | - application/javascript 14 | - text/xml 15 | - application/xml 16 | - application/xml+rss 17 | - text/javascript 18 | - text/html; 19 | level: 4 20 | brotli: 21 | types: 22 | - text/plain 23 | - text/css 24 | - application/json 25 | - application/javascript 26 | - text/xml 27 | - application/xml 28 | - application/xml+rss 29 | - text/javascript 30 | - text/html; 31 | level: 4 32 | 33 | headers: 34 | - statusCodes: [200] 35 | values: 36 | "cache-control": "max-age=84000, public" 37 | - statusCodes: [404, 400] 38 | values: 39 | "cache-control": "max-age=60, public" 40 | - statusCodes: [500, 503] 41 | values: 42 | "cache-control": "max-age=10, public" 43 | 44 | buckets: 45 | media: 46 | keys: 47 | - accessKey: "$MEDIA_ACCESS_KEY" 48 | secretAccessKey: "sec" 49 | transform: 50 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 51 | kind: "presets" 52 | parentBucket: "media" 53 | resultKey: "hash" 54 | presets: 55 | small: 56 | quality: 75 57 | filters: 58 | thumbnail: 59 | width: 150 60 | blur: 61 | quality: 80 62 | filters: 63 | thumbnail: 64 | width: 700 65 | blur: 66 | sigma: 5.0 67 | webp: 68 | quality: 100 69 | format: webp 70 | filters: 71 | thumbnail: 72 | width: 1000 73 | watermark: 74 | quality: 100 75 | filters: 76 | thumbnail: 77 | width: 1300 78 | watermark: 79 | image: "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Imgur_logo.svg/150px-Imgur_logo.svg.png" 80 | position: "center-center" 81 | opacity: 0.5 82 | storages: 83 | basic: 84 | kind: "http" 85 | url: "https://i.imgur.com/" 86 | headers: 87 | "x--key": "sec" 88 | transform: 89 | kind: "local-meta" 90 | rootPath: "/data/buckets" 91 | pathPrefix: "transforms" 92 | 93 | query: 94 | keys: 95 | - accessKey: "acc" 96 | secretAccessKey: "sec" 97 | transform: 98 | kind: "query" 99 | resultKey: "hash" 100 | storages: 101 | basic: 102 | kind: "http" 103 | url: "https://i.imgur.com/" 104 | headers: 105 | "x--key": "sec" 106 | transform: 107 | kind: "local-meta" 108 | rootPath: "/data/buckets/" 109 | pathPrefix: "transforms" 110 | local: 111 | keys: 112 | - accessKey: "acc" 113 | secretAccessKey: "sec" 114 | storages: 115 | basic: 116 | kind: "local-meta" 117 | rootPath: "/data/buckets/" 118 | -------------------------------------------------------------------------------- /configuration/config-glacier-example.yml: -------------------------------------------------------------------------------- 1 | # Mort Configuration Example with GLACIER Restore 2 | # 3 | # This example shows how to configure automatic S3 GLACIER object restoration 4 | # See docs/GLACIER_RESTORE.md for detailed documentation 5 | 6 | server: 7 | logLevel: "info" 8 | listens: 9 | - ":8080" # HTTP port 10 | - ":8081" # Metrics port 11 | 12 | # Cache configuration (used for both response caching and GLACIER restore tracking) 13 | cache: 14 | type: "redis" # Use Redis for distributed cache across instances 15 | address: 16 | - "localhost:6379" 17 | maxCacheItemSizeMB: 5 18 | cacheSize: 100 19 | 20 | # Request processing 21 | requestTimeout: 70 # Seconds 22 | internalListen: ":8081" 23 | 24 | buckets: 25 | # Example bucket with GLACIER restore enabled 26 | production-images: 27 | glacier: 28 | enabled: true # REQUIRED: Must be true to enable auto-restore 29 | restoreTier: "Standard" # Expedited (1-5min, $$$) | Standard (3-5hrs, $$) | Bulk (5-12hrs, $) 30 | restoreDays: 7 # Keep restored copy for 7 days 31 | retryAfterSeconds: 14400 # Optional: Auto-calculated (4 hours for Standard) 32 | 33 | storages: 34 | basic: 35 | kind: "s3" 36 | accessKey: "${S3_ACCESS_KEY}" 37 | secretAccessKey: "${S3_SECRET_KEY}" 38 | region: "us-east-1" 39 | bucket: "my-production-bucket" 40 | 41 | transform: 42 | kind: "s3" 43 | accessKey: "${S3_ACCESS_KEY}" 44 | secretAccessKey: "${S3_SECRET_KEY}" 45 | region: "us-east-1" 46 | bucket: "my-transforms-bucket" 47 | pathPrefix: "transforms" 48 | 49 | transform: 50 | kind: "presets-query" 51 | path: "/{presets}/{parent}.{ext}" 52 | parentStorage: "basic" 53 | 54 | presets: 55 | thumbnail: 56 | quality: 75 57 | filters: 58 | thumbnail: 59 | width: 150 60 | height: 150 61 | 62 | large: 63 | quality: 85 64 | filters: 65 | thumbnail: 66 | width: 1200 67 | height: 1200 68 | 69 | # Example bucket WITHOUT GLACIER restore (default behavior) 70 | # Objects in GLACIER will return 503 without attempting restore 71 | legacy-images: 72 | storages: 73 | basic: 74 | kind: "s3" 75 | accessKey: "${S3_ACCESS_KEY}" 76 | secretAccessKey: "${S3_SECRET_KEY}" 77 | region: "us-west-2" 78 | bucket: "legacy-bucket" 79 | 80 | # Example with Expedited tier for low-latency serving 81 | critical-assets: 82 | glacier: 83 | enabled: true 84 | restoreTier: "Expedited" # Fast but expensive - use sparingly 85 | restoreDays: 1 # Minimal storage cost 86 | retryAfterSeconds: 300 # 5 minutes 87 | 88 | storages: 89 | basic: 90 | kind: "s3" 91 | region: "eu-west-1" 92 | accessKey: "${S3_ACCESS_KEY}" 93 | secretAccessKey: "${S3_SECRET_KEY}" 94 | bucket: "critical-bucket" 95 | 96 | # Example with Bulk tier for cost optimization 97 | archive-images: 98 | glacier: 99 | enabled: true 100 | restoreTier: "Bulk" # Slow but cheap 101 | restoreDays: 30 # Long availability 102 | retryAfterSeconds: 43200 # 12 hours 103 | 104 | storages: 105 | basic: 106 | kind: "s3" 107 | region: "ap-southeast-1" 108 | accessKey: "${S3_ACCESS_KEY}" 109 | secretAccessKey: "${S3_SECRET_KEY}" 110 | bucket: "archive-bucket" 111 | 112 | # Headers to include in responses 113 | headers: 114 | - statusCodes: [200] 115 | override: false 116 | values: 117 | Cache-Control: "public, max-age=3600" 118 | X-Mort-Server: "v1" 119 | -------------------------------------------------------------------------------- /pkg/config/testdata/config.yml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | - "unix:/tmp/mort.sock" 5 | monitoring: "prometheus" 6 | placeholder: "./testdata/config.yml" 7 | plugins: 8 | webp: ~ 9 | compress: 10 | gzip: 11 | types: 12 | - text/plain 13 | - text/css 14 | - application/json 15 | - application/javascript 16 | - text/xml 17 | - application/xml 18 | - application/xml+rss 19 | - text/javascript 20 | - text/html; 21 | level: 4 22 | brotli: 23 | types: 24 | - text/plain 25 | - text/css 26 | - application/json 27 | - application/javascript 28 | - text/xml 29 | - application/xml 30 | - application/xml+rss 31 | - text/javascript 32 | - text/html; 33 | level: 4 34 | 35 | 36 | headers: 37 | - statusCodes: [200] 38 | values: 39 | "cache-control": "max-age=84000, public" 40 | - statusCodes: [404, 400] 41 | values: 42 | "cache-control": "max-age=60, public" 43 | - statusCodes: [500, 503] 44 | values: 45 | "cache-control": "max-age=10, public" 46 | 47 | buckets: 48 | tengo: 49 | transform: 50 | kind: "tengo" 51 | tengoPath: 'parse.tengo' 52 | storages: 53 | basic: 54 | kind: "http" 55 | url: "https://i.imgur.com/" 56 | headers: 57 | "x--key": "sec" 58 | transform: 59 | kind: "local-meta" 60 | rootPath: "/tmp/mort/" 61 | pathPrefix: "transforms" 62 | media: 63 | keys: 64 | - accessKey: "acc" 65 | secretAccessKey: "sec" 66 | transform: 67 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 68 | kind: "presets" 69 | parentBucket: "media" 70 | resultKey: "hash" 71 | presets: 72 | small: 73 | quality: 75 74 | filters: 75 | thumbnail: 76 | width: 150 77 | blur: 78 | quality: 80 79 | filters: 80 | thumbnail: 81 | width: 700 82 | blur: 83 | sigma: 5.0 84 | webp: 85 | quality: 100 86 | format: webp 87 | filters: 88 | thumbnail: 89 | width: 1000 90 | watermark: 91 | quality: 100 92 | filters: 93 | thumbnail: 94 | width: 1300 95 | watermark: 96 | image: "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Imgur_logo.svg/150px-Imgur_logo.svg.png" 97 | position: "center-center" 98 | opacity: 0.5 99 | storages: 100 | basic: 101 | kind: "http" 102 | url: "https://i.imgur.com/" 103 | headers: 104 | "x--key": "sec" 105 | transform: 106 | kind: "local-meta" 107 | rootPath: "/tmp/mort" 108 | pathPrefix: "transf" 109 | query: 110 | keys: 111 | - accessKey: "acc" 112 | secretAccessKey: "sec" 113 | transform: 114 | kind: "query" 115 | parentBucket: "media" 116 | storages: 117 | basic: 118 | kind: "http" 119 | url: "https://i.imgur.com/" 120 | headers: 121 | "x--key": "sec" 122 | transform: 123 | kind: "local-meta" 124 | rootPath: "/tmp/mort" 125 | pathPrefix: "transf" -------------------------------------------------------------------------------- /pkg/cache/redis.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | "github.com/aldor007/mort/pkg/monitoring" 9 | "github.com/aldor007/mort/pkg/object" 10 | "github.com/aldor007/mort/pkg/response" 11 | redisCache "github.com/go-redis/cache/v8" 12 | goRedis "github.com/go-redis/redis/v8" 13 | "github.com/vmihailenco/msgpack" 14 | ) 15 | 16 | func parseAddress(addrs []string) map[string]string { 17 | mp := make(map[string]string, len(addrs)) 18 | 19 | for _, addr := range addrs { 20 | parts := strings.Split(addr, ":") 21 | mp[parts[0]] = parts[0] + ":" + parts[1] 22 | } 23 | 24 | return mp 25 | } 26 | 27 | type CacheCfg struct { 28 | MaxItemSize int64 29 | MinUseCount uint64 30 | } 31 | 32 | type redisClient interface { 33 | Incr(ctx context.Context, key string) *goRedis.IntCmd 34 | Get(ctx context.Context, key string) *goRedis.StringCmd 35 | Del(ctx context.Context, keys ...string) *goRedis.IntCmd 36 | } 37 | 38 | // RedisCache store response in redis 39 | type RedisCache struct { 40 | cache *redisCache.Cache 41 | client redisClient 42 | 43 | cfg CacheCfg 44 | } 45 | 46 | // NewRedis create connection to redis and update it config from clientConfig map 47 | // Uses shared connection pool to avoid duplicate Redis connections 48 | func NewRedis(redisAddress []string, clientConfig map[string]string, cfg CacheCfg) *RedisCache { 49 | // Use shared Redis client from pool 50 | sharedClient := getRedisClient(redisAddress, clientConfig, false) 51 | 52 | cache := redisCache.New(&redisCache.Options{ 53 | Redis: sharedClient, 54 | LocalCache: redisCache.NewTinyLFU(10, time.Minute), 55 | }) 56 | 57 | return &RedisCache{cache, sharedClient, cfg} 58 | } 59 | 60 | func NewRedisCluster(redisAddress []string, clientConfig map[string]string, cfg CacheCfg) *RedisCache { 61 | // Use shared Redis client from pool 62 | sharedClient := getRedisClient(redisAddress, clientConfig, true) 63 | 64 | cache := redisCache.New(&redisCache.Options{ 65 | Redis: sharedClient, 66 | LocalCache: redisCache.NewTinyLFU(10, time.Minute), 67 | }) 68 | 69 | return &RedisCache{cache, sharedClient, cfg} 70 | } 71 | 72 | func (c *RedisCache) getKey(obj *object.FileObject) string { 73 | // Prepend prefix efficiently to avoid string concatenation allocation 74 | cacheKey := obj.GetResponseCacheKey() 75 | key := make([]byte, 0, 8+len(cacheKey)) // "mort-v1:" is 8 bytes 76 | key = append(key, "mort-v1:"...) 77 | key = append(key, cacheKey...) 78 | return string(key) 79 | } 80 | 81 | // Set put response into cache 82 | func (c *RedisCache) Set(obj *object.FileObject, res *response.Response) error { 83 | if res.ContentLength > c.cfg.MaxItemSize { 84 | return nil 85 | } 86 | 87 | if c.cfg.MinUseCount > 0 { 88 | countKey := "count" + c.getKey(obj) 89 | r := c.client.Incr(obj.Ctx, countKey) 90 | if counter, err := r.Uint64(); err == nil && counter < c.cfg.MinUseCount { 91 | // Not yet reached min use count, skip caching 92 | return nil 93 | } 94 | // Reached min use count or error occurred, delete counter and cache the item 95 | c.client.Del(obj.Ctx, countKey) 96 | } 97 | 98 | monitoring.Report().Inc("cache_ratio;status:set") 99 | v, err := msgpack.Marshal(res) 100 | if err != nil { 101 | return err 102 | } 103 | item := redisCache.Item{ 104 | Key: c.getKey(obj), 105 | Value: v, 106 | TTL: time.Second * time.Duration(res.GetTTL()), 107 | } 108 | return c.cache.Set(obj.Ctx, &item) 109 | } 110 | 111 | // Get returns response from cache or error 112 | func (c *RedisCache) Get(obj *object.FileObject) (*response.Response, error) { 113 | var buf []byte 114 | var res response.Response 115 | err := c.cache.Get(obj.Ctx, c.getKey(obj), &buf) 116 | if err != nil { 117 | monitoring.Report().Inc("cache_ratio;status:miss") 118 | } else { 119 | monitoring.Report().Inc("cache_ratio;status:hit") 120 | err = msgpack.Unmarshal(buf, &res) 121 | if res.Headers != nil { 122 | res.SetCacheHit() 123 | } 124 | } 125 | 126 | return &res, err 127 | } 128 | 129 | // Delete remove response from cache 130 | func (c *RedisCache) Delete(obj *object.FileObject) error { 131 | return c.cache.Delete(obj.Ctx, c.getKey(obj)) 132 | } 133 | -------------------------------------------------------------------------------- /pkg/processor/plugins/compress.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "compress/gzip" 5 | "io" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/aldor007/mort/pkg/helpers" 10 | "github.com/aldor007/mort/pkg/object" 11 | "github.com/aldor007/mort/pkg/response" 12 | brEnc "github.com/google/brotli/go/cbrotli" 13 | "github.com/klauspost/compress/zstd" 14 | ) 15 | 16 | func init() { 17 | RegisterPlugin("compress", &CompressPlugin{}) 18 | } 19 | 20 | type compressConfig struct { 21 | level int 22 | types []string 23 | enabled bool 24 | } 25 | 26 | // CompressPlugin plugins that transform image to webp if web browser can handle that format 27 | type CompressPlugin struct { 28 | brotli compressConfig 29 | gzip compressConfig 30 | zstd compressConfig 31 | } 32 | 33 | func parseConfig(cType *compressConfig, cfg interface{}) { 34 | cfgKeys := cfg.(map[interface{}]interface{}) 35 | 36 | if types, ok := cfgKeys["types"]; ok { 37 | typesArr := types.([]interface{}) 38 | for _, t := range typesArr { 39 | cType.types = append(cType.types, t.(string)) 40 | } 41 | } else { 42 | cType.types = []string{"text/html"} 43 | } 44 | 45 | if cLevel, ok := cfgKeys["level"]; ok { 46 | cType.level = cLevel.(int) 47 | } else { 48 | cType.level = 4 49 | } 50 | 51 | cType.enabled = true 52 | } 53 | 54 | func (c *CompressPlugin) configure(config interface{}) { 55 | cfg := config.(map[interface{}]interface{}) 56 | 57 | if tmpCfg, ok := cfg["brotli"]; ok { 58 | parseConfig(&c.brotli, tmpCfg) 59 | } 60 | 61 | if tmpCfg, ok := cfg["gzip"]; ok { 62 | parseConfig(&c.gzip, tmpCfg) 63 | } 64 | 65 | if tmpCfg, ok := cfg["zstd"]; ok { 66 | parseConfig(&c.zstd, tmpCfg) 67 | } 68 | } 69 | 70 | // PreProcess add webp transform to object 71 | func (CompressPlugin) preProcess(obj *object.FileObject, req *http.Request) { 72 | 73 | } 74 | 75 | // PostProcess update vary header 76 | func (c CompressPlugin) postProcess(obj *object.FileObject, req *http.Request, res *response.Response) { 77 | acceptEnc := req.Header.Get("Accept-Encoding") 78 | contentType := res.Headers.Get("Content-Type") 79 | if acceptEnc == "" || contentType == "" || helpers.IsRangeOrCondition(req) || (res.ContentLength < 1000 && res.ContentLength != -1) { 80 | return 81 | } 82 | 83 | if c.brotli.enabled && strings.Contains(acceptEnc, "br") { 84 | for _, supportedType := range c.brotli.types { 85 | if strings.Contains(contentType, supportedType) { 86 | res.Headers.Set("Content-Encoding", "br") 87 | res.Headers.Add("Vary", "Accept-Encoding") 88 | res.BodyTransformer(func(w io.Writer) io.WriteCloser { 89 | br := brEnc.NewWriter(w, brEnc.WriterOptions{Quality: c.brotli.level}) 90 | return br 91 | }) 92 | return 93 | } 94 | } 95 | 96 | } 97 | 98 | if c.zstd.enabled && strings.Contains(acceptEnc, "zstd") { 99 | for _, supportedType := range c.zstd.types { 100 | if strings.Contains(contentType, supportedType) { 101 | res.Headers.Set("Content-Encoding", "zstd") 102 | res.Headers.Add("Vary", "Accept-Encoding") 103 | res.BodyTransformer(func(w io.Writer) io.WriteCloser { 104 | // Map compression level (1-10) to zstd level 105 | // zstd levels go from 1 (fastest) to 19 (best compression), default is 3 106 | zstdLevel := zstd.SpeedDefault 107 | if c.zstd.level >= 1 && c.zstd.level <= 4 { 108 | zstdLevel = zstd.SpeedFastest 109 | } else if c.zstd.level >= 5 && c.zstd.level <= 7 { 110 | zstdLevel = zstd.SpeedDefault 111 | } else if c.zstd.level >= 8 { 112 | zstdLevel = zstd.SpeedBestCompression 113 | } 114 | 115 | zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstdLevel)) 116 | if err != nil { 117 | panic(err) 118 | } 119 | return zw 120 | }) 121 | return 122 | } 123 | } 124 | } 125 | 126 | if c.gzip.enabled && strings.Contains(acceptEnc, "gzip") { 127 | for _, supportedType := range c.gzip.types { 128 | if strings.Contains(contentType, supportedType) { 129 | res.Headers.Set("Content-Encoding", "gzip") 130 | res.Headers.Add("Vary", "Accept-Encoding") 131 | res.BodyTransformer(func(w io.Writer) io.WriteCloser { 132 | gzipW, err := gzip.NewWriterLevel(w, c.gzip.level) 133 | if err != nil { 134 | panic(err) 135 | } 136 | 137 | return gzipW 138 | }) 139 | return 140 | } 141 | } 142 | } 143 | 144 | } 145 | -------------------------------------------------------------------------------- /pkg/cache/memory.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "math" 5 | "time" 6 | "unsafe" 7 | 8 | "github.com/aldor007/mort/pkg/monitoring" 9 | "github.com/aldor007/mort/pkg/object" 10 | "github.com/aldor007/mort/pkg/response" 11 | "github.com/karlseguin/ccache/v3" 12 | "github.com/pkg/errors" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | type ( 17 | // MemoryCache uses memory for cache purpose 18 | MemoryCache struct { 19 | cache *ccache.Cache[responseSizeProvider] // cache for created image transformations 20 | } 21 | 22 | // responseSizeProvider adapts response.Response to how ccache size computation requirements. 23 | responseSizeProvider struct { 24 | *response.Response 25 | cachedSize int64 // Pre-calculated size to avoid recalculation 26 | } 27 | ) 28 | 29 | // Size returns the pre-calculated size of the cached response 30 | func (r responseSizeProvider) Size() int64 { 31 | return r.cachedSize 32 | } 33 | 34 | // calculateResponseSize computes the size once during cache entry creation 35 | func calculateResponseSize(res *response.Response) int64 { 36 | // Use ContentLength if available (more accurate and faster) 37 | size := res.ContentLength 38 | if size <= 0 { 39 | // If not available, try to get body size if already buffered 40 | if res.IsBuffered() { 41 | body, err := res.Body() 42 | if err != nil { 43 | // Return large value to prevent caching problematic responses 44 | return math.MaxInt64 45 | } 46 | size = int64(len(body)) 47 | } else { 48 | // For unbuffered responses, use a conservative estimate 49 | size = 1024 * 1024 // 1MB default estimate 50 | } 51 | } 52 | 53 | // Add header overhead (estimate) 54 | headerSize := int64(unsafe.Sizeof(res.Headers)) 55 | for k, v := range res.Headers { 56 | headerSize += int64(len(k)) 57 | for i := 0; i < len(v); i++ { 58 | headerSize += int64(len(v[i])) 59 | } 60 | } 61 | 62 | // Add ccache overhead (350 bytes) + response struct overhead 63 | return size + headerSize + 350 + int64(unsafe.Sizeof(*res)) 64 | } 65 | 66 | // NewMemoryCache returns instance of memory cache 67 | func NewMemoryCache(maxSize int64) *MemoryCache { 68 | return &MemoryCache{ccache.New[responseSizeProvider](ccache.Configure[responseSizeProvider]().MaxSize(maxSize).ItemsToPrune(50))} 69 | } 70 | 71 | // Set put response to cache. Cache takes ownership of the response - no copying. 72 | // The response must be buffered before caching. This eliminates one full copy 73 | // compared to the previous implementation that copied on both Set and Get. 74 | func (c *MemoryCache) Set(obj *object.FileObject, res *response.Response) error { 75 | // Ensure response is buffered before caching 76 | if !res.IsBuffered() { 77 | _, err := res.Body() 78 | if err != nil { 79 | return err 80 | } 81 | } 82 | 83 | monitoring.Report().Inc("cache_ratio;status:set") 84 | 85 | // Cache takes ownership - NO COPY! 86 | // Calculate size once when creating cache entry 87 | provider := responseSizeProvider{ 88 | Response: res, 89 | cachedSize: calculateResponseSize(res), 90 | } 91 | c.cache.Set(obj.GetResponseCacheKey(), provider, time.Second*time.Duration(res.GetTTL())) 92 | return nil 93 | } 94 | 95 | // Get returns a view of the cached response (zero-copy). 96 | // The view shares the underlying buffer with the cached response, eliminating 97 | // the need to copy the full response body on every cache hit. 98 | func (c *MemoryCache) Get(obj *object.FileObject) (*response.Response, error) { 99 | cacheValue := c.cache.Get(obj.GetResponseCacheKey()) 100 | if cacheValue != nil { 101 | monitoring.Log().Info("Handle Get cache", zap.String("cache", "hit"), zap.String("obj.Key", obj.Key)) 102 | monitoring.Report().Inc("cache_ratio;status:hit") 103 | 104 | cached := cacheValue.Value().Response 105 | 106 | // Create view instead of copy - zero memory allocation for body! 107 | view, err := cached.CreateView() 108 | if err != nil { 109 | monitoring.Report().Inc("cache_ratio;status:miss") 110 | return nil, errors.New("not found") 111 | } 112 | view.SetCacheHit() 113 | return view, nil 114 | } 115 | 116 | monitoring.Report().Inc("cache_ratio;status:miss") 117 | return nil, errors.New("not found") 118 | } 119 | 120 | // Delete remove given response from cache 121 | func (c *MemoryCache) Delete(obj *object.FileObject) error { 122 | c.cache.Delete(obj.GetResponseCacheKey()) 123 | return nil 124 | } 125 | -------------------------------------------------------------------------------- /example/config.yml: -------------------------------------------------------------------------------- 1 | server: 2 | listens: 3 | - ":8080" 4 | monitoring: "prometheus" 5 | plugins: 6 | webp: ~ 7 | compress: 8 | gzip: 9 | types: 10 | - text/plain 11 | - text/css 12 | - application/json 13 | - application/javascript 14 | - text/xml 15 | - application/xml 16 | - application/xml+rss 17 | - text/javascript 18 | - text/html; 19 | level: 4 20 | brotli: 21 | types: 22 | - text/plain 23 | - text/css 24 | - application/json 25 | - application/javascript 26 | - text/xml 27 | - application/xml 28 | - application/xml+rss 29 | - text/javascript 30 | - text/html; 31 | level: 4 32 | 33 | headers: 34 | - statusCodes: [200] 35 | values: 36 | "cache-control": "max-age=84000, public" 37 | - statusCodes: [404, 400] 38 | values: 39 | "cache-control": "max-age=60, public" 40 | - statusCodes: [500, 503] 41 | values: 42 | "cache-control": "max-age=10, public" 43 | 44 | buckets: 45 | media: 46 | keys: 47 | - accessKey: "acc" 48 | secretAccessKey: "sec" 49 | transform: 50 | path: "\\/(?P[a-z0-9_]+)\\/(?P.*)" 51 | kind: "presets" 52 | parentBucket: "media" 53 | resultKey: "hash" 54 | presets: 55 | small: 56 | quality: 75 57 | filters: 58 | thumbnail: 59 | width: 150 60 | blur: 61 | quality: 80 62 | filters: 63 | thumbnail: 64 | width: 700 65 | blur: 66 | sigma: 5.0 67 | webp: 68 | quality: 100 69 | format: webp 70 | filters: 71 | thumbnail: 72 | width: 1000 73 | watermark: 74 | quality: 100 75 | filters: 76 | thumbnail: 77 | width: 1300 78 | watermark: 79 | image: "https://upload.wikimedia.org/wikipedia/commons/thumb/e/e9/Imgur_logo.svg/150px-Imgur_logo.svg.png" 80 | position: "center-center" 81 | opacity: 0.5 82 | storages: 83 | basic: 84 | kind: "http" 85 | url: "https://i.imgur.com/" 86 | headers: 87 | "x--key": "sec" 88 | transform: 89 | kind: "local-meta" 90 | rootPath: "/data/buckets" 91 | pathPrefix: "transforms" 92 | 93 | query: 94 | keys: 95 | - accessKey: "acc" 96 | secretAccessKey: "sec" 97 | transform: 98 | kind: "query" 99 | resultKey: "hash" 100 | storages: 101 | basic: 102 | kind: "http" 103 | url: "https://i.imgur.com/" 104 | headers: 105 | "x--key": "sec" 106 | transform: 107 | kind: "local-meta" 108 | rootPath: "/data/buckets/" 109 | pathPrefix: "transforms" 110 | local: 111 | keys: 112 | - accessKey: "acc" 113 | secretAccessKey: "sec" 114 | storages: 115 | basic: 116 | kind: "local-meta" 117 | rootPath: "/data/buckets/" 118 | 119 | cloudinary: 120 | transform: 121 | kind: "cloudinary" 122 | resultKey: "hash" 123 | parentBucket: "cloudinary" 124 | path: "(?:\\/)(?P[^\\/]+)?(?P\\/[^\\/]*)" 125 | storages: 126 | basic: 127 | kind: "http" 128 | url: "https://i.imgur.com/" 129 | headers: 130 | "x--key": "sec" 131 | transform: 132 | kind: "noop" 133 | # transform: 134 | # kind: "local-meta" 135 | # rootPath: "/tmp/cache" 136 | # pathPrefix: "transforms" 137 | --------------------------------------------------------------------------------