3 | 
4 |
77 | A: Increase timeout. For miniflux it's HTTP_CLIENT_TIMEOUT, for other clients - read their documentation
78 | 79 | 80 | ## Development 81 | 82 | You need 83 | - Go 1.23 (most of application) 84 | - Node.js 20 (frontend) 85 | - Nats (with jetstream) 86 | - Redis 87 | 88 | Instaling dependencies example for MacOS: 89 | 90 | ```bash 91 | brew install go@1.23 92 | brew install node@20 93 | brew install redis 94 | brew install nats-server # Don't use brew services to manage nats because it lacks config support 95 | go mod download 96 | cd frontend/wizard-vue && npm install 97 | nats -js 98 | ``` 99 | 100 | Also this repository contains some useful git hooks. To enable them, use: 101 | ```bash 102 | git config --local core.hooksPath .githooks/ 103 | ``` 104 | -------------------------------------------------------------------------------- /cmd/extractor/extractor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "github.com/egor3f/rssalchemy/internal/config" 8 | dummycookies "github.com/egor3f/rssalchemy/internal/cookiemgr/dummy" 9 | "github.com/egor3f/rssalchemy/internal/dateparser" 10 | "github.com/egor3f/rssalchemy/internal/extractors/pwextractor" 11 | "github.com/egor3f/rssalchemy/internal/limiter/dummy" 12 | "github.com/egor3f/rssalchemy/internal/models" 13 | "github.com/felixge/fgprof" 14 | "github.com/labstack/gommon/log" 15 | "io" 16 | "os" 17 | "time" 18 | ) 19 | 20 | func main() { 21 | log.SetLevel(log.DEBUG) 22 | log.SetHeader(`${time_rfc3339_nano} ${level}`) 23 | 24 | outFile := flag.String("o", "", "Output file name") 25 | skipOutput := flag.Bool("s", false, "Skip json output; show just logs") 26 | useProfiler := flag.Bool("p", false, "Use profiler") 27 | flag.Parse() 28 | 29 | if *useProfiler { 30 | //goland:noinspection GoUnhandledErrorResult 31 | //defer fgtrace.Config{Dst: fgtrace.File(fmt.Sprintf("fgtrace_%d.json", time.Now().Unix()))}.Trace().Stop() 32 | w, err := os.Create(fmt.Sprintf("fgprof_%d.prof", time.Now().Unix())) 33 | if err != nil { 34 | panic(fmt.Sprintf("frprof create file: %v", err)) 35 | } 36 | stop := fgprof.Start(w, fgprof.FormatPprof) 37 | defer stop() 38 | } 39 | 40 | taskFileName := "task.json" 41 | if flag.NArg() > 0 { 42 | taskFileName = flag.Arg(0) 43 | } 44 | 45 | out := os.Stdout 46 | if len(*outFile) > 0 { 47 | var err error 48 | out, err = os.OpenFile(*outFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 49 | if err != nil { 50 | log.Panicf("open output file: %v", err) 51 | } 52 | //goland:noinspection GoUnhandledErrorResult 53 | defer out.Close() 54 | } 55 | 56 | task, err := loadTask(taskFileName) 57 | if err != nil { 58 | log.Panicf("load task: %v", err) 59 | } 60 | 61 | cfg, err := config.Read() 62 | if err != nil { 63 | log.Panicf("read config: %v", err) 64 | } 65 | 66 | pwe, err := pwextractor.New(pwextractor.Config{ 67 | Proxy: cfg.Proxy, 68 | DateParser: &dateparser.DateParser{ 69 | CurrentTimeFunc: func() time.Time { 70 | return time.Date(2025, 01, 10, 10, 00, 00, 00, time.UTC) 71 | }, 72 | }, 73 | CookieManager: dummycookies.New(), 74 | Limiter: &dummy.Limiter{}, 75 | }) 76 | if err != nil { 77 | log.Panicf("create pw extractor: %v", err) 78 | } 79 | defer func() { 80 | if err := pwe.Stop(); err != nil { 81 | log.Errorf("stop pw extractor: %v", err) 82 | } 83 | }() 84 | 85 | start := time.Now() 86 | result, err := pwe.Extract(task) 87 | log.Infof("Extract took %v ms", time.Since(start).Milliseconds()) 88 | if err != nil { 89 | log.Errorf("extract: %v", err) 90 | scrResult, err := pwe.Screenshot(task) 91 | if err != nil { 92 | log.Errorf("screenshot failed: %v", err) 93 | panic(err) 94 | } 95 | err = os.WriteFile("screenshot.png", scrResult.Image, 0600) 96 | if err != nil { 97 | log.Errorf("screenshot save failed: %v", err) 98 | } 99 | panic(err) 100 | } 101 | 102 | if !*skipOutput { 103 | resultStr, err := json.MarshalIndent(result, "", "\t") 104 | if err != nil { 105 | log.Panicf("marshal result: %v", err) 106 | } 107 | n, err := out.Write(resultStr) 108 | if err != nil { 109 | log.Panicf("write output: %v", err) 110 | } 111 | log.Infof("Result written (%d bytes)", n) 112 | } 113 | } 114 | 115 | func loadTask(taskFileName string) (models.Task, error) { 116 | taskFile, err := os.Open(taskFileName) 117 | if err != nil { 118 | return models.Task{}, fmt.Errorf("open task file: %w", err) 119 | } 120 | defer taskFile.Close() 121 | 122 | fileContents, err := io.ReadAll(taskFile) 123 | if err != nil { 124 | return models.Task{}, fmt.Errorf("read file: %w", err) 125 | } 126 | 127 | var task models.Task 128 | if err := json.Unmarshal(fileContents, &task); err != nil { 129 | return models.Task{}, fmt.Errorf("unmarshal task: %w", err) 130 | } 131 | 132 | return task, err 133 | } 134 | -------------------------------------------------------------------------------- /cmd/webserver/ipranges.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | var IpRanges = []string{ 4 | // Cloudflare: 5 | "173.245.48.0/20", 6 | "103.21.244.0/22", 7 | "103.22.200.0/22", 8 | "103.31.4.0/22", 9 | "141.101.64.0/18", 10 | "108.162.192.0/18", 11 | "190.93.240.0/20", 12 | "188.114.96.0/20", 13 | "197.234.240.0/22", 14 | "198.41.128.0/17", 15 | "162.158.0.0/15", 16 | "104.16.0.0/13", 17 | "104.24.0.0/14", 18 | "172.64.0.0/13", 19 | "131.0.72.0/22", 20 | } 21 | -------------------------------------------------------------------------------- /cmd/webserver/webserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | wizard_vue "github.com/egor3f/rssalchemy/frontend/wizard-vue" 7 | "github.com/egor3f/rssalchemy/internal/adapters/natsadapter" 8 | httpApi "github.com/egor3f/rssalchemy/internal/api/http" 9 | "github.com/egor3f/rssalchemy/internal/config" 10 | "github.com/labstack/echo/v4" 11 | "github.com/labstack/echo/v4/middleware" 12 | "github.com/labstack/gommon/log" 13 | "github.com/nats-io/nats.go" 14 | "golang.org/x/time/rate" 15 | "net" 16 | "net/http" 17 | "os" 18 | "os/signal" 19 | "slices" 20 | "time" 21 | ) 22 | 23 | func main() { 24 | cfg, err := config.Read() 25 | if err != nil { 26 | log.Panicf("reading config failed: %v", err) 27 | } 28 | 29 | log.SetHeader(`${time_rfc3339_nano} ${level}`) 30 | if cfg.Debug { 31 | log.SetLevel(log.DEBUG) 32 | } 33 | 34 | baseCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt) 35 | defer stop() 36 | 37 | natsc, err := nats.Connect(cfg.NatsUrl) 38 | if err != nil { 39 | log.Panicf("nats connect failed: %v", err) 40 | } 41 | defer func() { 42 | if err := natsc.Drain(); err != nil { 43 | log.Errorf("nats drain failed: %v", err) 44 | } 45 | }() 46 | 47 | na, err := natsadapter.New(natsc, "RENDER_TASKS") 48 | if err != nil { 49 | log.Panicf("create nats adapter: %v", err) 50 | } 51 | 52 | e := echo.New() 53 | e.Use(middleware.Logger()) 54 | if !cfg.Debug { 55 | e.Use(middleware.Recover()) 56 | } 57 | 58 | setIPExtractor(e, cfg) 59 | 60 | cacheGroup := e.Group("", addCacheControlHeader(1*time.Hour)) 61 | cacheGroup.Use(middleware.StaticWithConfig(middleware.StaticConfig{ 62 | Root: wizard_vue.FSPrefix, 63 | Filesystem: http.FS(wizard_vue.EmbedFS), 64 | })) 65 | 66 | apiHandler := httpApi.New( 67 | na, 68 | na, 69 | rate.Every(time.Duration(float64(time.Second)*cfg.TaskRateLimitEvery)), 70 | cfg.TaskRateLimitBurst, 71 | cfg.Debug, 72 | ) 73 | apiHandler.SetupRoutes(e.Group("/api/v1")) 74 | 75 | go func() { 76 | if err := e.Start(cfg.WebserverAddress); err != nil && err != http.ErrServerClosed { 77 | e.Logger.Errorf("http server error, shutting down: %v", err) 78 | } 79 | }() 80 | <-baseCtx.Done() 81 | log.Infof("stopping webserver gracefully") 82 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 83 | defer cancel() 84 | if err := e.Shutdown(ctx); err != nil { 85 | e.Logger.Errorf("failed to shutdown server: %v", err) 86 | } 87 | } 88 | 89 | func addCacheControlHeader(ttl time.Duration) echo.MiddlewareFunc { 90 | return func(next echo.HandlerFunc) echo.HandlerFunc { 91 | return func(c echo.Context) error { 92 | c.Response().Header().Set( 93 | echo.HeaderCacheControl, 94 | fmt.Sprintf("public, max-age=%d", int(ttl.Seconds())), 95 | ) 96 | return next(c) 97 | } 98 | } 99 | } 100 | 101 | func setIPExtractor(e *echo.Echo, cfg config.Config) { 102 | if len(cfg.RealIpHeader) > 0 { 103 | // Real ip header 104 | e.IPExtractor = func(req *http.Request) string { 105 | if len(req.Header.Get(cfg.RealIpHeader)) > 0 { 106 | return req.Header.Get(cfg.RealIpHeader) 107 | } 108 | // fallback 109 | ra, _, _ := net.SplitHostPort(req.RemoteAddr) 110 | return ra 111 | } 112 | } else { 113 | // X-Forwarded-For with trusted ip ranges 114 | var trustOptions []echo.TrustOption 115 | for _, ipRange := range slices.Concat(IpRanges, cfg.TrustedIpRanges) { 116 | _, network, err := net.ParseCIDR(ipRange) 117 | if err != nil { 118 | log.Panicf("Invalid ip range: %s", ipRange) 119 | } 120 | trustOptions = append(trustOptions, echo.TrustIPRange(network)) 121 | } 122 | e.IPExtractor = echo.ExtractIPFromXFFHeader(trustOptions...) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /cmd/worker/worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/egor3f/rssalchemy/internal/adapters/natsadapter" 8 | "github.com/egor3f/rssalchemy/internal/config" 9 | natscookies "github.com/egor3f/rssalchemy/internal/cookiemgr/nats" 10 | "github.com/egor3f/rssalchemy/internal/dateparser" 11 | "github.com/egor3f/rssalchemy/internal/extractors/pwextractor" 12 | "github.com/egor3f/rssalchemy/internal/limiter/redisleaky" 13 | "github.com/egor3f/rssalchemy/internal/models" 14 | "github.com/labstack/gommon/log" 15 | "github.com/nats-io/nats.go" 16 | "github.com/redis/go-redis/v9" 17 | "golang.org/x/time/rate" 18 | "os" 19 | "os/signal" 20 | "time" 21 | ) 22 | 23 | func main() { 24 | cfg, err := config.Read() 25 | if err != nil { 26 | log.Panicf("reading config failed: %v", err) 27 | } 28 | 29 | log.SetHeader(`${time_rfc3339_nano} ${level}`) 30 | if cfg.Debug { 31 | log.SetLevel(log.DEBUG) 32 | } 33 | 34 | defer func() { 35 | log.Infof("worker gracefully stopped") 36 | }() 37 | 38 | baseCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt) 39 | defer stop() 40 | 41 | natsc, err := nats.Connect(cfg.NatsUrl) 42 | if err != nil { 43 | log.Panicf("nats connect failed: %v", err) 44 | } 45 | defer func() { 46 | if err := natsc.Drain(); err != nil { 47 | log.Errorf("nats drain failed: %v", err) 48 | } 49 | }() 50 | 51 | qc, err := natsadapter.New(natsc, "RENDER_TASKS") 52 | if err != nil { 53 | log.Panicf("create nats adapter: %v", err) 54 | } 55 | 56 | cookieManager, err := natscookies.New(natsc) 57 | if err != nil { 58 | log.Panicf("create cookie manager: %v", err) 59 | } 60 | 61 | redisClient := redis.NewClient(&redis.Options{ 62 | Addr: cfg.RedisUrl, 63 | }) 64 | defer func() { 65 | if err := redisClient.Close(); err != nil { 66 | log.Errorf("close redis client: %v", err) 67 | } 68 | }() 69 | if err := redisClient.Ping(baseCtx).Err(); err != nil { 70 | log.Panicf("redis ping: %v", err) 71 | } 72 | 73 | perDomainLimiter := redisleaky.New( 74 | rate.Every(time.Duration(float64(time.Second)*cfg.PerDomainRateLimitEvery)), 75 | int64(cfg.PerDomainRateLimitCapacity), 76 | redisClient, 77 | "per_domain_limiter", 78 | ) 79 | 80 | pwe, err := pwextractor.New(pwextractor.Config{ 81 | Proxy: cfg.Proxy, 82 | DateParser: &dateparser.DateParser{ 83 | CurrentTimeFunc: time.Now, 84 | }, 85 | CookieManager: cookieManager, 86 | Limiter: perDomainLimiter, 87 | }) 88 | if err != nil { 89 | log.Panicf("create pw extractor: %v", err) 90 | } 91 | defer func() { 92 | if err := pwe.Stop(); err != nil { 93 | log.Errorf("stop pw extractor: %v", err) 94 | } 95 | }() 96 | 97 | err = qc.ConsumeQueue(baseCtx, func(taskPayload []byte) (cacheKey string, resultPayoad []byte, errRet error) { 98 | var task models.Task 99 | if err := json.Unmarshal(taskPayload, &task); err != nil { 100 | errRet = fmt.Errorf("unmarshal task: %w", err) 101 | return 102 | } 103 | var result any 104 | switch task.TaskType { 105 | case models.TaskTypeExtract: 106 | result, err = pwe.Extract(task) 107 | case models.TaskTypePageScreenshot: 108 | result, err = pwe.Screenshot(task) 109 | } 110 | if err != nil { 111 | errRet = fmt.Errorf("task processing: %w", err) 112 | return 113 | } 114 | resultPayoad, err = json.Marshal(result) 115 | if err != nil { 116 | errRet = fmt.Errorf("marshal result: %w", err) 117 | return 118 | } 119 | return task.CacheKey(), resultPayoad, errRet 120 | }) 121 | if err != nil { 122 | log.Panicf("consume queue: %v", err) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /deploy/.env: -------------------------------------------------------------------------------- 1 | WEBSERVER_ADDRESS=0.0.0.0:8080 2 | NATS_URL=nats://nats:4222 3 | REDIS_URL=redis:6379 4 | DEBUG=false 5 | -------------------------------------------------------------------------------- /deploy/Dockerfile_webserver: -------------------------------------------------------------------------------- 1 | FROM node:20 AS frontend 2 | 3 | WORKDIR /buildfront 4 | COPY frontend/wizard-vue/package.json frontend/wizard-vue/package-lock.json ./ 5 | RUN npm install 6 | COPY frontend/wizard-vue ./ 7 | RUN npm run build 8 | 9 | FROM golang:1.23 10 | 11 | WORKDIR /app 12 | 13 | COPY go.mod go.sum ./ 14 | RUN go mod download 15 | 16 | COPY . . 17 | COPY --from=frontend /buildfront/dist ./frontend/wizard-vue/dist 18 | RUN go build -o bin/webserver github.com/egor3f/rssalchemy/cmd/webserver 19 | 20 | EXPOSE 8080 21 | CMD ["/app/bin/webserver"] 22 | -------------------------------------------------------------------------------- /deploy/Dockerfile_worker: -------------------------------------------------------------------------------- 1 | # todo: multi-stage build 2 | # todo: let playwright install all deps by itself 3 | 4 | FROM golang:1.23 5 | 6 | RUN apt-get update && apt-get install -y ca-certificates tzdata libasound2 libatk-bridge2.0-0 libatk1.0-0 \ 7 | libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libdrm2 libgbm1 libglib2.0-0 libnspr4 libnss3 \ 8 | libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 \ 9 | libxrandr2 xvfb fonts-noto-color-emoji fonts-unifont libfontconfig1 libfreetype6 xfonts-scalable \ 10 | fonts-liberation fonts-ipafont-gothic fonts-wqy-zenhei fonts-tlwg-loma-otf fonts-freefont-ttf && \ 11 | rm -rf /var/lib/apt/lists/* 12 | 13 | RUN useradd -ms /bin/bash pwuser 14 | WORKDIR /app 15 | 16 | COPY go.mod go.sum ./ 17 | RUN go mod download 18 | 19 | RUN PWGO_VER=$(grep -oE "playwright-go v\S+" go.mod | sed 's/playwright-go //g') \ 20 | && go install github.com/playwright-community/playwright-go/cmd/playwright@${PWGO_VER} 21 | RUN playwright install --with-deps --no-shell chromium && \ 22 | mkdir /home/pwuser/.cache && \ 23 | mv /root/.cache/ms-playwright* /home/pwuser/.cache/ && \ 24 | chown -R pwuser:pwuser /home/pwuser/.cache && \ 25 | rm -rf /var/lib/apt/lists/* 26 | 27 | COPY . . 28 | RUN go build -o bin/worker github.com/egor3f/rssalchemy/cmd/worker 29 | 30 | CMD ["/app/bin/worker"] 31 | -------------------------------------------------------------------------------- /deploy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | webserver: 3 | build: 4 | context: ../ 5 | dockerfile: deploy/Dockerfile_webserver 6 | env_file: .env 7 | depends_on: 8 | - nats 9 | ports: 10 | - "8080:8080" 11 | restart: unless-stopped 12 | 13 | worker: 14 | build: 15 | context: ../ 16 | dockerfile: deploy/Dockerfile_worker 17 | env_file: .env 18 | depends_on: 19 | - nats 20 | - redis 21 | ipc: host 22 | user: pwuser 23 | security_opt: 24 | - seccomp:seccomp_profile.json 25 | deploy: 26 | replicas: 1 27 | restart: unless-stopped 28 | 29 | nats: 30 | image: nats:2.10 31 | command: "-config /nats_config.conf" 32 | volumes: 33 | - ./nats_config.conf:/nats_config.conf:ro 34 | - natsdata:/data 35 | restart: unless-stopped 36 | 37 | redis: 38 | image: redis:7.4 39 | restart: unless-stopped 40 | 41 | volumes: 42 | natsdata: 43 | -------------------------------------------------------------------------------- /deploy/nats_config.conf: -------------------------------------------------------------------------------- 1 | jetstream: enabled 2 | max_payload: 4MB 3 | 4 | jetstream { 5 | store_dir: /data 6 | max_memory_store: 1GB 7 | max_file_store: 10GB 8 | } 9 | -------------------------------------------------------------------------------- /deploy/seccomp_profile.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultAction": "SCMP_ACT_ERRNO", 3 | "archMap": [ 4 | { 5 | "architecture": "SCMP_ARCH_X86_64", 6 | "subArchitectures": [ 7 | "SCMP_ARCH_X86", 8 | "SCMP_ARCH_X32" 9 | ] 10 | }, 11 | { 12 | "architecture": "SCMP_ARCH_AARCH64", 13 | "subArchitectures": [ 14 | "SCMP_ARCH_ARM" 15 | ] 16 | }, 17 | { 18 | "architecture": "SCMP_ARCH_MIPS64", 19 | "subArchitectures": [ 20 | "SCMP_ARCH_MIPS", 21 | "SCMP_ARCH_MIPS64N32" 22 | ] 23 | }, 24 | { 25 | "architecture": "SCMP_ARCH_MIPS64N32", 26 | "subArchitectures": [ 27 | "SCMP_ARCH_MIPS", 28 | "SCMP_ARCH_MIPS64" 29 | ] 30 | }, 31 | { 32 | "architecture": "SCMP_ARCH_MIPSEL64", 33 | "subArchitectures": [ 34 | "SCMP_ARCH_MIPSEL", 35 | "SCMP_ARCH_MIPSEL64N32" 36 | ] 37 | }, 38 | { 39 | "architecture": "SCMP_ARCH_MIPSEL64N32", 40 | "subArchitectures": [ 41 | "SCMP_ARCH_MIPSEL", 42 | "SCMP_ARCH_MIPSEL64" 43 | ] 44 | }, 45 | { 46 | "architecture": "SCMP_ARCH_S390X", 47 | "subArchitectures": [ 48 | "SCMP_ARCH_S390" 49 | ] 50 | } 51 | ], 52 | "syscalls": [ 53 | { 54 | "comment": "Allow create user namespaces", 55 | "names": [ 56 | "clone", 57 | "setns", 58 | "unshare" 59 | ], 60 | "action": "SCMP_ACT_ALLOW", 61 | "args": [], 62 | "includes": {}, 63 | "excludes": {} 64 | }, 65 | { 66 | "names": [ 67 | "accept", 68 | "accept4", 69 | "access", 70 | "adjtimex", 71 | "alarm", 72 | "bind", 73 | "brk", 74 | "capget", 75 | "capset", 76 | "chdir", 77 | "chmod", 78 | "chown", 79 | "chown32", 80 | "clock_adjtime", 81 | "clock_adjtime64", 82 | "clock_getres", 83 | "clock_getres_time64", 84 | "clock_gettime", 85 | "clock_gettime64", 86 | "clock_nanosleep", 87 | "clock_nanosleep_time64", 88 | "close", 89 | "connect", 90 | "copy_file_range", 91 | "creat", 92 | "dup", 93 | "dup2", 94 | "dup3", 95 | "epoll_create", 96 | "epoll_create1", 97 | "epoll_ctl", 98 | "epoll_ctl_old", 99 | "epoll_pwait", 100 | "epoll_wait", 101 | "epoll_wait_old", 102 | "eventfd", 103 | "eventfd2", 104 | "execve", 105 | "execveat", 106 | "exit", 107 | "exit_group", 108 | "faccessat", 109 | "fadvise64", 110 | "fadvise64_64", 111 | "fallocate", 112 | "fanotify_mark", 113 | "fchdir", 114 | "fchmod", 115 | "fchmodat", 116 | "fchown", 117 | "fchown32", 118 | "fchownat", 119 | "fcntl", 120 | "fcntl64", 121 | "fdatasync", 122 | "fgetxattr", 123 | "flistxattr", 124 | "flock", 125 | "fork", 126 | "fremovexattr", 127 | "fsetxattr", 128 | "fstat", 129 | "fstat64", 130 | "fstatat64", 131 | "fstatfs", 132 | "fstatfs64", 133 | "fsync", 134 | "ftruncate", 135 | "ftruncate64", 136 | "futex", 137 | "futex_time64", 138 | "futimesat", 139 | "getcpu", 140 | "getcwd", 141 | "getdents", 142 | "getdents64", 143 | "getegid", 144 | "getegid32", 145 | "geteuid", 146 | "geteuid32", 147 | "getgid", 148 | "getgid32", 149 | "getgroups", 150 | "getgroups32", 151 | "getitimer", 152 | "getpeername", 153 | "getpgid", 154 | "getpgrp", 155 | "getpid", 156 | "getppid", 157 | "getpriority", 158 | "getrandom", 159 | "getresgid", 160 | "getresgid32", 161 | "getresuid", 162 | "getresuid32", 163 | "getrlimit", 164 | "get_robust_list", 165 | "getrusage", 166 | "getsid", 167 | "getsockname", 168 | "getsockopt", 169 | "get_thread_area", 170 | "gettid", 171 | "gettimeofday", 172 | "getuid", 173 | "getuid32", 174 | "getxattr", 175 | "inotify_add_watch", 176 | "inotify_init", 177 | "inotify_init1", 178 | "inotify_rm_watch", 179 | "io_cancel", 180 | "ioctl", 181 | "io_destroy", 182 | "io_getevents", 183 | "io_pgetevents", 184 | "io_pgetevents_time64", 185 | "ioprio_get", 186 | "ioprio_set", 187 | "io_setup", 188 | "io_submit", 189 | "io_uring_enter", 190 | "io_uring_register", 191 | "io_uring_setup", 192 | "ipc", 193 | "kill", 194 | "lchown", 195 | "lchown32", 196 | "lgetxattr", 197 | "link", 198 | "linkat", 199 | "listen", 200 | "listxattr", 201 | "llistxattr", 202 | "_llseek", 203 | "lremovexattr", 204 | "lseek", 205 | "lsetxattr", 206 | "lstat", 207 | "lstat64", 208 | "madvise", 209 | "membarrier", 210 | "memfd_create", 211 | "mincore", 212 | "mkdir", 213 | "mkdirat", 214 | "mknod", 215 | "mknodat", 216 | "mlock", 217 | "mlock2", 218 | "mlockall", 219 | "mmap", 220 | "mmap2", 221 | "mprotect", 222 | "mq_getsetattr", 223 | "mq_notify", 224 | "mq_open", 225 | "mq_timedreceive", 226 | "mq_timedreceive_time64", 227 | "mq_timedsend", 228 | "mq_timedsend_time64", 229 | "mq_unlink", 230 | "mremap", 231 | "msgctl", 232 | "msgget", 233 | "msgrcv", 234 | "msgsnd", 235 | "msync", 236 | "munlock", 237 | "munlockall", 238 | "munmap", 239 | "nanosleep", 240 | "newfstatat", 241 | "_newselect", 242 | "open", 243 | "openat", 244 | "pause", 245 | "pipe", 246 | "pipe2", 247 | "poll", 248 | "ppoll", 249 | "ppoll_time64", 250 | "prctl", 251 | "pread64", 252 | "preadv", 253 | "preadv2", 254 | "prlimit64", 255 | "pselect6", 256 | "pselect6_time64", 257 | "pwrite64", 258 | "pwritev", 259 | "pwritev2", 260 | "read", 261 | "readahead", 262 | "readlink", 263 | "readlinkat", 264 | "readv", 265 | "recv", 266 | "recvfrom", 267 | "recvmmsg", 268 | "recvmmsg_time64", 269 | "recvmsg", 270 | "remap_file_pages", 271 | "removexattr", 272 | "rename", 273 | "renameat", 274 | "renameat2", 275 | "restart_syscall", 276 | "rmdir", 277 | "rseq", 278 | "rt_sigaction", 279 | "rt_sigpending", 280 | "rt_sigprocmask", 281 | "rt_sigqueueinfo", 282 | "rt_sigreturn", 283 | "rt_sigsuspend", 284 | "rt_sigtimedwait", 285 | "rt_sigtimedwait_time64", 286 | "rt_tgsigqueueinfo", 287 | "sched_getaffinity", 288 | "sched_getattr", 289 | "sched_getparam", 290 | "sched_get_priority_max", 291 | "sched_get_priority_min", 292 | "sched_getscheduler", 293 | "sched_rr_get_interval", 294 | "sched_rr_get_interval_time64", 295 | "sched_setaffinity", 296 | "sched_setattr", 297 | "sched_setparam", 298 | "sched_setscheduler", 299 | "sched_yield", 300 | "seccomp", 301 | "select", 302 | "semctl", 303 | "semget", 304 | "semop", 305 | "semtimedop", 306 | "semtimedop_time64", 307 | "send", 308 | "sendfile", 309 | "sendfile64", 310 | "sendmmsg", 311 | "sendmsg", 312 | "sendto", 313 | "setfsgid", 314 | "setfsgid32", 315 | "setfsuid", 316 | "setfsuid32", 317 | "setgid", 318 | "setgid32", 319 | "setgroups", 320 | "setgroups32", 321 | "setitimer", 322 | "setpgid", 323 | "setpriority", 324 | "setregid", 325 | "setregid32", 326 | "setresgid", 327 | "setresgid32", 328 | "setresuid", 329 | "setresuid32", 330 | "setreuid", 331 | "setreuid32", 332 | "setrlimit", 333 | "set_robust_list", 334 | "setsid", 335 | "setsockopt", 336 | "set_thread_area", 337 | "set_tid_address", 338 | "setuid", 339 | "setuid32", 340 | "setxattr", 341 | "shmat", 342 | "shmctl", 343 | "shmdt", 344 | "shmget", 345 | "shutdown", 346 | "sigaltstack", 347 | "signalfd", 348 | "signalfd4", 349 | "sigprocmask", 350 | "sigreturn", 351 | "socket", 352 | "socketcall", 353 | "socketpair", 354 | "splice", 355 | "stat", 356 | "stat64", 357 | "statfs", 358 | "statfs64", 359 | "statx", 360 | "symlink", 361 | "symlinkat", 362 | "sync", 363 | "sync_file_range", 364 | "syncfs", 365 | "sysinfo", 366 | "tee", 367 | "tgkill", 368 | "time", 369 | "timer_create", 370 | "timer_delete", 371 | "timer_getoverrun", 372 | "timer_gettime", 373 | "timer_gettime64", 374 | "timer_settime", 375 | "timer_settime64", 376 | "timerfd_create", 377 | "timerfd_gettime", 378 | "timerfd_gettime64", 379 | "timerfd_settime", 380 | "timerfd_settime64", 381 | "times", 382 | "tkill", 383 | "truncate", 384 | "truncate64", 385 | "ugetrlimit", 386 | "umask", 387 | "uname", 388 | "unlink", 389 | "unlinkat", 390 | "utime", 391 | "utimensat", 392 | "utimensat_time64", 393 | "utimes", 394 | "vfork", 395 | "vmsplice", 396 | "wait4", 397 | "waitid", 398 | "waitpid", 399 | "write", 400 | "writev" 401 | ], 402 | "action": "SCMP_ACT_ALLOW", 403 | "args": [], 404 | "comment": "", 405 | "includes": {}, 406 | "excludes": {} 407 | }, 408 | { 409 | "names": [ 410 | "ptrace" 411 | ], 412 | "action": "SCMP_ACT_ALLOW", 413 | "args": null, 414 | "comment": "", 415 | "includes": { 416 | "minKernel": "4.8" 417 | }, 418 | "excludes": {} 419 | }, 420 | { 421 | "names": [ 422 | "personality" 423 | ], 424 | "action": "SCMP_ACT_ALLOW", 425 | "args": [ 426 | { 427 | "index": 0, 428 | "value": 0, 429 | "valueTwo": 0, 430 | "op": "SCMP_CMP_EQ" 431 | } 432 | ], 433 | "comment": "", 434 | "includes": {}, 435 | "excludes": {} 436 | }, 437 | { 438 | "names": [ 439 | "personality" 440 | ], 441 | "action": "SCMP_ACT_ALLOW", 442 | "args": [ 443 | { 444 | "index": 0, 445 | "value": 8, 446 | "valueTwo": 0, 447 | "op": "SCMP_CMP_EQ" 448 | } 449 | ], 450 | "comment": "", 451 | "includes": {}, 452 | "excludes": {} 453 | }, 454 | { 455 | "names": [ 456 | "personality" 457 | ], 458 | "action": "SCMP_ACT_ALLOW", 459 | "args": [ 460 | { 461 | "index": 0, 462 | "value": 131072, 463 | "valueTwo": 0, 464 | "op": "SCMP_CMP_EQ" 465 | } 466 | ], 467 | "comment": "", 468 | "includes": {}, 469 | "excludes": {} 470 | }, 471 | { 472 | "names": [ 473 | "personality" 474 | ], 475 | "action": "SCMP_ACT_ALLOW", 476 | "args": [ 477 | { 478 | "index": 0, 479 | "value": 131080, 480 | "valueTwo": 0, 481 | "op": "SCMP_CMP_EQ" 482 | } 483 | ], 484 | "comment": "", 485 | "includes": {}, 486 | "excludes": {} 487 | }, 488 | { 489 | "names": [ 490 | "personality" 491 | ], 492 | "action": "SCMP_ACT_ALLOW", 493 | "args": [ 494 | { 495 | "index": 0, 496 | "value": 4294967295, 497 | "valueTwo": 0, 498 | "op": "SCMP_CMP_EQ" 499 | } 500 | ], 501 | "comment": "", 502 | "includes": {}, 503 | "excludes": {} 504 | }, 505 | { 506 | "names": [ 507 | "sync_file_range2" 508 | ], 509 | "action": "SCMP_ACT_ALLOW", 510 | "args": [], 511 | "comment": "", 512 | "includes": { 513 | "arches": [ 514 | "ppc64le" 515 | ] 516 | }, 517 | "excludes": {} 518 | }, 519 | { 520 | "names": [ 521 | "arm_fadvise64_64", 522 | "arm_sync_file_range", 523 | "sync_file_range2", 524 | "breakpoint", 525 | "cacheflush", 526 | "set_tls" 527 | ], 528 | "action": "SCMP_ACT_ALLOW", 529 | "args": [], 530 | "comment": "", 531 | "includes": { 532 | "arches": [ 533 | "arm", 534 | "arm64" 535 | ] 536 | }, 537 | "excludes": {} 538 | }, 539 | { 540 | "names": [ 541 | "arch_prctl" 542 | ], 543 | "action": "SCMP_ACT_ALLOW", 544 | "args": [], 545 | "comment": "", 546 | "includes": { 547 | "arches": [ 548 | "amd64", 549 | "x32" 550 | ] 551 | }, 552 | "excludes": {} 553 | }, 554 | { 555 | "names": [ 556 | "modify_ldt" 557 | ], 558 | "action": "SCMP_ACT_ALLOW", 559 | "args": [], 560 | "comment": "", 561 | "includes": { 562 | "arches": [ 563 | "amd64", 564 | "x32", 565 | "x86" 566 | ] 567 | }, 568 | "excludes": {} 569 | }, 570 | { 571 | "names": [ 572 | "s390_pci_mmio_read", 573 | "s390_pci_mmio_write", 574 | "s390_runtime_instr" 575 | ], 576 | "action": "SCMP_ACT_ALLOW", 577 | "args": [], 578 | "comment": "", 579 | "includes": { 580 | "arches": [ 581 | "s390", 582 | "s390x" 583 | ] 584 | }, 585 | "excludes": {} 586 | }, 587 | { 588 | "names": [ 589 | "open_by_handle_at" 590 | ], 591 | "action": "SCMP_ACT_ALLOW", 592 | "args": [], 593 | "comment": "", 594 | "includes": { 595 | "caps": [ 596 | "CAP_DAC_READ_SEARCH" 597 | ] 598 | }, 599 | "excludes": {} 600 | }, 601 | { 602 | "names": [ 603 | "bpf", 604 | "clone", 605 | "fanotify_init", 606 | "lookup_dcookie", 607 | "mount", 608 | "name_to_handle_at", 609 | "perf_event_open", 610 | "quotactl", 611 | "setdomainname", 612 | "sethostname", 613 | "setns", 614 | "syslog", 615 | "umount", 616 | "umount2", 617 | "unshare" 618 | ], 619 | "action": "SCMP_ACT_ALLOW", 620 | "args": [], 621 | "comment": "", 622 | "includes": { 623 | "caps": [ 624 | "CAP_SYS_ADMIN" 625 | ] 626 | }, 627 | "excludes": {} 628 | }, 629 | { 630 | "names": [ 631 | "clone" 632 | ], 633 | "action": "SCMP_ACT_ALLOW", 634 | "args": [ 635 | { 636 | "index": 0, 637 | "value": 2114060288, 638 | "valueTwo": 0, 639 | "op": "SCMP_CMP_MASKED_EQ" 640 | } 641 | ], 642 | "comment": "", 643 | "includes": {}, 644 | "excludes": { 645 | "caps": [ 646 | "CAP_SYS_ADMIN" 647 | ], 648 | "arches": [ 649 | "s390", 650 | "s390x" 651 | ] 652 | } 653 | }, 654 | { 655 | "names": [ 656 | "clone" 657 | ], 658 | "action": "SCMP_ACT_ALLOW", 659 | "args": [ 660 | { 661 | "index": 1, 662 | "value": 2114060288, 663 | "valueTwo": 0, 664 | "op": "SCMP_CMP_MASKED_EQ" 665 | } 666 | ], 667 | "comment": "s390 parameter ordering for clone is different", 668 | "includes": { 669 | "arches": [ 670 | "s390", 671 | "s390x" 672 | ] 673 | }, 674 | "excludes": { 675 | "caps": [ 676 | "CAP_SYS_ADMIN" 677 | ] 678 | } 679 | }, 680 | { 681 | "names": [ 682 | "reboot" 683 | ], 684 | "action": "SCMP_ACT_ALLOW", 685 | "args": [], 686 | "comment": "", 687 | "includes": { 688 | "caps": [ 689 | "CAP_SYS_BOOT" 690 | ] 691 | }, 692 | "excludes": {} 693 | }, 694 | { 695 | "names": [ 696 | "chroot" 697 | ], 698 | "action": "SCMP_ACT_ALLOW", 699 | "args": [], 700 | "comment": "", 701 | "includes": { 702 | "caps": [ 703 | "CAP_SYS_CHROOT" 704 | ] 705 | }, 706 | "excludes": {} 707 | }, 708 | { 709 | "names": [ 710 | "delete_module", 711 | "init_module", 712 | "finit_module" 713 | ], 714 | "action": "SCMP_ACT_ALLOW", 715 | "args": [], 716 | "comment": "", 717 | "includes": { 718 | "caps": [ 719 | "CAP_SYS_MODULE" 720 | ] 721 | }, 722 | "excludes": {} 723 | }, 724 | { 725 | "names": [ 726 | "acct" 727 | ], 728 | "action": "SCMP_ACT_ALLOW", 729 | "args": [], 730 | "comment": "", 731 | "includes": { 732 | "caps": [ 733 | "CAP_SYS_PACCT" 734 | ] 735 | }, 736 | "excludes": {} 737 | }, 738 | { 739 | "names": [ 740 | "kcmp", 741 | "process_vm_readv", 742 | "process_vm_writev", 743 | "ptrace" 744 | ], 745 | "action": "SCMP_ACT_ALLOW", 746 | "args": [], 747 | "comment": "", 748 | "includes": { 749 | "caps": [ 750 | "CAP_SYS_PTRACE" 751 | ] 752 | }, 753 | "excludes": {} 754 | }, 755 | { 756 | "names": [ 757 | "iopl", 758 | "ioperm" 759 | ], 760 | "action": "SCMP_ACT_ALLOW", 761 | "args": [], 762 | "comment": "", 763 | "includes": { 764 | "caps": [ 765 | "CAP_SYS_RAWIO" 766 | ] 767 | }, 768 | "excludes": {} 769 | }, 770 | { 771 | "names": [ 772 | "settimeofday", 773 | "stime", 774 | "clock_settime" 775 | ], 776 | "action": "SCMP_ACT_ALLOW", 777 | "args": [], 778 | "comment": "", 779 | "includes": { 780 | "caps": [ 781 | "CAP_SYS_TIME" 782 | ] 783 | }, 784 | "excludes": {} 785 | }, 786 | { 787 | "names": [ 788 | "vhangup" 789 | ], 790 | "action": "SCMP_ACT_ALLOW", 791 | "args": [], 792 | "comment": "", 793 | "includes": { 794 | "caps": [ 795 | "CAP_SYS_TTY_CONFIG" 796 | ] 797 | }, 798 | "excludes": {} 799 | }, 800 | { 801 | "names": [ 802 | "get_mempolicy", 803 | "mbind", 804 | "set_mempolicy" 805 | ], 806 | "action": "SCMP_ACT_ALLOW", 807 | "args": [], 808 | "comment": "", 809 | "includes": { 810 | "caps": [ 811 | "CAP_SYS_NICE" 812 | ] 813 | }, 814 | "excludes": {} 815 | }, 816 | { 817 | "names": [ 818 | "syslog" 819 | ], 820 | "action": "SCMP_ACT_ALLOW", 821 | "args": [], 822 | "comment": "", 823 | "includes": { 824 | "caps": [ 825 | "CAP_SYSLOG" 826 | ] 827 | }, 828 | "excludes": {} 829 | } 830 | ] 831 | } 832 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{js,jsx,mjs,cjs,ts,tsx,mts,cts,vue}] 2 | charset = utf-8 3 | indent_size = 2 4 | indent_style = space 5 | insert_final_newline = true 6 | trim_trailing_whitespace = true 7 | 8 | end_of_line = lf 9 | max_line_length = 100 10 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.env.development: -------------------------------------------------------------------------------- 1 | VITE_API_BASE=http://localhost:5000 2 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.env.production: -------------------------------------------------------------------------------- 1 | VITE_API_BASE= 2 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | .DS_Store 12 | dist 13 | dist-ssr 14 | coverage 15 | *.local 16 | 17 | /cypress/videos/ 18 | /cypress/screenshots/ 19 | 20 | # Editor directories and files 21 | .vscode/* 22 | !.vscode/extensions.json 23 | .idea 24 | *.suo 25 | *.ntvs* 26 | *.njsproj 27 | *.sln 28 | *.sw? 29 | 30 | *.tsbuildinfo 31 | -------------------------------------------------------------------------------- /frontend/wizard-vue/.prettierrc.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "$schema": "https://json.schemastore.org/prettierrc", 4 | "semi": false, 5 | "singleQuote": true, 6 | "printWidth": 100 7 | } 8 | -------------------------------------------------------------------------------- /frontend/wizard-vue/embed.go: -------------------------------------------------------------------------------- 1 | package wizard_vue 2 | 3 | import "embed" 4 | 5 | //go:embed dist 6 | var EmbedFS embed.FS 7 | 8 | const FSPrefix = "dist" 9 | -------------------------------------------------------------------------------- /frontend/wizard-vue/env.d.ts: -------------------------------------------------------------------------------- 1 | ///
8 |
12 |
9 | 
10 |
11 |
10 |
12 |
14 |
15 |
16 |
45 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/components/Copyable.vue:
--------------------------------------------------------------------------------
1 |
21 |
22 |
23 |
24 | {{ contents }}
25 | Copied!
26 |
28 |
29 |
30 |
61 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/components/EditUrlModal.vue:
--------------------------------------------------------------------------------
1 |
48 |
49 |
50 | 
6 |
7 |
8 |
9 |
21 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/components/Modal.vue:
--------------------------------------------------------------------------------
1 |
4 |
5 |
6 |
8 |
12 |
9 |
11 |
22 |
45 |
46 |
47 |
52 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/components/inputs/RadioButtons.vue:
--------------------------------------------------------------------------------
1 |
17 |
18 |
19 |
23 |
24 |
44 |
20 |
21 |
22 |
29 |
30 |
31 |
32 |
33 |
34 |
52 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/components/inputs/TextField.vue:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
22 |
23 |
27 |
28 |
29 |
47 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/main.ts:
--------------------------------------------------------------------------------
1 | import './assets/base.scss';
2 |
3 | import { createApp } from 'vue'
4 | import { createPinia } from 'pinia'
5 |
6 | import App from './App.vue'
7 | import router from './router'
8 |
9 | const app = createApp(App)
10 |
11 | app.use(createPinia())
12 | app.use(router)
13 |
14 | app.mount('#app')
15 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/pages/WizardPage.vue:
--------------------------------------------------------------------------------
1 |
59 |
60 |
61 |
24 |
25 |
26 |
62 | Screenshot
64 | Edit existing task / import preset
65 | Reset Form
66 |
72 |
73 |
74 |
90 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/router/index.ts:
--------------------------------------------------------------------------------
1 | import { createRouter, createWebHistory } from 'vue-router'
2 | import WizardPage from "@/pages/WizardPage.vue";
3 |
4 | const router = createRouter({
5 | history: createWebHistory(import.meta.env.BASE_URL),
6 | routes: [
7 | {
8 | path: '/',
9 | name: 'wizard',
10 | component: WizardPage,
11 | },
12 | ],
13 | })
14 |
15 | export default router
16 |
--------------------------------------------------------------------------------
/frontend/wizard-vue/src/stores/wizard.ts:
--------------------------------------------------------------------------------
1 | import {defineStore} from "pinia";
2 | import {defaultSpecs, type SpecField, fields, type Specs, type SpecValue} from "@/urlmaker/specs.ts";
3 | import {computed, reactive} from "vue";
4 | import {debounce} from "es-toolkit";
5 |
6 | const LOCAL_STORAGE_KEY = 'rssalchemy_store_wizard';
7 |
8 | export const useWizardStore = defineStore('wizard', () => {
9 |
10 | const locStorageContent = localStorage.getItem(LOCAL_STORAGE_KEY);
11 | const initialSpecs = locStorageContent ? JSON.parse(locStorageContent) as Specs : defaultSpecs;
12 |
13 | const specs = reactive(Object.assign({}, initialSpecs));
14 |
15 | const formValid = computed(() => {
16 | return fields.every(field => (
17 | !specs[field.name] && !(field as SpecField).required || field.validate(specs[field.name]!)
18 | ));
19 | });
20 |
21 | const updateLocalStorage = debounce(() => {
22 | localStorage.setItem(LOCAL_STORAGE_KEY, JSON.stringify(specs));
23 | }, 100);
24 |
25 | function updateSpecLink for RSS reader:
67 | Preset for sharing:
69 | " + paragraph + "
"; 8 | paragraph = ""; 9 | } 10 | 11 | const addImage = img => { 12 | let imgSrc = img.getAttribute('src'); 13 | if (imgSrc.startsWith('/')) { 14 | imgSrc = `${document.location.origin}/${imgSrc}`; 15 | } 16 | content += `