├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .dockerignore ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── Bug_report.md │ └── Feature_request.md ├── codeql │ └── codeql-config.yml ├── dependabot.yml ├── linters │ ├── .dockerfilelintrc │ ├── .golangci.yml │ ├── .hadolint.yaml │ └── .markdown-lint.yml └── workflows │ ├── analysis.yml │ ├── builder.yml │ ├── dispatch.yml │ ├── docker.yml │ ├── license.yml │ ├── linter.yml │ ├── pages.yml │ ├── release.yml │ ├── snapcraft.yml │ ├── stale.yml │ └── testing.yml ├── .gitignore ├── .gitmodules ├── .golangci.yml ├── .licenserc.yaml ├── .semgrepignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Procfile ├── README.md ├── app.json ├── bina.tpl.json ├── build ├── binary.sh ├── binary │ └── .gitkeep ├── debian │ ├── Dockerfile │ ├── builder │ ├── compat │ ├── control │ ├── copyright │ ├── rules │ ├── wayback.manpages │ └── wayback.postinst ├── docker │ ├── Dockerfile.all │ ├── Dockerfile.dev │ ├── Dockerfile.render │ └── README.md ├── flatpak │ └── org.wabarc.wayback.yml ├── package │ └── .gitkeep ├── redhat │ ├── Dockerfile │ ├── entrypoint.sh │ └── wayback.spec └── systemd │ └── wayback.service ├── cmd ├── playback │ └── main.go └── wayback │ ├── main.go │ ├── pprof.go │ ├── serve.go │ └── wayback.go ├── codecov.yml ├── config ├── config.go ├── config_test.go ├── doc.go ├── options.go ├── parser.go └── parser_test.go ├── cosign.pub ├── doc.go ├── docker-bake.hcl ├── docker-compose.yml ├── docs ├── assets │ ├── discord-server.png │ ├── irc.png │ ├── logo.png │ ├── mastodon.png │ ├── matrix-room.png │ ├── slack-channel.png │ ├── telegram-channel.png │ ├── telegram.png │ ├── wayback.drawio │ ├── wayback.svg │ ├── web.png │ └── xmpp.png ├── changelog.md ├── command-line.md ├── contributing.md ├── contributing.zh.md ├── deployment.md ├── deployment.zh.md ├── environment.md ├── index.md ├── index.zh.md ├── installation.md ├── installation.zh.md ├── integrations │ ├── archive-today.md │ ├── archive-today.zh.md │ ├── discord.md │ ├── discord.zh.md │ ├── github.md │ ├── github.zh.md │ ├── internet-archive.md │ ├── internet-archive.zh.md │ ├── ipfs.md │ ├── ipfs.zh.md │ ├── irc.md │ ├── irc.zh.md │ ├── mastodon.md │ ├── mastodon.zh.md │ ├── matrix.md │ ├── matrix.zh.md │ ├── meilisearch.md │ ├── meilisearch.zh.md │ ├── metrics.md │ ├── metrics.zh.md │ ├── nostr.md │ ├── nostr.zh.md │ ├── notion.md │ ├── notion.zh.md │ ├── omnivore.md │ ├── omnivore.zh.md │ ├── playback.md │ ├── playback.zh.md │ ├── slack.md │ ├── slack.zh.md │ ├── telegram.md │ ├── telegram.zh.md │ ├── telegraph.md │ ├── telegraph.zh.md │ ├── twitter.md │ ├── twitter.zh.md │ ├── web.md │ ├── web.zh.md │ ├── xmpp.md │ └── xmpp.zh.md ├── privacy.md ├── privacy.zh.md ├── resources.md ├── resources.zh.md ├── service.md ├── service.zh.md ├── troubleshooting.md └── troubleshooting.zh.md ├── entity ├── doc.go ├── entity.go └── playback.go ├── errors ├── doc.go └── errors.go ├── go.mod ├── go.sum ├── ingress ├── doc.go ├── http.go ├── init.go └── register │ ├── publish.go │ └── service.go ├── install.sh ├── metrics ├── doc.go ├── exports.go └── metrics.go ├── mkdocs.yml ├── pooling ├── doc.go ├── options.go ├── options_test.go ├── pooling.go └── pooling_test.go ├── publish ├── discord │ ├── discord.go │ ├── discord_test.go │ ├── doc.go │ └── setup.go ├── doc.go ├── example.go ├── github │ ├── doc.go │ ├── github.go │ ├── github_test.go │ └── setup.go ├── mastodon │ ├── doc.go │ ├── mastodon.go │ ├── mastodon_test.go │ └── setup.go ├── matrix │ ├── doc.go │ ├── matrix.go │ ├── matrix_test.go │ └── setup.go ├── meili │ ├── doc.go │ ├── meili.go │ ├── meili_test.go │ └── setup.go ├── module.go ├── module_test.go ├── nostr │ ├── doc.go │ ├── nostr.go │ ├── nostr_test.go │ └── setup.go ├── notion │ ├── doc.go │ ├── notion.go │ ├── notion_test.go │ └── setup.go ├── omnivore │ ├── doc.go │ ├── omnivore.go │ ├── omnivore_test.go │ └── setup.go ├── publish.go ├── publish_test.go ├── relaychat │ ├── doc.go │ ├── relaychat.go │ ├── relaychat_test.go │ └── setup.go ├── slack │ ├── doc.go │ ├── setup.go │ └── slack.go ├── telegram │ ├── doc.go │ ├── setup.go │ ├── telegram.go │ └── telegram_test.go ├── twitter │ ├── doc.go │ ├── setup.go │ ├── twitter.go │ └── twitter_test.go ├── utils.go └── utils_test.go ├── reduxer ├── doc.go ├── example.go ├── media.go ├── media_test.go ├── reduxer.go ├── reduxer_test.go ├── sites ├── video.go └── video_stub.go ├── render.yaml ├── renovate.json ├── requirements.txt ├── service ├── discord │ ├── discord.go │ ├── discord_test.go │ ├── doc.go │ └── setup.go ├── doc.go ├── httpd │ ├── doc.go │ ├── httpd.go │ ├── httpd_test.go │ ├── libtor.go │ ├── libtor_stub.go │ ├── onion.go │ ├── setup.go │ ├── web.go │ └── web_test.go ├── mastodon │ ├── doc.go │ ├── mastodon.go │ ├── mastodon_test.go │ └── setup.go ├── matrix │ ├── doc.go │ ├── matrix.go │ ├── matrix_test.go │ └── setup.go ├── module.go ├── module_test.go ├── options.go ├── options_test.go ├── relaychat │ ├── doc.go │ ├── relaychat.go │ ├── relaychat_test.go │ └── setup.go ├── service.go ├── service_test.go ├── slack │ ├── doc.go │ ├── setup.go │ ├── slack.go │ └── slack_test.go ├── telegram │ ├── doc.go │ ├── setup.go │ ├── telegram.go │ └── telegram_test.go ├── twitter │ ├── doc.go │ ├── setup.go │ ├── twitter.go │ └── twitter_test.go ├── utils.go ├── utils_test.go └── xmpp │ ├── doc.go │ ├── setup.go │ ├── xmpp.go │ └── xmpp_test.go ├── snapcraft.yaml ├── storage ├── doc.go ├── storage.go ├── storage_test.go ├── telegram.go └── telegram_test.go ├── systemd ├── doc.go ├── systemd.go ├── systemd_test.go └── systemd_windows.go ├── template ├── assets │ ├── image │ │ ├── favicon-16.png │ │ ├── favicon-32.png │ │ ├── favicon.ico │ │ ├── icon-120.png │ │ ├── icon-128.png │ │ ├── icon-152.png │ │ ├── icon-167.png │ │ ├── icon-180.png │ │ ├── icon-192.png │ │ ├── icon-512.png │ │ ├── search.svg │ │ └── send.svg │ └── js │ │ ├── index.js │ │ └── service-worker.js ├── doc.go ├── render │ ├── discord.go │ ├── discord_test.go │ ├── doc.go │ ├── github.go │ ├── github_test.go │ ├── mastodon.go │ ├── mastodon_test.go │ ├── matrix.go │ ├── matrix_test.go │ ├── nostr.go │ ├── nostr_test.go │ ├── notion.go │ ├── notion_test.go │ ├── relaychat.go │ ├── relaychat_test.go │ ├── render.go │ ├── render_test.go │ ├── slack.go │ ├── slack_test.go │ ├── telegram.go │ ├── telegram_test.go │ ├── twitter.go │ ├── twitter_test.go │ ├── xmpp.go │ └── xmpp_test.go ├── template.go └── views │ ├── layout.html │ └── offline.html ├── version ├── doc.go └── version.go ├── wayback.1 ├── wayback.conf └── wayback.go /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.195.0/containers/go/.devcontainer/base.Dockerfile 6 | # [Choice] Go version (use -bookworm variants on local arm64/Apple Silicon): 1, 1.16, 1.17, 1-bullseye, 1.16-bullseye, 1.17-bullseye, 1-buster, 1.16-buster, 1.17-buster 7 | ARG VARIANT=1-bookworm 8 | FROM mcr.microsoft.com/devcontainers/go:dev-${VARIANT} 9 | 10 | # [Choice] Node.js version: lts/*, 16, 14, 12, 10 11 | # ARG NODE_VERSION="lts/*" 12 | # RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c ". /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi 13 | 14 | # [Optional] Uncomment this section to install additional OS packages. 15 | RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 16 | && apt-get -y install --no-install-recommends \ 17 | git \ 18 | vim \ 19 | tor \ 20 | ffmpeg \ 21 | chromium \ 22 | youtube-dl \ 23 | webp \ 24 | fonts-freefont-ttf \ 25 | fonts-font-awesome \ 26 | fonts-noto \ 27 | fonts-noto-core \ 28 | fonts-noto-cjk \ 29 | fonts-noto-extra 30 | 31 | # [Optional] Uncomment the next line to use go get to install anything else you need 32 | # RUN go get -x 33 | 34 | # [Optional] Uncomment this line to install global node packages. 35 | # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 36 | RUN su vscode -c "curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.60.3" 37 | 38 | # Change directory permission 39 | RUN chown -R vscode:golang /go/pkg 40 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | #.git 6 | .github/ 7 | .semgrepignore 8 | build/binary 9 | #docs required by rpm build 10 | #docs/ 11 | install.sh 12 | mkdocs.yml 13 | snapcraft.yaml 14 | requirements.txt 15 | renovate.json 16 | codecov.yml 17 | cosign.pub 18 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Go 2 | *.go text eol=lf 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐛 Bug Report 3 | about: If something isn't working as expected 🤔. 4 | --- 5 | 6 | ## Bug Report 7 | 8 | **Current Behavior** 9 | A clear and concise description of the behavior. 10 | 11 | ```shell 12 | // Your error message 13 | ``` 14 | 15 | **Expected behavior/code** 16 | A clear and concise description of what you expected to happen (or code). 17 | 18 | **Environment** 19 | 20 | - Wayback version(s): [e.g. v0.8.0] 21 | - Golang version: [e.g. Go 1.16] 22 | - OS: [e.g. OSX 10.13.4, Windows 10] 23 | 24 | **Possible Solution** 25 | 26 | 27 | 28 | **Additional context/Screenshots** 29 | Add any other context about the problem here. If applicable, add screenshots to help explain. 30 | 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🚀 Feature Request 3 | about: I have a suggestion (and may want to implement it 🙂)! 4 | --- 5 | 6 | ## Feature Request 7 | 8 | **Is your feature request related to a problem? Please describe.** 9 | A clear and concise description of what the problem is. Ex. I have an issue when [...] 10 | 11 | **Describe the solution you'd like** 12 | A clear and concise description of what you want to happen. Add any considered drawbacks. 13 | 14 | **Describe alternatives you've considered** 15 | A clear and concise description of any alternative solutions or features you've considered. 16 | 17 | **Teachability, Documentation, Adoption, Migration Strategy** 18 | If you can, explain how users will be able to use this and possibly write out a version the docs. 19 | Maybe a screenshot or design? 20 | -------------------------------------------------------------------------------- /.github/codeql/codeql-config.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: "CodeQL config" 6 | queries: 7 | # Run all extra query suites, both because we want to 8 | # and because it'll act as extra testing. This is why 9 | # we include both even though one is a superset of the 10 | # other, because we're testing the parsing logic and 11 | # that the suites exist in the codeql bundle. 12 | - uses: security-extended 13 | - uses: security-and-quality 14 | paths-ignore: 15 | - tests 16 | - lib 17 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | # To get started with Dependabot version updates, you'll need to specify which 6 | # package ecosystems to update and where the package manifests are located. 7 | # Please see the documentation for all configuration options: 8 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 9 | 10 | version: 2 11 | updates: 12 | - package-ecosystem: "gomod" 13 | directory: "/" 14 | schedule: 15 | interval: "daily" 16 | open-pull-requests-limit: 10 17 | 18 | - package-ecosystem: "github-actions" 19 | directory: "/" 20 | schedule: 21 | interval: "daily" 22 | 23 | -------------------------------------------------------------------------------- /.github/linters/.dockerfilelintrc: -------------------------------------------------------------------------------- 1 | rules: 2 | missing_tag: off 3 | -------------------------------------------------------------------------------- /.github/linters/.golangci.yml: -------------------------------------------------------------------------------- 1 | ../../.golangci.yml -------------------------------------------------------------------------------- /.github/linters/.hadolint.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | --- 6 | ########################## 7 | ## Hadolint config file ## 8 | ########################## 9 | ignored: 10 | -------------------------------------------------------------------------------- /.github/linters/.markdown-lint.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | --- 6 | ########################### 7 | ########################### 8 | ## Markdown Linter rules ## 9 | ########################### 10 | ########################### 11 | 12 | # Linter rules doc: 13 | # - https://github.com/DavidAnson/markdownlint 14 | # 15 | # Note: 16 | # To comment out a single error: 17 | # 18 | # any violations you want 19 | # 20 | # 21 | 22 | ############### 23 | # Rules by id # 24 | ############### 25 | MD004: false # Unordered list style 26 | MD007: 27 | indent: 2 # Unordered list indentation 28 | MD013: 29 | line_length: 400 # Line length 80 is far to short 30 | MD026: 31 | punctuation: ".,;:!。，；:" # List of not allowed 32 | MD029: false # Ordered list item prefix 33 | MD033: false # Allow inline HTML 34 | MD034: false # Allow Bare URL 35 | MD036: false # Emphasis used instead of a heading 36 | MD041: false # Allow top-level heading first line 37 | 38 | ################# 39 | # Rules by tags # 40 | ################# 41 | blank_lines: false # Error on blank lines 42 | -------------------------------------------------------------------------------- /.github/workflows/dispatch.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Dispatch 6 | 7 | on: 8 | release: 9 | types: [published] 10 | 11 | permissions: {} 12 | 13 | jobs: 14 | release-brew: 15 | if: github.repository == 'wabarc/wayback' 16 | name: Repository Dispatch 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Harden Runner 20 | uses: step-security/harden-runner@2e205a28d0e1da00c5f53b161f4067b052c61f34 # v1.5.0 21 | with: 22 | egress-policy: block 23 | disable-telemetry: true 24 | allowed-endpoints: > 25 | github.com:443 26 | api.github.com:443 27 | 28 | - name: Dispatch repository in wabarc/aur 29 | uses: peter-evans/repository-dispatch@f2696244ec00ed5c659a5cc77f7138ad0302dffb # v2.1.0 30 | with: 31 | repository: wabarc/homebrew-wayback 32 | event-type: publish 33 | token: ${{ secrets.PAT_WORKFLOW }} 34 | client-payload: '{"from": "${{ github.repository }}", "channel": "stable"}' 35 | -------------------------------------------------------------------------------- /.github/workflows/license.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: License 6 | 7 | on: 8 | push: 9 | branches: 10 | - '**' 11 | paths-ignore: 12 | - 'docs/**' 13 | - 'mkdocs.yml' 14 | pull_request: 15 | branches: 16 | - '**' 17 | types: [ opened, synchronize, reopened ] 18 | paths-ignore: 19 | - 'docs/**' 20 | - 'mkdocs.yml' 21 | 22 | permissions: 23 | contents: read 24 | 25 | jobs: 26 | license: 27 | name: License Checker 28 | uses: wabarc/.github/.github/workflows/reusable-license.yml@main 29 | with: 30 | egress-policy: audit 31 | -------------------------------------------------------------------------------- /.github/workflows/linter.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Linter 6 | 7 | on: 8 | push: 9 | branches: 10 | - '**' 11 | pull_request: 12 | branches: 13 | - '**' 14 | types: [ opened, synchronize, reopened ] 15 | 16 | permissions: 17 | contents: read 18 | 19 | # New runs to only cancel in-progress runs of the same workflow. 20 | concurrency: 21 | group: ${{ github.workflow }}-${{ github.ref }} 22 | cancel-in-progress: true 23 | 24 | jobs: 25 | super-linter: 26 | name: Super Linter 27 | uses: wabarc/.github/.github/workflows/reusable-super-linter.yml@main 28 | with: 29 | filter-regex-exclude: 'install.sh' 30 | permissions: 31 | contents: read 32 | packages: read 33 | statuses: write 34 | 35 | golangci: 36 | name: golangci-lint 37 | uses: wabarc/.github/.github/workflows/reusable-golangci.yml@main 38 | with: 39 | egress-policy: audit 40 | 41 | shellcheck: 42 | name: ShellCheck 43 | uses: wabarc/.github/.github/workflows/reusable-shellcheck.yml@main 44 | 45 | misspell: 46 | name: Misspell 47 | uses: wabarc/.github/.github/workflows/reusable-misspell.yml@main 48 | 49 | alex: 50 | name: Alex 51 | uses: wabarc/.github/.github/workflows/reusable-alex.yml@main 52 | 53 | urlcheck: 54 | name: URLCheck 55 | uses: wabarc/.github/.github/workflows/reusable-urlcheck.yml@main 56 | with: 57 | exclude-patterns: ${{ vars.URLCHECH_EXCLUDE }} 58 | 59 | goreportcard: 60 | name: Go Report Card 61 | uses: wabarc/.github/.github/workflows/reusable-goreportcard.yml@main 62 | -------------------------------------------------------------------------------- /.github/workflows/pages.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Pages 6 | 7 | on: 8 | push: 9 | branches: 10 | - main 11 | paths: 12 | - docs/** 13 | - mkdocs.yml 14 | - retuirements.txt 15 | - .github/workflows/mkdocs.yml 16 | workflow_dispatch: 17 | 18 | concurrency: 19 | group: 'pages' 20 | cancel-in-progress: true 21 | 22 | permissions: 23 | contents: read 24 | 25 | jobs: 26 | deploy: 27 | name: Generate Documents 28 | runs-on: ubuntu-latest 29 | permissions: 30 | pages: write # to deploy to Pages 31 | id-token: write # to verify the deployment originates from an appropriate source 32 | environment: 33 | name: github-pages 34 | url: ${{ steps.deployment.outputs.page_url }} 35 | steps: 36 | - name: Harden Runner 37 | uses: step-security/harden-runner@c8454efe5d0bdefd25384362fe217428ca277d57 # v2.2.0 38 | with: 39 | egress-policy: audit 40 | disable-telemetry: true 41 | allowed-endpoints: > 42 | github.com:443 43 | api.github.com:443 44 | pypi.org:443 45 | pip.pypa.io:443 46 | files.pythonhosted.org:443 47 | 48 | - name: Check out code base 49 | uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 50 | with: 51 | fetch-depth: 0 52 | persist-credentials: false 53 | 54 | - name: Setup Python 55 | uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 56 | with: 57 | python-version: '3.x' 58 | 59 | - name: Install Requirements 60 | run: | 61 | pip install -r requirements.txt 62 | 63 | - name: Generate Documents 64 | run: | 65 | mkdocs build 66 | 67 | - name: Upload Pages 68 | uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 69 | with: 70 | path: 'site' 71 | 72 | - name: Deployment 73 | uses: actions/deploy-pages@decdde0ac072f6dcbe43649d82d9c635fff5b4e4 # v4.0.4 74 | id: deployment 75 | -------------------------------------------------------------------------------- /.github/workflows/snapcraft.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Snapcraft 6 | 7 | on: 8 | push: 9 | tags: 10 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 11 | workflow_dispatch: 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | snapcraft: 18 | uses: wabarc/.github/.github/workflows/reusable-builder-snap.yml@main 19 | with: 20 | product: wayback 21 | channel: stable 22 | publish: true 23 | release: true 24 | secrets: 25 | wayback-ipfs-target: ${{ secrets.WAYBACK_IPFS_TARGET }} 26 | wayback-ipfs-apikey: ${{ secrets.WAYBACK_IPFS_APIKEY }} 27 | wayback-ipfs-secret: ${{ secrets.WAYBACK_IPFS_SECRET }} 28 | snapcraft-token: ${{ secrets.SNAPCRAFT_TOKEN }} 29 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: Stale 6 | 7 | on: 8 | schedule: 9 | - cron: "0 3 * * 6" 10 | workflow_dispatch: 11 | 12 | permissions: 13 | issues: write 14 | pull-requests: write 15 | 16 | jobs: 17 | stale: 18 | name: Stale 19 | uses: wabarc/.github/.github/workflows/reusable-stale.yml@main 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | # Binaries for programs and plugins 6 | *.exe 7 | *.exe~ 8 | *.dll 9 | *.so 10 | *.swp 11 | *.dylib 12 | /run 13 | 14 | # Test binary, built with `go test -c` 15 | *.test 16 | 17 | # Output of the go coverage tool, specifically when used with LiteIDE 18 | *.out 19 | /coverage.* 20 | 21 | # Dependency directories (remove the comment below to include it) 22 | # vendor/ 23 | 24 | /bina.json 25 | /build/binary 26 | /build/package 27 | /.flatpak-builder 28 | 29 | # Ignore Tor embedded process data directory 30 | **/tor-data* 31 | **/data-dir* 32 | wayback.db 33 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | [submodule "build/aur"] 6 | path = build/aur 7 | url = https://github.com/wabarc/aur.git 8 | -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | header: 6 | license: 7 | spdx-id: GPL-3.0-or-later 8 | copyright-owner: Wayback Archiver 9 | software-name: wayback 10 | content: | 11 | Copyright 2020 Wayback Archiver. All rights reserved. 12 | Use of this source code is governed by the GNU GPL v3 13 | license that can be found in the LICENSE file. 14 | 15 | paths-ignore: 16 | - '.github/ISSUE_TEMPLATE' 17 | - '.github/PULL_REQUEST_TEMPLATE' 18 | - '.github/linters/.dockerfilelintrc' 19 | - '.gitattributes' 20 | - '.semgrepignore' 21 | - 'build/aur' 22 | - 'build/binary' 23 | - 'build/debian' 24 | - 'build/package' 25 | - 'build/docker/Dockerfile.*' 26 | - 'build/systemd/wayback.service' 27 | - 'docs/assets/*' 28 | - '**/*.md' 29 | - '**/*.out' 30 | - '**/*.json' 31 | - '**/go.mod' 32 | - '**/go.sum' 33 | - 'LICENSE' 34 | - 'reduxer/sites' 35 | - 'template/assets/**' 36 | - 'template/views/*.html' 37 | - 'wayback.1' 38 | - 'wayback.conf' 39 | - 'docker-bake.hcl' 40 | - 'install.sh' 41 | - 'Makefile' 42 | - 'Procfile' 43 | - 'cosign.pub' 44 | - 'codecov.yml' 45 | - 'mkdocs.yml' 46 | 47 | comment: on-failure 48 | 49 | dependency: 50 | files: 51 | - go.mod 52 | licenses: 53 | - name: github.com/multiformats/go-base36 54 | version: v0.2.0 55 | license: Apache-2.0 OR MIT 56 | - name: github.com/multiformats/go-multicodec 57 | version: v0.9.0 58 | license: Apache-2.0 OR MIT 59 | -------------------------------------------------------------------------------- /.semgrepignore: -------------------------------------------------------------------------------- 1 | # Common large paths 2 | template/views/ 3 | 4 | # Common test paths 5 | *_test.go 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | docs/changelog.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | docs/contributing.md -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: wayback -d web -c wayback.conf 2 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wayback", 3 | "description": "A toolkit for snapshot webpage to Internet Archive, archive.today, IPFS and beyond.", 4 | "keywords": [ 5 | "wayback" 6 | ], 7 | "website": "https://wabarc.eu.org/", 8 | "repository": "https://github.com/wabarc/wayback", 9 | "logo": "https://avatars.githubusercontent.com/u/66000339", 10 | "success_url": "/", 11 | "scripts": { 12 | "postdeploy": "wayback --version" 13 | }, 14 | "env": { 15 | "PORT": { 16 | "description": "Port for web service", 17 | "value": "8964" 18 | } 19 | }, 20 | "addons": [], 21 | "buildpacks": [ 22 | { 23 | "url": "https://github.com/heroku/heroku-buildpack-google-chrome.git" 24 | }, 25 | { 26 | "url": "https://github.com/tor-actions/heroku-buildpack-tor.git" 27 | }, 28 | { 29 | "url": "heroku/go" 30 | } 31 | ], 32 | "environments": { 33 | "test": { 34 | "scripts": { 35 | "test": "wayback -d web" 36 | } 37 | } 38 | }, 39 | "formation":{ 40 | "web":{ 41 | "quantity":1 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /build/binary.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2020 Wayback Archiver. All rights reserved. 3 | # Use of this source code is governed by the GNU GPL v3 4 | # license that can be found in the LICENSE file. 5 | # 6 | # 7 | # Perform package builder 8 | # 9 | set -eux 10 | 11 | GOOS=linux 12 | GOARCH=amd64 13 | 14 | for arg in "$@"; do 15 | case $arg in 16 | *arm/v6|*arm32v6) 17 | GOARCH=armv6 18 | ;; 19 | *armv7|*arm/v7|*arm32v7) 20 | GOARCH=armv7 21 | ;; 22 | *arm64|*arm64v8) 23 | GOARCH=armv8 24 | ;; 25 | *386) 26 | GOARCH=386 27 | ;; 28 | *ppc64le) 29 | GOARCH=ppc64le 30 | ;; 31 | *s390x) 32 | GOARCH=s390x 33 | ;; 34 | windows*) 35 | GOOS=windows 36 | ;; 37 | darwin*) 38 | GOOS=darwin 39 | ;; 40 | esac 41 | done 42 | 43 | TARGET="${GOOS}-${GOARCH}" 44 | 45 | make $TARGET 46 | 47 | -------------------------------------------------------------------------------- /build/binary/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/build/binary/.gitkeep -------------------------------------------------------------------------------- /build/debian/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/golang:1.24-bookworm AS build 2 | 3 | ARG WAYBACK_IPFS_APIKEY 4 | 5 | ENV DEBIAN_FRONTEND noninteractive 6 | 7 | RUN apt-get update -q && \ 8 | apt-get install -y -qq build-essential devscripts dh-make debhelper && \ 9 | mkdir -p /build/debian /pkg && \ 10 | rm -rf /var/lib/apt/lists/* 11 | 12 | COPY . /src 13 | 14 | ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} 15 | ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} 16 | ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} 17 | 18 | CMD ["/src/build/debian/builder"] 19 | -------------------------------------------------------------------------------- /build/debian/builder: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | set -eu pipefail 4 | 5 | WORKSPACE="/src" 6 | PKG_ARCH="${PKG_ARCH:-$(dpkg --print-architecture)}" 7 | PKG_DATE=$(date -R) 8 | PKG_VERSION="${PKG_VERSION:-$(cd /src && git describe --tags --abbrev=0 | sed 's/^v//')}" 9 | 10 | echo "WORKSPACE=${WORKSPACE}" 11 | echo "PKG_VERSION=${PKG_VERSION}" 12 | echo "PKG_ARCH=${PKG_ARCH}" 13 | echo "PKG_DATE=${PKG_DATE}" 14 | 15 | cd "${WORKSPACE}" && \ 16 | make build && \ 17 | mkdir -p /build/debian /pkg && \ 18 | cd /build && \ 19 | cp "${WORKSPACE}/build/binary/wayback" /build/wayback && \ 20 | cp "${WORKSPACE}/wayback.1" /build/ && \ 21 | cp "${WORKSPACE}/LICENSE" /build/ && \ 22 | cp "${WORKSPACE}/build/systemd/wayback.service" /build/debian/ && \ 23 | cp "${WORKSPACE}/build/debian/compat" /build/debian/compat && \ 24 | cp "${WORKSPACE}/build/debian/copyright" /build/debian/copyright && \ 25 | cp "${WORKSPACE}/build/debian/wayback.manpages" /build/debian/wayback.manpages && \ 26 | cp "${WORKSPACE}/build/debian/wayback.postinst" /build/debian/wayback.postinst && \ 27 | cp "${WORKSPACE}/build/debian/rules" /build/debian/rules && \ 28 | echo "wayback (${PKG_VERSION}) experimental; urgency=low" > /build/debian/changelog && \ 29 | echo " * Wayback version ${PKG_VERSION}" >> /build/debian/changelog && \ 30 | echo " -- Wayback Archiver ${PKG_DATE}" >> /build/debian/changelog && \ 31 | sed "s/__PKG_ARCH__/$(dpkg --print-architecture)/g" "${WORKSPACE}/build/debian/control" > /build/debian/control && \ 32 | dpkg-buildpackage -us -uc -b && \ 33 | cp ../*.deb /pkg/ 34 | -------------------------------------------------------------------------------- /build/debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /build/debian/control: -------------------------------------------------------------------------------- 1 | Source: wayback 2 | Maintainer: Wayback Archiver 3 | Build-Depends: debhelper (>= 9.20160709) | dh-systemd 4 | 5 | Package: wayback 6 | Architecture: __PKG_ARCH__ 7 | Section: web 8 | Priority: optional 9 | Description: The wayback is a toolkit for snapshot webpage to 10 | Internet Archive, archive.today, IPFS and beyond. 11 | Homepage: https://github.com/wabarc/wayback 12 | Depends: ${misc:Depends}, ${shlibs:Depends}, adduser 13 | Recommends: 14 | chromium, 15 | libwebp-dev, 16 | youtube-dl, 17 | wget, 18 | nss, 19 | tor 20 | Suggests: 21 | ffmpeg, 22 | you-get 23 | -------------------------------------------------------------------------------- /build/debian/copyright: -------------------------------------------------------------------------------- 1 | Files: * 2 | Copyright: Wayback Archiver 3 | License: GNU General Public License v3.0 4 | -------------------------------------------------------------------------------- /build/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | DESTDIR=/build/debian/wayback 4 | 5 | %: 6 | dh $@ --with=systemd 7 | 8 | override_dh_auto_clean: 9 | override_dh_auto_test: 10 | override_dh_auto_build: 11 | override_dh_auto_install: 12 | mkdir -p $(DESTDIR)/etc 13 | mkdir -p $(DESTDIR)/usr/bin 14 | cp wayback $(DESTDIR)/usr/bin/wayback 15 | 16 | override_dh_installinit: 17 | dh_installinit --noscripts 18 | -------------------------------------------------------------------------------- /build/debian/wayback.manpages: -------------------------------------------------------------------------------- 1 | wayback.1 2 | -------------------------------------------------------------------------------- /build/debian/wayback.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | case "$1" in 6 | configure) 7 | adduser --system --disabled-password --disabled-login --home /var/empty \ 8 | --no-create-home --quiet --force-badname --group wayback 9 | ;; 10 | esac 11 | 12 | #DEBHELPER# 13 | 14 | exit 0 15 | -------------------------------------------------------------------------------- /build/docker/Dockerfile.render: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | # syntax=docker/dockerfile:1.2 6 | ARG WAYBACK_IMAGE_TAG=latest-bundle 7 | 8 | FROM ghcr.io/wabarc/wayback:${WAYBACK_IMAGE_TAG} 9 | 10 | ENV BASE_DIR /wayback 11 | ENV WAYBACK_ONION_LOCAL_PORT 80 12 | 13 | WORKDIR ${BASE_DIR} 14 | 15 | RUN set -ex; \ 16 | chown wayback:nogroup /var/log/tor; \ 17 | chown wayback:nogroup /var/lib/tor; \ 18 | \ 19 | setcap 'cap_net_bind_service=+ep' /usr/local/bin/wayback; \ 20 | \ 21 | sed -i 's/User/#User/g' /etc/tor/torrc 22 | 23 | USER wayback 24 | 25 | EXPOSE 80 26 | 27 | CMD ["/usr/local/bin/wayback", "-d", "web"] 28 | -------------------------------------------------------------------------------- /build/docker/README.md: -------------------------------------------------------------------------------- 1 | ## How do I verify images? 2 | 3 | All images are signed by [cosign](https://github.com/sigstore/cosign). We recommend verifying any wayback image you use. 4 | 5 | Once you've installed cosign, you can use the [wayback public key](https://github.com/wabarc/wayback/blob/main/cosign.pub) to verify any wayback image with: 6 | 7 | ```bash 8 | $ cat cosign.pub 9 | -----BEGIN PUBLIC KEY----- 10 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAET+oZJBKR2xJF6Jj6yH7rEL+5/LP4 11 | jtCZHNwHP1b1CP1mWRIFjzSZbEo0/4ZopFHs2d5qNDbphvCXI6gjEZHmnw== 12 | -----END PUBLIC KEY----- 13 | 14 | $ cosign verify --key cosign.pub $IMAGE_NAME 15 | ``` 16 | 17 | -------------------------------------------------------------------------------- /build/flatpak/org.wabarc.wayback.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | app-id: org.wabarc.wayback 6 | runtime: org.freedesktop.Platform 7 | runtime-version: '20.08' 8 | sdk: org.freedesktop.Sdk 9 | command: wayback 10 | modules: 11 | - name: wayback 12 | buildsystem: simple 13 | build-commands: 14 | - install -D wayback /app/bin/wayback 15 | sources: 16 | - type: file 17 | path: wayback 18 | -------------------------------------------------------------------------------- /build/package/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/build/package/.gitkeep -------------------------------------------------------------------------------- /build/redhat/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | FROM golang:1.24-alpine AS builder 6 | 7 | ARG WAYBACK_IPFS_TARGET 8 | ARG WAYBACK_IPFS_APIKEY 9 | ARG WAYBACK_IPFS_SECRET 10 | 11 | RUN apk update && apk add --no-cache build-base ca-certificates git 12 | # Required by statically linked binary with OpenSSL 13 | # RUN apk add linux-headers 14 | 15 | ENV WAYBACK_IPFS_TARGET ${WAYBACK_IPFS_TARGET} 16 | ENV WAYBACK_IPFS_APIKEY ${WAYBACK_IPFS_APIKEY} 17 | ENV WAYBACK_IPFS_SECRET ${WAYBACK_IPFS_SECRET} 18 | 19 | WORKDIR /go/src/app 20 | 21 | COPY . . 22 | 23 | RUN make linux-amd64 24 | 25 | # FROM fedora:39 AS runtime 26 | FROM docker.io/library/fedora@sha256:61864fd19bbd64d620f338eb11dae9e8759bf7fa97302ac6c43865c48dccd679 AS runtime 27 | 28 | WORKDIR /rpmbuild 29 | 30 | RUN dnf install -y rpm-build rpm-sign systemd 31 | 32 | COPY --from=builder /go/src/app/build/binary/wayback-linux-amd64 /rpmbuild/SOURCES/wayback 33 | COPY --from=builder /go/src/app/LICENSE /rpmbuild/SOURCES/ 34 | COPY --from=builder /go/src/app/CHANGELOG.md /rpmbuild/SOURCES/ 35 | COPY --from=builder /go/src/app/wayback.1 /rpmbuild/SOURCES/ 36 | COPY --from=builder /go/src/app/build/systemd/wayback.service /rpmbuild/SOURCES/ 37 | COPY --from=builder /go/src/app/build/redhat/wayback.spec /rpmbuild/SPECS/wayback.spec 38 | 39 | COPY build/redhat/entrypoint.sh /entrypoint.sh 40 | 41 | ENTRYPOINT "/entrypoint.sh" 42 | -------------------------------------------------------------------------------- /build/redhat/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2024 Wayback Archiver. All rights reserved. 4 | # Use of this source code is governed by the GNU GPL v3 5 | # license that can be found in the LICENSE file. 6 | 7 | set -eu pipefail 8 | 9 | WAYBACK_SIGNING_KEY="${WAYBACK_SIGNING_KEY:-}" 10 | WAYBACK_SIGNING_PASSPHARSE="${WAYBACK_SIGNING_PASSPHARSE:-}" 11 | VERSION="${VERSION:-1.0}" 12 | WORKDIR="/rpmbuild" 13 | 14 | cat > ~/.rpmmacros<< EOF 15 | %_topdir /rpmbuild 16 | %_signature gpg 17 | %_gpg_name Wayback Archiver 18 | EOF 19 | 20 | mkdir -p "${WORKDIR}/{BUILD,RPMS,SOURCES,SPECS,SRPMS}" 21 | 22 | rpmbuild -bb --define "_wayback_version ${VERSION}" "${WORKDIR}/SPECS/wayback.spec" 23 | 24 | if [ -z "${WAYBACK_SIGNING_KEY}" ]; then 25 | echo 'Build RPM package without signing.' 26 | exit 0 27 | fi 28 | 29 | GPG_TTY="$(tty || true)" 30 | 31 | export GPG_TTY 32 | 33 | gpg --import --yes --pinentry-mode loopback --passphrase "${WAYBACK_SIGNING_PASSPHARSE}" <<< "${WAYBACK_SIGNING_KEY}" 34 | 35 | find "${WORKDIR}/RPMS/x86_64" -type f -name "*.rpm" -exec rpm --verbose --define "_gpg_sign_cmd_extra_args --pinentry-mode loopback --passphrase ${WAYBACK_SIGNING_PASSPHARSE}" --addsign {} \; 36 | 37 | find "${WORKDIR}/RPMS/x86_64" -type f -name "*.rpm" -exec rpm -qpi {} \; 38 | 39 | -------------------------------------------------------------------------------- /build/redhat/wayback.spec: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | %undefine _disable_source_fetch 6 | 7 | Name: wayback 8 | Version: %{_wayback_version} 9 | Release: 1.0 10 | Summary: Easy and fast to wayback webpage. 11 | URL: https://github.com/wabarc/wayback 12 | License: GNU General Public License v3.0 13 | Source0: wayback 14 | Source1: wayback.service 15 | Source2: wayback.1 16 | Source3: LICENSE 17 | Source4: CHANGELOG.md 18 | BuildRoot: %{_topdir}/BUILD/%{name}-%{version}-%{release} 19 | BuildArch: x86_64 20 | Requires(pre): shadow-utils 21 | Recommends: chromium, youtube-dl, tor, wget, libwebp, nss 22 | 23 | %{?systemd_requires} 24 | BuildRequires: systemd 25 | 26 | %description 27 | %{summary} 28 | 29 | %install 30 | mkdir -p %{buildroot}%{_bindir} 31 | install -p -m 755 %{SOURCE0} %{buildroot}%{_bindir}/wayback 32 | install -D -m 644 %{SOURCE1} %{buildroot}%{_unitdir}/wayback.service 33 | install -D -m 644 %{SOURCE2} %{buildroot}%{_mandir}/man1/wayback.1 34 | install -D -m 644 %{SOURCE3} %{buildroot}%{_docdir}/wayback/LICENSE 35 | install -D -m 644 %{SOURCE4} %{buildroot}%{_docdir}/wayback/CHANGELOG.md 36 | 37 | %files 38 | %defattr(755,root,root) 39 | %{_bindir}/wayback 40 | %{_docdir}/wayback 41 | %defattr(644,root,root) 42 | %{_unitdir}/wayback.service 43 | %{_mandir}/man1/wayback.1* 44 | %{_docdir}/wayback/* 45 | %defattr(600,root,root) 46 | 47 | %pre 48 | getent group wayback >/dev/null || groupadd -r wayback 49 | getent passwd wayback >/dev/null || \ 50 | useradd -r -g wayback -d /dev/null -s /sbin/nologin \ 51 | -c "Wayback Daemon" wayback 52 | exit 0 53 | 54 | %post 55 | %systemd_post wayback.service 56 | 57 | %preun 58 | %systemd_preun wayback.service 59 | 60 | %postun 61 | %systemd_postun_with_restart wayback.service 62 | 63 | %changelog 64 | -------------------------------------------------------------------------------- /build/systemd/wayback.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Wayback webpage 3 | Documentation=https://docs.wabarc.eu.org/ 4 | After=network.target 5 | 6 | [Service] 7 | Type=notify 8 | User=wayback 9 | ExecStart=/usr/bin/wayback -d web 10 | Restart=always 11 | 12 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#NoNewPrivileges= 13 | NoNewPrivileges=true 14 | 15 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#PrivateDevices= 16 | PrivateDevices=true 17 | 18 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#ProtectControlGroups= 19 | ProtectControlGroups=true 20 | 21 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#ProtectHome= 22 | ProtectHome=true 23 | 24 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#ProtectKernelModules= 25 | ProtectKernelModules=true 26 | 27 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#ProtectKernelTunables= 28 | ProtectKernelTunables=true 29 | 30 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#ProtectSystem= 31 | ProtectSystem=strict 32 | 33 | # https://www.freedesktop.org/software/systemd/man/systemd.exec.html#RestrictRealtime= 34 | RestrictRealtime=true 35 | 36 | [Install] 37 | WantedBy=multi-user.target 38 | -------------------------------------------------------------------------------- /cmd/playback/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "net/url" 10 | "os" 11 | 12 | "github.com/spf13/cobra" 13 | "github.com/wabarc/logger" 14 | "github.com/wabarc/playback" 15 | "github.com/wabarc/wayback" 16 | "github.com/wabarc/wayback/config" 17 | ) 18 | 19 | func main() { 20 | opts, err := config.NewParser().ParseEnvironmentVariables() 21 | if err != nil { 22 | logger.Fatal("Parse environment variables or flags failed, error: %v", err) 23 | } 24 | 25 | var rootCmd = &cobra.Command{ 26 | Use: "playback", 27 | Short: "A toolkit to playback archived webpage from time capsules.", 28 | Example: ` playback https://example.com https://example.org`, 29 | Version: playback.Version, 30 | Run: func(cmd *cobra.Command, args []string) { 31 | handle(cmd, opts, args) 32 | }, 33 | } 34 | 35 | // nolint:errcheck 36 | rootCmd.Execute() 37 | } 38 | 39 | func handle(cmd *cobra.Command, opts *config.Options, args []string) { 40 | if len(args) < 1 { 41 | // nolint:errcheck 42 | cmd.Usage() 43 | os.Exit(0) 44 | } 45 | 46 | urls, err := unmarshalArgs(args) 47 | if err != nil { 48 | cmd.Println(err) 49 | os.Exit(1) 50 | } 51 | 52 | collects, err := wayback.Playback(context.TODO(), opts, urls...) 53 | if err != nil { 54 | cmd.Println(err) 55 | os.Exit(1) 56 | } 57 | 58 | for _, collect := range collects { 59 | fmt.Printf("[%s]\n", collect.Arc) 60 | for orig, dest := range collect.Dst { 61 | fmt.Println(orig, "=>", dest) 62 | } 63 | fmt.Printf("\n") 64 | } 65 | } 66 | 67 | func unmarshalArgs(args []string) (urls []*url.URL, err error) { 68 | for _, s := range args { 69 | uri, er := url.Parse(s) 70 | if er != nil { 71 | err = fmt.Errorf("%w: unexpected url: %s", err, s) 72 | continue 73 | } 74 | urls = append(urls, uri) 75 | } 76 | return 77 | } 78 | -------------------------------------------------------------------------------- /cmd/wayback/pprof.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | package main 5 | 6 | // nosemgrep: gitlab.gosec.G108-1 7 | import ( 8 | "log" 9 | "net" 10 | "net/http" 11 | 12 | "github.com/wabarc/logger" 13 | 14 | _ "net/http/pprof" 15 | ) 16 | 17 | func profiling() { 18 | listener, err := net.Listen("tcp4", "127.0.0.1:0") 19 | if err != nil { 20 | logger.Fatal("Net listen err: %v", err) 21 | } 22 | 23 | addr := listener.Addr().(*net.TCPAddr) 24 | logger.Info("Go profiling via: http://%s", addr) 25 | logger.Info("More details can be found at https://go.dev/blog/pprof") 26 | 27 | go func() { 28 | //#nosec G114 -- Ignored for convenience 29 | log.Println(http.Serve(listener, nil)) 30 | }() 31 | } 32 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | informational: true # non-blocking status checks 6 | patch: 7 | default: 8 | informational: true # non-blocking status checks 9 | -------------------------------------------------------------------------------- /config/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package config handles configuration management for the application. 7 | */ 8 | package config // import "github.com/wabarc/wayback/config" 9 | -------------------------------------------------------------------------------- /cosign.pub: -------------------------------------------------------------------------------- 1 | -----BEGIN PUBLIC KEY----- 2 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE+knrnL1JBz66Wcq63ambIGQV6H4a 3 | X1R7hTBu0UKV3ok1TE/4nYvypox0rMvtTTjnsAo1VOa+ur7AH4BdAcI0JA== 4 | -----END PUBLIC KEY----- 5 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package wayback is a toolkit for snapshot webpage to 7 | Internet Archive, archive.today, IPFS and beyond. 8 | */ 9 | package wayback // import "github.com/wabarc/wayback" 10 | -------------------------------------------------------------------------------- /docker-bake.hcl: -------------------------------------------------------------------------------- 1 | // Go version 2 | variable "GO_VERSION" { 3 | default = "1.24" 4 | } 5 | 6 | variable "WAYBACK_IPFS_TARGET" { 7 | default = "" 8 | } 9 | 10 | variable "WAYBACK_IPFS_APIKEY" { 11 | default = "" 12 | } 13 | 14 | variable "WAYBACK_IPFS_SECRET" { 15 | default = "" 16 | } 17 | 18 | // GitHub reference as defined in GitHub Actions (eg. refs/head/master)) 19 | variable "GITHUB_REF" { 20 | default = "" 21 | } 22 | 23 | target "_common" { 24 | args = { 25 | GO_VERSION = GO_VERSION 26 | WAYBACK_IPFS_TARGET = WAYBACK_IPFS_TARGET 27 | WAYBACK_IPFS_APIKEY = WAYBACK_IPFS_APIKEY 28 | WAYBACK_IPFS_SECRET = WAYBACK_IPFS_SECRET 29 | } 30 | } 31 | 32 | group "default" { 33 | targets = ["image-local"] 34 | } 35 | 36 | // Special target: https://github.com/docker/docker-metadata-action#bake-definition 37 | target "docker-metadata-action" { 38 | tags = ["wabarc/wayback:local"] 39 | } 40 | 41 | target "image" { 42 | inherits = ["_common", "docker-metadata-action"] 43 | } 44 | 45 | target "image-local" { 46 | inherits = ["image"] 47 | output = ["type=docker"] 48 | } 49 | 50 | target "artifact" { 51 | inherits = ["image"] 52 | output = ["./dist"] 53 | } 54 | 55 | target "artifact-all" { 56 | inherits = ["artifact"] 57 | platforms = [ 58 | "linux/386", 59 | "linux/amd64", 60 | "linux/arm/v6", 61 | "linux/arm/v7", 62 | "linux/arm64", 63 | "linux/ppc64le" 64 | ] 65 | } 66 | 67 | target "release" { 68 | inherits = ["image"] 69 | context = "./" 70 | platforms = [ 71 | "linux/386", 72 | "linux/amd64", 73 | "linux/arm/v6", 74 | "linux/arm/v7", 75 | "linux/arm64", 76 | "linux/ppc64le" 77 | ] 78 | } 79 | 80 | target "bundle" { 81 | inherits = ["image"] 82 | context = "./" 83 | dockerfile = "./build/docker/Dockerfile.all" 84 | platforms = [ 85 | "linux/386", 86 | "linux/amd64", 87 | "linux/arm/v6", 88 | "linux/arm/v7", 89 | "linux/arm64", 90 | "linux/ppc64le" 91 | ] 92 | } 93 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | 5 | version: "3" 6 | 7 | services: 8 | browser: 9 | image: chromedp/headless-shell 10 | ports: 11 | - 9222:9222 12 | networks: 13 | - back-tier 14 | volumes: 15 | - /dev/shm:/dev/shm 16 | restart: unless-stopped 17 | hostname: browser 18 | 19 | meilisearch: 20 | image: getmeili/meilisearch:v1.1.0 21 | ports: 22 | - 7700:7700 23 | networks: 24 | - back-tier 25 | volumes: 26 | - storage:/meili_data 27 | restart: unless-stopped 28 | hostname: meilisearch 29 | environment: 30 | - MEILI_NO_ANALYTICS=true 31 | - MEILI_HTTP_ADDR=0.0.0.0:7700 32 | - MEILI_ENV=production 33 | 34 | wayback: 35 | image: wabarc/wayback 36 | ports: 37 | - 8964:8964 38 | networks: 39 | - back-tier 40 | volumes: 41 | - storage:/data 42 | depends_on: 43 | - browser 44 | - meilisearch 45 | environment: 46 | - WAYBACK_STORAGE_DIR=/data 47 | - CHROME_REMOTE_ADDR=browser:9222 48 | - WAYBACK_MEILI_ENDPOINT=http://meilisearch:7700 49 | - PLAYBACK_MEILI_ENDPOINT=http://meilisearch:7700 50 | command: wayback -d web 51 | restart: unless-stopped 52 | build: ./ 53 | 54 | networks: 55 | back-tier: 56 | 57 | volumes: 58 | storage: 59 | driver: local 60 | -------------------------------------------------------------------------------- /docs/assets/discord-server.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/discord-server.png -------------------------------------------------------------------------------- /docs/assets/irc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/irc.png -------------------------------------------------------------------------------- /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/logo.png -------------------------------------------------------------------------------- /docs/assets/mastodon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/mastodon.png -------------------------------------------------------------------------------- /docs/assets/matrix-room.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/matrix-room.png -------------------------------------------------------------------------------- /docs/assets/slack-channel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/slack-channel.png -------------------------------------------------------------------------------- /docs/assets/telegram-channel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/telegram-channel.png -------------------------------------------------------------------------------- /docs/assets/telegram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/telegram.png -------------------------------------------------------------------------------- /docs/assets/web.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/web.png -------------------------------------------------------------------------------- /docs/assets/xmpp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/docs/assets/xmpp.png -------------------------------------------------------------------------------- /docs/contributing.zh.md: -------------------------------------------------------------------------------- 1 | # 贡献 2 | 3 | 你好！非常高兴看到你想为这个项目做出贡献。你的帮助对于保持项目的优秀非常重要。 4 | 5 | 请注意，这个项目发布了[贡献者行为准则][code-of-conduct]。参与这个项目的过程中，你同意遵守这个准则。 6 | 7 | ## 发布新版本 8 | 9 | 我们使用[semantic-release](https://github.com/semantic-release/semantic-release)来自动发布新版本。 10 | 11 | 一次只会提升一个版本号，最高版本的更改会覆盖其他版本的更改。除了将 Docker 镜像发布到 Docker Hub 和 GitHub Packages，semantic-release 还会在 GitHub 上创建 Git 标签和发布，生成版本二进制文件摘要并将其放入发布说明中。 12 | 13 | ## 提交 Pull Requests 14 | 15 | 1. [Fork][fork] 并克隆仓库。 16 | 2. 确保你的机器上测试通过：`make test` 或 `go test -v ./...` 17 | 3. 创建新分支：`git checkout -b my-branch-name` 18 | 4. 进行更改、添加测试，并确保测试仍然通过。 19 | 5. 推送到你的 Fork 上并[提交 Pull Requests][pr]。 20 | 6. 为自己鼓掌，等待你的 Pull Request 被审查并合并。 21 | 22 | 我们也欢迎 Work in Progress 的 Pull Requests，以便你可以尽早得到反馈，或者如果你有什么困难。 23 | 24 | ## 报告 Bug 25 | 26 | ### 先决条件 27 | 28 | 创建 Bug 报告时，最重要的细节是确定是否真的需要创建一个 Bug 报告。 29 | 30 | #### 先做研究 31 | 32 | 你是否研究过现有的问题、已关闭和开放的问题，看看其他用户是否遇到并可能已经解决了你遇到的相同问题？ 33 | 34 | #### 描述问题，不要轻易得出结论 35 | 36 | 最近，我和另一位维护者讨论了这个话题。我们想知道我们处理了多少个标题中包含 "Bug" 这个词或类似的内容的问题，最终发现是用户错误或绝对不是 Bug。这只是一个猜测，但我认为说只有 10 个标题中带有 "Bug" 的报告中，只有 1 个最终是 Bug。 37 | 38 | ### 重要细节 39 | 40 | 当需要 Bug 报告时，绝大多数的 Bug 报告应该包括以下四个信息： 41 | 42 | 1. `版本` 43 | 2. `描述` 44 | 3. `错误信息` 45 | 4. `代码` 46 | 47 | ## 功能请求 48 | 49 | 在提交功能请求之前，请尝试熟悉项目。了解项目是否有特定的目标或指导方针，描述功能请求的方式应该是怎样的。 50 | 51 | ## 刚开始？寻找如何帮助？ 52 | 53 | 使用[这个搜索工具][good-first-issue-search]查找已标记为 `good-first-issue` 的 Wayback Archiver 问题。 54 | 55 | ## 资源 56 | 57 | - [如何贡献开源项目](https://opensource.guide/how-to-contribute/) 58 | - [使用 Pull Requests](https://help.github.com/articles/about-pull-requests/) 59 | - [GitHub 帮助](https://help.github.com) 60 | 61 | [fork]: https://github.com/wabarc/wayback/fork 62 | [pr]: https://github.com/wabarc/wayback/compare 63 | [code-of-conduct]: ./CODE_OF_CONDUCT.md 64 | [good-first-issue-search]: https://github.com/search?q=org%3Awabarc+good-first-issues%3A%3E0 65 | -------------------------------------------------------------------------------- /docs/deployment.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Deployment 3 | --- 4 | 5 | Wayback can be deployed using Docker/Podman containers, and can also be easily deployed on cloud platforms such as Heroku and Render. 6 | 7 | ## Docker/Podman 8 | 9 | ```sh 10 | docker pull wabarc/wayback 11 | docker run -d wabarc/wayback wayback -d telegram -t YOUR-BOT-TOKEN # without telegram channel 12 | docker run -d wabarc/wayback wayback -d telegram -t YOUR-BOT-TOKEN -c YOUR-CHANNEL-USERNAME # with telegram channel 13 | ``` 14 | 15 | ## 1-Click Deploy 16 | 17 | [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/wabarc/wayback) 18 | 19 | Deploy to Render 24 | 25 | 26 | ## Deployment 27 | 28 | - [wabarc/on-heroku](https://github.com/wabarc/on-heroku) 29 | - [wabarc/on-github](https://github.com/wabarc/on-github) 30 | - [wabarc/on-render](https://github.com/wabarc/on-render) 31 | -------------------------------------------------------------------------------- /docs/deployment.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 部署 3 | --- 4 | 5 | Wayback可以使用Docker/Podman容器部署，也可以轻松部署在Heroku和Render等云平台上。 6 | 7 | ## Docker/Podman 8 | 9 | ```sh 10 | docker pull wabarc/wayback 11 | docker run -d wabarc/wayback wayback -d telegram -t YOUR-BOT-TOKEN # without telegram channel 12 | docker run -d wabarc/wayback wayback -d telegram -t YOUR-BOT-TOKEN -c YOUR-CHANNEL-USERNAME # with telegram channel 13 | ``` 14 | 15 | ## 一键部署 16 | 17 | [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/wabarc/wayback) 18 | 19 | Deploy to Render 24 | 25 | 26 | ## 快速部署 27 | 28 | - [wabarc/on-heroku](https://github.com/wabarc/on-heroku) 29 | - [wabarc/on-github](https://github.com/wabarc/on-github) 30 | - [wabarc/on-render](https://github.com/wabarc/on-render) 31 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | Wayback is a web archiving and playback tool that allows users to capture and preserve web content. It provides an IM-style interface for receiving and presenting archived web content, and a search and playback service for retrieving previously archived pages. Wayback is designed to be used by web archivists, researchers, and anyone who wants to preserve web content and access it in the future. 2 | 3 | Wayback is an open-source web archiving application written in Go. With a modular and customizable architecture, it is designed to be flexible and adaptable to various use cases and environments. It provides support for multiple storage backends and integration with other services. 4 | 5 | Whether you need to archive a single web page or a large collection of web sites, Wayback can help you capture and preserve web content for posterity. With an easy-to-use interface and powerful features, Wayback is a valuable tool for anyone interested in web archiving and preservation. 6 | 7 | ## Features 8 | 9 | - Free and open-source 10 | - Expose prometheus metrics 11 | - Cross-platform compatibility 12 | - Batch wayback URLs for faster archiving 13 | - Built-in CLI (`wayback`) for convenient use 14 | - Serve as a Tor Hidden Service or local web entry for added privacy and accessibility 15 | - Easier wayback to Internet Archive, archive.today, IPFS and Telegraph integration 16 | - Interactive with IRC, Matrix, Telegram bot, Discord bot, Mastodon, Twitter, and XMPP as a daemon service for convenient use 17 | - Supports publishing wayback results to Telegram channel, Mastodon, and GitHub Issues for sharing 18 | - Supports storing archived files to disk for offline use 19 | - Download streaming media (requires [FFmpeg](https://ffmpeg.org/)) for convenient media archiving. 20 | 21 | ## How it works 22 | 23 | ![How wayback works](./assets/wayback.svg "How wayback works") 24 | -------------------------------------------------------------------------------- /docs/index.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 简介 3 | --- 4 | 5 | Wayback是一个网络归档和回放工具，允许用户抓取和保存网络内容。它提供了一个IM风格的界面来接收和展示存档的网络内容，以及一个搜索和回放服务来检索以前存档的网页。Wayback是为网络档案员、研究人员和任何想保存网络内容并在未来访问这些内容的人设计的。 6 | 7 | Wayback是用Go编写的开源网络存档应用程序。具有模块化和可定制化的架构，它被设计成灵活和适应各种用例和环境。它提供了对多个存储后端的支持，并与其他服务集成。 8 | 9 | 无论您是需要归档一个网页，还是需要归档一大批网站，Wayback都可以帮助你抓取和保存网络内容，以备后用。凭借易于使用的界面和强大的功能，Wayback对于任何对网络归档和保存感兴趣的人来说都是一个宝贵的工具。 10 | 11 | ## 特性 12 | 13 | - 完全开源 14 | - 跨平台兼容 15 | - 输出Prometheus度量指标 16 | - 批量存档URL以加快存档速度 17 | - 内置CLI工具（`wayback`）以便于使用 18 | - 可作为Tor隐藏服务或本地Web入口，增加隐私和可访问性 19 | - 更容易地集成到Internet Archive、archive.today、IPFS和Telegraph中 20 | - 可与IRC、Matrix、Telegram机器人、Discord机器人、Mastodon、Twitter和XMPP进行交互，作为守护进程服务，以便于使用 21 | - 支持将存档结果发布到Telegram频道、Mastodon和GitHub Issues中进行共享 22 | - 支持将存档文件存储到磁盘中以供离线使用 23 | - 下载流媒体（需要[FFmpeg](https://ffmpeg.org/)）以便于媒体存档。 24 | 25 | ## 工作原理 26 | 27 | ![How wayback works](./assets/wayback.svg "How wayback works") 28 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | ## Prerequisites 2 | 3 | Wayback requires at least 512MB of memory, and some optional packages that can be installed below. 4 | 5 | - [Chromium](https://www.chromium.org/Home): Wayback uses a headless Chromium to capture web pages for archiving purposes. 6 | - [Tor](https://www.torproject.org/): Wayback can use Tor as a proxy to scrape web pages anonymously, and it can also serve as an onion service to allow users to access archived content via the Tor network. 7 | - [youtube-dl](https://github.com/ytdl-org/youtube-dl/) or [You-Get](https://you-get.org/): Wayback can use either of these tools to download media for archiving purposes. 8 | - [libwebp](https://developers.google.com/speed/webp/) library: Wayback uses libwebp to convert WebP images to other formats when necessary. 9 | 10 | ## Installation 11 | 12 | The simplest, cross-platform way is to download from [GitHub Releases](https://github.com/wabarc/wayback/releases) and place the executable file in your PATH. 13 | 14 | From source: 15 | 16 | ```sh 17 | go install github.com/wabarc/wayback/cmd/wayback@latest 18 | ``` 19 | 20 | From GitHub Releases: 21 | 22 | ```sh 23 | curl -fsSL https://github.com/wabarc/wayback/raw/main/install.sh | sh 24 | ``` 25 | 26 | or via [Bina](https://bina.egoist.dev/): 27 | 28 | ```sh 29 | curl -fsSL https://bina.egoist.dev/wabarc/wayback | sh 30 | ``` 31 | 32 | Using [Snapcraft](https://snapcraft.io/wayback) (on GNU/Linux) 33 | 34 | ```sh 35 | sudo snap install wayback 36 | ``` 37 | 38 | Via [APT](https://repo.wabarc.eu.org/deb:wayback): 39 | 40 | ```bash 41 | curl -fsSL https://repo.wabarc.eu.org/apt/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/packages.wabarc.gpg 42 | echo "deb [arch=amd64,arm64,armhf signed-by=/usr/share/keyrings/packages.wabarc.gpg] https://repo.wabarc.eu.org/apt/ /" | sudo tee /etc/apt/sources.list.d/wayback.list 43 | sudo apt update 44 | sudo apt install wayback 45 | ``` 46 | 47 | Via [RPM](https://repo.wabarc.eu.org/rpm:wayback): 48 | 49 | ```bash 50 | sudo rpm --import https://repo.wabarc.eu.org/yum/gpg.key 51 | sudo tee /etc/yum.repos.d/wayback.repo > /dev/null < /dev/null < "[Development](https://mastodon.social/settings/applications)" > "[New Application](https://mastodon.social/settings/applications/new)". 13 | 3. Enter the following information: 14 | - **Application name**: The name of your application. 15 | - **Application website**: The website associated with your application. 16 | - **Redirect URI**: The URI where users will be redirected after authentication. This can be any valid URI, but it must match the redirect URI specified in your code. 17 | - Scopes: The scopes your application requires. These determine the actions your application can perform on behalf of the user. The required minimum scopes are: `read:statuses`, `read:notifications`, `write:statuses`, `write:notifications` and `write:conversations`. 18 | 4. Click "Submit". 19 | 5. On the next page, you will see your application's client ID and client secret. These will be needed to authenticate your application. 20 | 21 | You can find more information about creating Mastodon applications in the Mastodon documentation: https://docs.joinmastodon.org/client/token/ 22 | 23 | ## Configuration 24 | 25 | After creating a Mastodon application, you can find the `Client key`, `Client secret`, and `Your access token` on the application details page. 26 | 27 | Next, place these keys in the environment or configuration file: 28 | 29 | The next step, place them below key on env or config file: 30 | 31 | - `WAYBACK_MASTODON_KEY`: Client key 32 | - `WAYBACK_MASTODON_SECRET`: Client secret 33 | - `WAYBACK_MASTODON_TOKEN`: Your access token 34 | 35 | Additionally, you must specify the Mastodon server by setting the `WAYBACK_MASTODON_SERVER` variable. 36 | 37 | ## Further reading 38 | 39 | - [Fediverse Observer](https://mastodon.fediverse.observer/list) 40 | -------------------------------------------------------------------------------- /docs/integrations/mastodon.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Mastodon 3 | --- 4 | 5 | ## 如何构建Mastodon机器人 6 | 7 | 您可以选择任何Mastodon实例。这里，我们将使用Mastodon.social作为示例。 8 | 9 | 要创建Mastodon应用程序，您可以按照以下步骤操作： 10 | 11 | 1. 登录到您的Mastodon帐户。 12 | 2. 转到“设置”>“[开发](https://mastodon.social/settings/applications)”>“[新应用程序](https://mastodon.social/settings/applications/new)”。 13 | 3. 输入以下信息： 14 | - **应用程序名称**：您的应用程序名称。 15 | - **应用程序网站**：与您的应用程序相关联的网站。 16 | - **重定向URI**：用户在身份验证后将被重定向的URI。这可以是任何有效的URI，但它必须与您的代码中指定的重定向URI匹配。 17 | - 范围：您的应用程序需要的范围。这些范围确定您的应用程序可以代表用户执行的操作。所需的最小范围为：`read:statuses`、`read:notifications`、`write:statuses`、`write:notifications`和`write:conversations`。 18 | 4. 单击“提交”。 19 | 5. 在下一页上，您将看到您的应用程序的客户端ID和客户端密钥。这些将用于验证您的应用程序。 20 | 21 | 您可以在Mastodon文档中找到有关创建Mastodon应用程序的更多信息：https://docs.joinmastodon.org/client/token/ 22 | 23 | ## 配置 24 | 25 | 创建Mastodon应用程序后，在应用程序详细信息页面上可以找到`Client key`、`Client secret`和`Your access token`。 26 | 27 | 接下来，将这些密钥放置在环境或配置文件中： 28 | 29 | - `WAYBACK_MASTODON_KEY`：客户端密钥 30 | - `WAYBACK_MASTODON_SECRET`：客户端密钥 31 | - `WAYBACK_MASTODON_TOKEN`：您的访问令牌 32 | 33 | 另外，您必须通过设置`WAYBACK_MASTODON_SERVER`变量来指定Mastodon服务器。 34 | 35 | ## 相关资料 36 | 37 | - [Fediverse Observer](https://mastodon.fediverse.observer/list) 38 | -------------------------------------------------------------------------------- /docs/integrations/matrix.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive with Matrix 3 | --- 4 | 5 | ## How to build a Matrix Bot 6 | 7 | You can choose any Matrix server. Here, we will be using **matrix.org** and **Element** as an example. 8 | 9 | To register a Matrix account, follow these steps: 10 | 11 | 1. Open [Element](https://app.element.io/) and click "Create Account". 12 | 2. Fill in the required information. 13 | 3. Log in and create a **public room** for publishing (optional). 14 | 4. Go to **Room Settings** > **Advanced**, you can find **Internal room ID** (optional). 15 | 16 | ## Configuration 17 | 18 | After creating a Matrix account, you will have the `Homeserver`, `User ID`, `Password`, and `Internal room ID`. 19 | 20 | Next, place these keys in the environment or configuration file: 21 | 22 | - `WAYBACK_MATRIX_HOMESERVER`: Homeserver of your choice, defaults to `matrix.org` 23 | - `WAYBACK_MATRIX_USERID`: User ID, e.g. `@alice:matrix.org` 24 | - `WAYBACK_MATRIX_ROOMID`: Internal room ID 25 | - `WAYBACK_MATRIX_PASSWORD`: Password from your registration step. 26 | 27 | ## Further reading 28 | 29 | - [Guides for Developers](https://matrix.org/docs/develop/) 30 | -------------------------------------------------------------------------------- /docs/integrations/matrix.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Matrix 3 | --- 4 | 5 | ## 如何构建Matrix机器人 6 | 7 | 您可以选择任何Matrix服务器。这里，我们将使用**matrix.org**和**Element**作为示例。 8 | 9 | 要注册Matrix帐户，请按照以下步骤操作： 10 | 11 | 1. 打开[Element](https://app.element.io/)并单击“创建帐户”。 12 | 2. 填写所需的信息。 13 | 3. 登录并创建一个**公共房间**以发布（可选）。 14 | 4. 转到**房间设置**>**高级**，您可以找到**内部房间ID**（可选）。 15 | 16 | ## 配置 17 | 18 | 创建Matrix帐户后，您将拥有`Homeserver`、`User ID`、`Password`和`Internal room ID`。 19 | 20 | 接下来，将这些密钥放置在环境或配置文件中： 21 | 22 | - `WAYBACK_MATRIX_HOMESERVER`：您选择的Homeserver，默认为`matrix.org` 23 | - `WAYBACK_MATRIX_USERID`：用户ID，例如`@alice:matrix.org` 24 | - `WAYBACK_MATRIX_ROOMID`：内部房间ID 25 | - `WAYBACK_MATRIX_PASSWORD`：从您的注册步骤中获取的密码。 26 | 27 | ## 相关资料 28 | 29 | - [开发人员指南](https://matrix.org/docs/develop/) 30 | -------------------------------------------------------------------------------- /docs/integrations/meilisearch.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish to Meilisearch 3 | --- 4 | 5 | ## How to build a service 6 | 7 | Meilisearch is a fast and powerful open-source search engine that can deliver relevant search results in a matter of milliseconds. Since [v0.18.0](https://github.com/wabarc/wayback/releases/tag/v0.18.0), wayback has been supports Meilisearch to store archived results for playback. The following data structure is used: 8 | 9 | ```proto 10 | message Document { 11 | string Source = 1; 12 | string IA = 2; 13 | string IS = 3; 14 | string IP = 4; 15 | string PH = 5; 16 | } 17 | ``` 18 | 19 | To install Meilisearch, you can follow the installation guide available on the official Meilisearch website: . 20 | 21 | ## Configuration 22 | 23 | After running Meilisearch, you will have the endpoint. 24 | 25 | Next, place these keys in the environment or configuration file: 26 | 27 | - `WAYBACK_MEILI_ENDPOINT`: Meilisearch API endpoint. 28 | - `WAYBACK_MEILI_INDEXING`: Meilisearch indexing name, defaults to `capsules` (optional). 29 | - `WAYBACK_MEILI_APIKEY`: Meilisearch admin API key (optional). 30 | -------------------------------------------------------------------------------- /docs/integrations/meilisearch.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 发布到Meilisearch 3 | --- 4 | 5 | ## 如何构建服务 6 | 7 | Meilisearch是一款快速而强大的开源搜索引擎，可以在毫秒级别内提供相关的搜索结果。从[v0.18.0](https://github.com/wabarc/wayback/releases/tag/v0.18.0)版本开始，wayback支持使用Meilisearch存储归档结果以进行回放。使用以下数据结构： 8 | 9 | ```proto 10 | message Document { 11 | string Source = 1; 12 | string IA = 2; 13 | string IS = 3; 14 | string IP = 4; 15 | string PH = 5; 16 | } 17 | ``` 18 | 19 | 要安装Meilisearch，您可以按照官方Meilisearch网站上的安装指南进行操作：。 20 | 21 | ## 配置 22 | 23 | 运行Meilisearch后，您将拥有终端节点。 24 | 25 | 接下来，将这些密钥放置在环境或配置文件中： 26 | 27 | - `WAYBACK_MEILI_ENDPOINT`：Meilisearch API终端节点。 28 | - `WAYBACK_MEILI_INDEXING`：Meilisearch索引名称，默认为`capsules`（可选）。 29 | - `WAYBACK_MEILI_APIKEY`：Meilisearch管理员API密钥（可选）。 30 | -------------------------------------------------------------------------------- /docs/integrations/metrics.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Metrics 3 | --- 4 | 5 | Wayback metrics provide important insights into the performance and status of the Wayback service. 6 | These metrics include memory usage, which helps track the amount of memory used by the service; 7 | request status, which helps identify the status of incoming requests and whether they are being successfully handled; 8 | publish status, which helps track the status of published web pages; 9 | and garbage collection, which helps track the removal of unused web pages to free up memory. 10 | By monitoring these metrics, users can gain a better understanding of the overall health of the Wayback service and identify any issues that may need to be addressed. 11 | 12 | In each wayback service, the command `/metrics` can be used to obtain metrics. 13 | 14 | -------------------------------------------------------------------------------- /docs/integrations/metrics.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Metrics 3 | --- 4 | 5 | Wayback指标提供了对Wayback服务性能和状态的重要见解。这些指标包括内存使用情况，有助于跟踪服务使用的内存量；请求状态，有助于识别传入请求的状态以及它们是否被成功处理；发布状态，有助于跟踪已发布的网页的状态；以及垃圾回收，有助于跟踪删除未使用的网页以释放内存。通过监控这些指标，用户可以更好地了解Wayback服务的整体健康状况，并确定可能需要解决的任何问题。 6 | 7 | 在每个Wayback服务中，可以使用命令`/metrics`来获取指标。 8 | -------------------------------------------------------------------------------- /docs/integrations/nostr.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish to Nostr 3 | --- 4 | 5 | ## How to build a Nostr Bot 6 | 7 | Wayback currently only supports publishing to Nostr as the Nostr Protocol is still under development. 8 | 9 | Select any relay to generate a private key (here's a [guide](https://nostr.how/) to help you get started). 10 | 11 | ## Configuration 12 | 13 | After creating a new account, you will have the `private key`. 14 | 15 | Next, place these keys in the environment or configuration file: 16 | 17 | - `WAYBACK_NOSTR_RELAY_URL`: Nostr relay server url, multiple separated by comma. 18 | - `WAYBACK_NOSTR_PRIVATE_KEY`: The private key of a Nostr account. 19 | 20 | ## Further reading 21 | 22 | - [Nostr Protocol](https://github.com/nostr-protocol/nostr) 23 | -------------------------------------------------------------------------------- /docs/integrations/nostr.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 发布到Nostr 3 | --- 4 | 5 | ## 如何构建Nostr机器人 6 | 7 | 目前，Wayback仅支持将归档结果发布到Nostr，因为Nostr协议仍在开发中。 8 | 9 | 选择任何继电器以生成一个私钥（这里有一个[指南](https://nostr.how/)可以帮助您入门）。 10 | 11 | ## 配置 12 | 13 | 创建新帐户后，您将拥有`私钥`。 14 | 15 | 接下来，将这些密钥放置在环境或配置文件中： 16 | 17 | - `WAYBACK_NOSTR_RELAY_URL`：Nostr中继服务器URL，用逗号分隔的多个URL。 18 | - `WAYBACK_NOSTR_PRIVATE_KEY`：Nostr帐户的私钥。 19 | 20 | ## 相关资料 21 | 22 | - [Nostr协议](https://github.com/nostr-protocol/nostr) 23 | -------------------------------------------------------------------------------- /docs/integrations/notion.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish to Notion 3 | --- 4 | 5 | ## How to build a Notion Integration 6 | 7 | 1. Sign up for a Notion account, if you don't have one already. 8 | 2. Create a new Integration by going to the [Notion API page](https://www.notion.so/my-integrations) and clicking on "**My integrations**" in the top-right corner of the page. 9 | 3. Give your integration a name and click on "**Create Integration**". 10 | 4. On the integration page, click on "**Add a new integration**". 11 | 5. Select "**Internal Integration**" and click "**Submit**". 12 | 6. On the next page, you will see your "**Integration Token**". Copy this token as you will need it later. 13 | 7. Grant your integration access to a database by sharing the database with your integration. To do this, go to the database you want to use with your bot, click on the three-dot menu, and select "**Share**". 14 | 8. In the "**Connections**" field, navigate to "**Add connections**", find your integration and select it from the list. 15 | 9. Select the appropriate permissions for your integration and click on "**Confirm**". 16 | 17 | Note: A Notion database must be created using the "**New page**" option, not the "**Add a page**" option. 18 | 19 | ## Configuration 20 | 21 | After creating a new account, you will have the `Integration Token` and `Notion database ID`. 22 | 23 | Next, place these keys in the environment or configuration file: 24 | 25 | - `WAYBACK_NOTION_TOKEN`: Notion integration token. 26 | - `WAYBACK_NOTION_DATABASE_ID`: Notion database ID for archiving results. 27 | 28 | -------------------------------------------------------------------------------- /docs/integrations/notion.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 发布到Notion 3 | --- 4 | 5 | ## 如何构建Notion集成 6 | 7 | 1. 如果您没有Notion帐户，请注册一个Notion帐户。 8 | 2. 创建一个新的集成，方法是转到[Notion API页面](https://www.notion.so/my-integrations)，然后单击页面右上角的“我的集成”。 9 | 3. 给您的集成命名，然后单击“创建集成”。 10 | 4. 在集成页面上，单击“添加新集成”。 11 | 5. 选择“内部集成”，然后单击“提交”。 12 | 6. 在下一页上，您将看到您的“集成令牌”。复制此令牌，因为稍后您将需要它。 13 | 7. 通过与您的集成共享数据库，授予您的集成对数据库的访问权限。要执行此操作，请转到要与您的机器人一起使用的数据库，单击三点菜单，然后选择“共享”。 14 | 8. 点击展开右上角菜单，在“Connections”选项中，输入您的集成名称，并从列表中选择它。 15 | 9. 选择适当的权限，然后单击“确认”。 16 | 17 | 注意：一个Notion数据库必须通过"New page"选项创建, 不是"Add a page"选项。 18 | 19 | ## 配置 20 | 21 | 创建新帐户后，您将拥有“集成令牌”和“Notion数据库ID”。 22 | 23 | 接下来，将这些密钥放置在环境或配置文件中： 24 | 25 | - `WAYBACK_NOTION_TOKEN`：Notion集成令牌。 26 | - `WAYBACK_NOTION_DATABASE_ID`：用于归档结果的Notion数据库ID。 27 | -------------------------------------------------------------------------------- /docs/integrations/omnivore.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publish to Omnivore 3 | --- 4 | 5 | Omnivore lets you save and organize articles, newsletters, and documents for later reading. It also syncs with popular knowledge management systems and offers text-to-speech and distraction free features. 6 | 7 | ## Configuration 8 | 9 | Create API key, place key in environment or configuration file: 10 | 11 | - `WAYBACK_OMNIVORE_APIKEY`: Omnivore API key. 12 | -------------------------------------------------------------------------------- /docs/integrations/omnivore.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 发布到Omnivore 3 | --- 4 | 5 | Omnivore 可让您保存和整理文章、简报和文档以供日后阅读。它还可与流行的知识管理系统同步，并提供文本转语音和无干扰功能。 6 | 7 | ## 配置 8 | 9 | 创建 API 密钥，将密钥放置在环境或配置文件中： 10 | 11 | - `WAYBACK_OMNIVORE_APIKEY`: Omnivore API key. 12 | -------------------------------------------------------------------------------- /docs/integrations/playback.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Playback in wayback 3 | --- 4 | 5 | Wayback now supports playback from multiple archiving services. For [IPFS](./ipfs.md) and [Telegraph](./telegraph.md), you will need to set up self-hosted services, which can be Meilisearch or GitHub Issues. 6 | 7 | The following archiving services are currently supported for playback in wayback: 8 | 9 | - [Google Cache](https://webcache.googleusercontent.com/) 10 | - [Internet Archive](https://web.archive.org/) 11 | - [IPFS](https://ipfs.github.io/public-gateway-checker/) 12 | - [archive.today](https://archive.today/) 13 | - [Telegraph](https://telegra.ph/) 14 | - [Time Travel](http://timetravel.mementoweb.org/) 15 | 16 | ## How to config a playback service 17 | 18 | Place these keys in the environment or configuration file: 19 | 20 | For Meilisearch (recommended): 21 | 22 | - `PLAYBACK_MEILI_ENDPOINT`: Meilisearch API endpoint. 23 | - `PLAYBACK_MEILI_INDEXING`: Meilisearch indexing name, defaults to `capsules` (optional). 24 | - `PLAYBACK_MEILI_APIKEY`: Meilisearch admin API key (optional). 25 | 26 | For GitHub Issues: 27 | 28 | - `PLAYBACK_GITHUB_REPO`: GitHub repository to publish results. 29 | - `PLAYBACK_GITHUB_PAT`: GitHub Personal Access Token, which is used to reduce the rate limit for GitHub API requests. It requires the `repo` scope. 30 | 31 | ## Playback Integrations 32 | 33 | Playback is seamlessly integrated into Wayback to enhance the user experience. It currently supports playback from various platforms, including Discord, Mastodon, Matrix, Web, Slack, and Telegram. 34 | 35 | ### Web 36 | 37 | 1. To playback archived content, visit either the [Clear Web](https://wabarc.eu.org/) or [Onion Service](http://wabarcoww2bxmdbixj7sjwggv3fonh2rpflfiildegcydk5udkdckdyd.onion/). 38 | 2. Enter the URIs you want to play back. You can enter multiple URIs separated by commas or line breaks. 39 | 3. Click the button in the bottom left corner and wait a few moments for the playback to begin. 40 | 41 | ### Instant messaging tools 42 | 43 | Instant messaging tools such as Discord, Matrix, Slack and Telegram all have a playback feature that can be accessed using the same command: `/playback`. 44 | Enter `/playback https://example.com` and wait for the response. 45 | 46 | ### Others 47 | 48 | To use Playback on Mastodon, send a message to the bot with the keyword `/playback` at the beginning and the URI you wish to playback. 49 | Then, wait a moment for the bot to respond. 50 | -------------------------------------------------------------------------------- /docs/integrations/playback.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Playback in wayback 3 | --- 4 | 5 | Wayback现在支持从多个归档服务中进行回放。对于[IPFS](./ipfs.md)和[Telegraph](./telegraph.md)，您需要设置自托管服务，可以是Meilisearch或GitHub Issues。 6 | 7 | 目前，以下归档服务支持在wayback中进行回放： 8 | 9 | - [Google Cache](https://webcache.googleusercontent.com/) 10 | - [Internet Archive](https://web.archive.org/) 11 | - [IPFS](https://ipfs.github.io/public-gateway-checker/) 12 | - [archive.today](https://archive.today/) 13 | - [Telegraph](https://telegra.ph/) 14 | - [Time Travel](http://timetravel.mementoweb.org/) 15 | 16 | ## 如何配置回放服务 17 | 18 | 将这些密钥放置在环境或配置文件中： 19 | 20 | 对于Meilisearch（推荐）： 21 | 22 | - `PLAYBACK_MEILI_ENDPOINT`：Meilisearch API端点。 23 | - `PLAYBACK_MEILI_INDEXING`：Meilisearch索引名称，默认为`capsules`（可选）。 24 | - `PLAYBACK_MEILI_APIKEY`：Meilisearch管理员API密钥（可选）。 25 | 26 | 对于GitHub Issues： 27 | 28 | - `PLAYBACK_GITHUB_REPO`：GitHub存储库以发布结果。 29 | - `PLAYBACK_GITHUB_PAT`：GitHub个人访问令牌，用于降低GitHub API请求的速率限制。它需要`repo`作用域。 30 | 31 | ## 回放集成 32 | 33 | 回放无缝集成到Wayback中，以增强用户体验。它目前支持从各种平台进行回放，包括Discord、Mastodon、Matrix、Web、Slack和Telegram。 34 | 35 | ### Web 36 | 37 | 要回放归档内容，请访问[Clear Web](https://wabarc.eu.org/)或[Onion Service](http://wabarcoww2bxmdbixj7sjwggv3fonh2rpflfiildegcydk5udkdckdyd.onion/)。 38 | 输入要回放的URI。您可以输入多个URI，用逗号或换行符分隔。 39 | 单击左下角的按钮，等待几秒钟，然后开始回放。 40 | 41 | ### 即时通讯工具 42 | 43 | 即时通讯工具，如Discord、Matrix、Slack和Telegram，都具有回放功能，可以使用相同的命令访问：`/playback`。 44 | 输入`/playback https://example.com`，然后等待响应。 45 | 46 | ### 其他 47 | 48 | 要在Mastodon上使用Playback，请向机器人发送带有关键字`/playback`和要回放的URI的消息。 49 | 然后，等一会儿，机器人就会回复。 50 | -------------------------------------------------------------------------------- /docs/integrations/slack.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive with Slack 3 | --- 4 | 5 | ## How to build a Slack App 6 | 7 | Steps to create a new app: 8 | 9 | 1. Open [Slack API](https://api.slack.com/apps). 10 | 2. Click "Create New App" and "From Scratch". 11 | 2. Generate an App-Level Token with the `connections:write` scope. 12 | 3. Enable Socket Mode. 13 | 4. Enable Events 14 | - Subscribe to bot events: `app_mention` and `message.im`. 15 | - Subscribe to events on behalf of users: `message.im`. 16 | 5. Setting OAuth & Permissions User Token Scopes: `chat:write`, `files:write`. 17 | 6. Install the app to your workspace and obtain the `Bot User OAuth Token`. 18 | 7. In the App Home section, check `Allow users to send Slash commands and messages from the messages tab`. 19 | 8. Optionally, create a channel for publishing and note down the `Channel ID` by viewing the channel details. 20 | 21 | ## Configuration 22 | 23 | After creating a new app, you will have the `Bot User OAuth Token` and `Channel ID`. 24 | 25 | Next, place these keys in the environment or configuration file: 26 | 27 | - `WAYBACK_SLACK_BOT_TOKEN`: Bot User OAuth Token. 28 | - `WAYBACK_SLACK_CHANNEL`: Channel ID for publishing (optional). 29 | - `WAYBACK_SLACK_HELPTEXT`: Provide a help message for users to reference (optional). 30 | 31 | ## Further reading 32 | 33 | - [Slack API Documentation](https://api.slack.com/) 34 | -------------------------------------------------------------------------------- /docs/integrations/slack.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Slack 3 | --- 4 | 5 | ## 如何构建一个Slack App 6 | 7 | 创建新应用的步骤如下： 8 | 9 | 1. 打开[Slack API](https://api.slack.com/apps)。 10 | 2. 点击“创建新应用”和“从头开始”。 11 | 2. 生成具有`connections:write`范围的应用级令牌。 12 | 3. 启用Socket模式。 13 | 4. 启用事件 14 | - 订阅机器人事件：`app_mention`和`message.im`。 15 | - 代表用户订阅事件：`message.im`。 16 | 5. 设置OAuth和权限用户令牌范围：`chat:write`，`files:write`。 17 | 6. 将应用程序安装到您的工作区，并获取`Bot User OAuth Token`。 18 | 7. 在“应用主页”部分，勾选“允许用户从消息选项卡发送斜杠命令和消息”。 19 | 8. 可选地，创建一个用于发布的频道，并通过查看频道详情记录`Channel ID`。 20 | 21 | ## 配置 22 | 23 | 创建新应用后，您将获得`Bot User OAuth Token`和`Channel ID`。 24 | 25 | 接下来，将这些密钥放置在环境或配置文件中： 26 | 27 | - `WAYBACK_SLACK_BOT_TOKEN`：Bot User OAuth Token。 28 | - `WAYBACK_SLACK_CHANNEL`：用于发布的频道ID（可选）。 29 | - `WAYBACK_SLACK_HELPTEXT`：提供帮助消息供用户参考（可选）。 30 | 31 | ## 相关资料 32 | 33 | - [Slack API文档](https://api.slack.com/) 34 | -------------------------------------------------------------------------------- /docs/integrations/telegram.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive with Telegram 3 | --- 4 | 5 | ## How to build a Telegram Bot 6 | 7 | Steps to create a new bot: 8 | 9 | 1. Open the Telegram app and search for the BotFather bot. 10 | 2. Start a chat with BotFather by clicking the "Start" button. 11 | 3. Send the command `/newbot` to BotFather to create a new bot. 12 | 4. Follow the instructions from BotFather and provide a name and username for your bot. 13 | 5. After creating the bot, BotFather will provide you with a token. 14 | 6. To test your bot, open a chat with your bot by searching for its username and send a message. 15 | 7. You can also customize your bot by sending commands to BotFather, such as `/setdescription` and `/setuserpic`. 16 | 17 | Optionally, you can also create a channel for publishing. 18 | 19 | ## Configuration 20 | 21 | After creating a new bot, you will have the `Bot API Token`. 22 | 23 | Next, place these keys in the environment or configuration file: 24 | 25 | - `WAYBACK_TELEGRAM_TOKEN`: Bot API Token. 26 | - `WAYBACK_TELEGRAM_CHANNEL`: Channel ID for publishing (optional). 27 | - `WAYBACK_TELEGRAM_HELPTEXT`: Provide a help message for users to reference (optional). 28 | 29 | ## Further reading 30 | 31 | - [Bots: An introduction for developers](https://core.telegram.org/bots) 32 | -------------------------------------------------------------------------------- /docs/integrations/telegram.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Telegram 3 | --- 4 | 5 | ## 如何构建一个Telegram机器人 6 | 7 | 创建新机器人的步骤如下： 8 | 9 | 1. 打开Telegram应用并搜索BotFather机器人。 10 | 2. 点击“开始”按钮与BotFather开始聊天。 11 | 3. 向BotFather发送命令`/newbot`来创建一个新机器人。 12 | 4. 按照BotFather的指示提供机器人的名称和用户名。 13 | 5. 创建机器人后，BotFather将为您提供一个令牌。 14 | 6. 要测试您的机器人，请通过搜索其用户名打开与您的机器人的聊天，并发送一条消息。 15 | 7. 您还可以通过向BotFather发送命令，例如`/setdescription`和`/setuserpic`，来定制您的机器人。 16 | 17 | 可选地，您还可以创建一个用于发布的频道。 18 | 19 | ## 配置 20 | 21 | 创建新机器人后，您将获得`Bot API Token`。 22 | 23 | 接下来，将这些密钥放置在环境或配置文件中： 24 | 25 | - `WAYBACK_TELEGRAM_TOKEN`：Bot API Token。 26 | - `WAYBACK_TELEGRAM_CHANNEL`：用于发布的频道ID（可选）。 27 | - `WAYBACK_TELEGRAM_HELPTEXT`：提供帮助消息供用户参考（可选）。 28 | 29 | ## 相关资料 30 | 31 | - [开发人员入门：Bots](https://core.telegram.org/bots) 32 | -------------------------------------------------------------------------------- /docs/integrations/telegraph.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Wayback to Telegraph 3 | --- 4 | 5 | [Telegraph](https://telegra.ph/) – a publishing tool that lets you create richly formatted posts with photos and all sorts of embedded stuff. 6 | 7 | Wayback relies on [Telegraph](https://telegra.ph/) as an upstream service storing sanitized web pages. 8 | You can enable or disable this feature using the `--ph` flag or the `WAYBACK_ENABLE_PH` environment variable, which is enabled by default. 9 | The content is saved using a new anonymous account generated by wayback each time, and is not associated with the requester. 10 | 11 | The code for wayback's implementation of the Telegraph integration can be found in the [wabarc/telegra.ph](https://github.com/wabarc/telegra.ph) repository. 12 | -------------------------------------------------------------------------------- /docs/integrations/telegraph.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 归档到Telegraph 3 | --- 4 | 5 | [Telegraph](https://telegra.ph/)是一个内容发布工具，可以让您创建具有照片和各种嵌入式内容的丰富格式的帖子。 6 | 7 | Wayback依赖于[Telegraph](https://telegra.ph/)作为上游服务来存储已经过处理的网页。 8 | 您可以使用`--ph`标志或`WAYBACK_ENABLE_PH`环境变量启用或禁用此功能，默认情况下已启用。 9 | 内容使用wayback每次生成的新匿名帐户保存，与请求者无关。 10 | 11 | Wayback实现Telegraph集成的代码可以在[wabarc/telegra.ph](https://github.com/wabarc/telegra.ph)存储库中找到。 12 | -------------------------------------------------------------------------------- /docs/integrations/twitter.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Twitter 3 | --- 4 | 5 | ## 如何构建一个Twitter机器人 6 | 7 | 创建新机器人的步骤如下： 8 | 9 | 1. 创建Twitter帐户或使用现有帐户。如果需要创建新帐户，请转到[注册页面](https://twitter.com/signup)。 10 | 2. 转到[项目和应用程序](https://developer.twitter.com/en/portal/projects-and-apps)并使用您的Twitter帐户登录。 11 | 3. 点击“创建应用程序”并填写必要的信息，例如应用程序名称、描述和网站。 12 | 4. 在“应用程序”页面，单击“密钥和令牌”选项卡。 13 | 5. 在“客户密钥”部分下，单击“生成”按钮以生成您的应用程序的“客户密钥”和“客户密钥密钥”。 14 | 6. 滚动到“访问令牌和访问令牌密钥”部分，然后单击“生成”按钮以生成您的应用程序的“访问令牌”和“访问令牌密钥”。 15 | 7. 复制“客户密钥”、“客户密钥密钥”、“访问令牌”和“访问令牌密钥”，并将它们存储在安全位置。您稍后需要它们来验证您的机器人。 16 | 8. 放置您机器人的环境或配置文件，确保包括使用步骤5和6生成的密钥和令牌对API进行身份验证。 17 | 9. 测试您的机器人，并确保它按预期工作。 18 | 10. 一旦您的机器人准备就绪，请将其部署到托管服务或服务器中，以便它可以持续运行。 19 | 11. 监视您的机器人的活动和性能，并进行任何必要的调整。 20 | 21 | 创建Twitter机器人时需要注意的一些事项： 22 | 23 | - 遵守Twitter的规则和政策，违反这些规则可能会导致您的机器人被暂停或禁止。 24 | - 不要使用您的机器人向其他Twitter用户发送垃圾邮件或骚扰信息。 25 | - 确保您的机器人具有明确的目的，并且不会在平台上产生不必要的噪音。 26 | - 监视您的机器人的活动，并根据需要进行调整，以改善其性能和行为。 27 | 28 | ## 配置 29 | 30 | 创建新机器人后，您将获得`Consumer Keys`、`Consumer Secret`、`Access Token`和`Access Token Secret`。 31 | 32 | 接下来，将这些密钥放置在环境或配置文件中： 33 | 34 | - `WAYBACK_TWITTER_CONSUMER_KEY`：您的Twitter应用程序的客户密钥。 35 | - `WAYBACK_TWITTER_CONSUMER_SECRET`：您的Twitter应用程序的客户密钥密钥。 36 | - `WAYBACK_TWITTER_ACCESS_TOKEN`：您的Twitter应用程序的访问令牌。 37 | - `WAYBACK_TWITTER_ACCESS_SECRET`：您的Twitter应用程序的访问令牌密钥。 38 | -------------------------------------------------------------------------------- /docs/integrations/web.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive with Web Service 3 | --- 4 | 5 | ## How to build a Web Service 6 | 7 | Wayback supports serving both **Clear Web** and **Onion Service**. If the Tor binary is missing, the Onion Service feature will be ignored. 8 | 9 | ## Configuration 10 | 11 | After installation, you need to provide the required keys by placing them in the environment or configuration file. This allows you to customize the configuration based on your needs. 12 | 13 | - `WAYBACK_LISTEN_ADDR`: The listen address for the HTTP server, defaults to `0.0.0.0:8964`. 14 | - `WAYBACK_ONION_PRIVKEY`: The private key for Onion Service. 15 | - `WAYBACK_ONION_LOCAL_PORT`: Local port for Onion Service, also support for a reverse proxy. This is ignored if `WAYBACK_LISTEN_ADDR` is set. 16 | - `WAYBACK_ONION_REMOTE_PORTS`: Remote ports for Onion Service, e.g. `WAYBACK_ONION_REMOTE_PORTS=80,81`. 17 | 18 | Note: To run a Onion Service for the first time, you need to keep the `private key`, which can be seen from the log output. 19 | -------------------------------------------------------------------------------- /docs/integrations/web.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Web Service 3 | --- 4 | 5 | ## 如何构建Web服务 6 | 7 | Wayback支持服务于**明文Web**和**Onion服务**。如果缺少Tor二进制文件，则将忽略Onion服务功能。 8 | 9 | ## 配置 10 | 11 | 安装完成后，您需要通过将其放置在环境或配置文件中来提供所需的密钥。这允许您根据需要自定义配置。 12 | 13 | - `WAYBACK_LISTEN_ADDR`：HTTP服务器的侦听地址，默认为`0.0.0.0:8964`。 14 | - `WAYBACK_ONION_PRIVKEY`：Onion服务的私钥。 15 | - `WAYBACK_ONION_LOCAL_PORT`：Onion服务的本地端口，也支持反向代理。如果设置了`WAYBACK_LISTEN_ADDR`，则忽略此设置。 16 | - `WAYBACK_ONION_REMOTE_PORTS`：Onion服务的远程端口，例如`WAYBACK_ONION_REMOTE_PORTS=80,81`。 17 | 18 | 注意：要首次运行Onion服务，您需要保留`私钥`，该私钥可以从日志输出中看到。 19 | -------------------------------------------------------------------------------- /docs/integrations/xmpp.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Interactive with XMPP 3 | --- 4 | 5 | ## How to build a XMPP Service 6 | 7 | To create an XMPP account, you need to find a client and a public server. Here are some recommended collections of XMPP servers to help you get started. 8 | 9 | - [Directory 404](https://xmpp.404.city/) 10 | - [Public XMPP servers](https://list.jabber.at/) 11 | - [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top) 12 | 13 | ## Configuration 14 | 15 | To use the XMPP service, you will need to set the following environment variables or configuration file: 16 | 17 | - `WAYBACK_IRC_JID`: The JID for the XMPP client (required). 18 | - `WAYBACK_IRC_PASSWORD`: The password for the XMPP client (required). 19 | - `WAYBACK_IRC_NOTLS`: Connect to XMPP server without TLS (optional). 20 | - `WAYBACK_IRC_HELPTEXT`: The help text for XMPP command (optional). 21 | 22 | ## Further reading 23 | - [XMPP | The universal messaging standard](https://xmpp.org/) 24 | 25 | -------------------------------------------------------------------------------- /docs/integrations/xmpp.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: XMPP 3 | --- 4 | 5 | ## 如何构建XMPP服务 6 | 7 | 创建 XMPP 账户，您需要找到一个客户端和一个公共服务器。以下是一些推荐的 XMPP 服务器集合，可帮助您入门。 8 | 9 | - [Directory 404](https://xmpp.404.city/) 10 | - [Public XMPP servers](https://list.jabber.at/) 11 | - [Tracking the progress of OMEMO integration in XMPP clients](https://omemo.top) 12 | 13 | ## 配置 14 | 15 | 为了使用XMPP服务，您需要设置以下环境变量或配置文件： 16 | 17 | - `WAYBACK_IRC_JID`: XMPP客户端的JID（必填）。 18 | - `WAYBACK_IRC_PASSWORD`: XMPP客户端的密码（必填）。 19 | - `WAYBACK_IRC_NOTLS`: 连接到XMPP服务器时不使用TLS加密（可选）。 20 | - `WAYBACK_IRC_HELPTEXT`: XMPP命令的帮助文本（可选）。 21 | 22 | ## 相关资料 23 | 24 | - [XMPP | The universal messaging standard](https://xmpp.org/) 25 | 26 | -------------------------------------------------------------------------------- /docs/privacy.md: -------------------------------------------------------------------------------- 1 | We takes your privacy seriously. To better protect your privacy we provide this privacy policy notice explaining the way your personal information is collected and used. 2 | 3 | ### Collection of Routine Information 4 | 5 | We have a minimum-collection policy. Aside from essential data required to keep the service running, we are not collecting additional user or tracking data. 6 | 7 | #### IP logging 8 | 9 | Server log files can contain client IP addresses and user agent strings from connecting computers. 10 | 11 | ### Contact Information 12 | 13 | For any questions or concerns regarding the privacy policy, please send us an email to [wabarc@tuta.io](mailto:wabarc@tuta.io). 14 | -------------------------------------------------------------------------------- /docs/privacy.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 隐私条款 3 | --- 4 | 5 | 我们非常重视您的隐私。为了更好地保护您的隐私，我们提供本隐私政策通知，解释您的个人信息是如何被收集和使用的。 6 | 7 | ### 常规信息收集 8 | 9 | 我们实行最少收集政策。除了必要的数据以保持服务运行外，我们不收集额外的用户或追踪数据。 10 | 11 | #### IP日志记录 12 | 13 | 服务器日志文件可能包含连接计算机的客户端IP地址和用户代理字符串。 14 | 15 | ### 联系信息 16 | 17 | 如果您有任何关于隐私政策的问题或疑虑，请发送电子邮件到[wabarc@tuta.io](mailto:wabarc@tuta.io)与我们联系。 18 | -------------------------------------------------------------------------------- /docs/resources.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Resources 3 | --- 4 | 5 | Welcome to our list of resources about archiving services! 6 | 7 | In this collection, we have curated a variety of tools, platforms, and projects that offer archiving solutions for various types of data and content. 8 | Archiving services are important for preserving information and ensuring its availability for future generations. 9 | Whether you are a researcher, a historian, or interested in preserving your own content, these resources can help you achieve your goals. 10 | From the popular Wayback Machine to lesser-known platforms, there is something here for everyone. 11 | 12 | ## Organizations 13 | 14 | - [Internet Archive](https://web.archive.org/) 15 | - [List of Web archiving initiatives - Wikipedia](https://en.wikipedia.org/wiki/List_of_Web_archiving_initiatives) 16 | 17 | ## Collections 18 | 19 | - [Awesome Web Archiving - IIPC](https://github.com/iipc/awesome-web-archiving) 20 | - [Web Archiving Community - ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) 21 | 22 | ## Wiki 23 | 24 | - [Web archiving](https://en.wikipedia.org/wiki/Web_archiving) 25 | 26 | ## Tools 27 | 28 | - [Browserless](https://www.browserless.io/): Web Automation & Headless Browser Automation Tool. 29 | 30 | -------------------------------------------------------------------------------- /docs/resources.zh.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Resources 3 | --- 4 | 5 | 欢迎来到我们关于存档服务的资源列表！ 6 | 7 | 在这个收藏中，我们精选了各种工具、平台和项目，为不同类型的数据和内容提供存档解决方案。存档服务对于保留信息并确保其为未来世代所用是非常重要的。无论您是研究人员、历史学家还是有意保存自己内容的人，这些资源都可以帮助您实现您的目标。从流行的Wayback Machine到较少知名的平台，这里有适合每个人的东西。 8 | 9 | ## 组织 10 | 11 | - [互联网档案馆](https://web.archive.org/) 12 | - [Web存档计划列表 - 维基百科](https://en.wikipedia.org/wiki/List_of_Web_archiving_initiatives) 13 | 14 | ## 收藏 15 | 16 | - [Awesome Web Archiving - IIPC](https://github.com/iipc/awesome-web-archiving) 17 | - [Web Archiving Community - ArchiveBox](https://github.com/ArchiveBox/ArchiveBox/wiki/Web-Archiving-Community) 18 | 19 | ## 维基 20 | 21 | - [Web存档](https://en.wikipedia.org/wiki/Web_archiving) 22 | 23 | ## 工具 24 | 25 | - [Browserless](https://www.browserless.io/)：Web自动化和无头浏览器自动化工具。 26 | 27 | -------------------------------------------------------------------------------- /docs/service.zh.md: -------------------------------------------------------------------------------- 1 | ## 配置 2 | 3 | Wayback提供了两种配置格式：**环境变量**和**配置文件**，它们使用相同的键。配置文件采用INI文件格式，并可在[wayback.conf](https://github.com/wabarc/wayback/blob/main/wayback.conf)中找到。环境变量可以使用前缀`WAYBACK_`后跟配置键名来设置。 4 | 5 | 例如，要同时提供Discord和Telegram机器人服务，您可以运行命令`wayback -d discord -d telegram`。`-d`标志后跟平台的名称，应该是小写的。 6 | 7 | 要查看有关可用CLI标志的其他信息，请运行命令`wayback --help`。 8 | 9 | ## 服务 10 | 11 | Wayback可以与各种消息平台集成，包括Discord、IRC、Mastodon、Matrix、Slack、Telegram、Twitter和Web，作为响应用户查询的机器人。 12 | 13 | 有关如何为每个平台创建机器人的详细说明，请参见以下链接： 14 | 15 | - [Discord](integrations/discord.md) 16 | - [IRC](integrations/irc.md) 17 | - [Mastodon](integrations/mastodon.md) 18 | - [Matrix](integrations/matrix.md) 19 | - [Slack](integrations/slack.md) 20 | - [Telegram](integrations/telegram.md) 21 | - [Twitter](integrations/twitter.md) 22 | - [Web](integrations/web.md) 23 | - [XMPP](integrations/xmpp.md) 24 | 25 | 请注意，您需要在各自的平台上设置帐户并获取必要的凭据，例如访问令牌，才能将Wayback用作机器人。 26 | 27 | ## 发布 28 | 29 | Wayback的集成服务提供了将存档结果发布到各种消息和协作平台的功能。发布的结果不包括任何请求者信息，以确保隐私。 30 | 31 | **当您放置必要的配置时，该项的发布功能将自动启用。** 32 | 33 | 有关如何配置发布通道的详细说明，请参见以下链接： 34 | 35 | - [IRC](integrations/irc.md) 36 | - [Discord](integrations/discord.md) 37 | - [GitHub Issues](integrations/github.md) 38 | - [Mastodon](integrations/mastodon.md) 39 | - [Matrix](integrations/matrix.md) 40 | - [Meilisearch](integrations/meilisearch.md) 41 | - [Nostr](integrations/nostr.md) 42 | - [Notion](integrations/notion.md) 43 | - [Slack](integrations/slack.md) 44 | - [Telegram](integrations/telegram.md) 45 | - [Twitter](integrations/twitter.md) 46 | 47 | 每个平台都有自己的配置要求，因此请务必仔细按照说明操作，以确保成功发布存档结果。 48 | -------------------------------------------------------------------------------- /docs/troubleshooting.md: -------------------------------------------------------------------------------- 1 | ## archive.today is unavailable? 2 | 3 | Sometimes archive.today enforces a strict CAPTCHA policy, which may cause a request exception. 4 | To solve this, you can manually bypass the CAPTCHA in your browser and retrieve the `cf_clearance` cookie item. 5 | Then, set this item as a system environment variable named `ARCHIVE_COOKIE`. For example, you can set the value 6 | of `ARCHIVE_COOKIE` as `cf_clearance=ab170e4acc49bbnsaff8687212d2cdb987e5b798-1234542375-KDUKCHU`. 7 | 8 | ## Disable JavaScript for specific URIs when archiving with IPFS? 9 | 10 | To disable JavaScript when saving a webpage, you can set environment variables `DISABLEJS_URIS`. The values should be in the following format: 11 | 12 | ```sh 13 | export DISABLEJS_URIS=wikipedia.org|eff.org/tags 14 | ``` 15 | 16 | This will disable JavaScript for the entire site `wikipedia.org` or only for the path `eff.org/tags` if it matches. 17 | 18 | ## How can I keep the Tor Hidden Service hostname? 19 | 20 | When running the `wayback` service for the first time, it is important to keep the private key from the output message 21 | (the key is the part after `private key:`). The next time you run the wayback service, you can use the key by providing 22 | it to the `--tor-key` option or setting it as the `WAYBACK_ONION_PRIVKEY` environment variable. 23 | 24 | ```text 25 | [INFO] Web: Important: remember to keep the private key: d005473a611d2b23e54d6446dfe209cb6c52ddd698818d1233b1d750f790445fcfb5ece556fe5ee3b4724ac6bea7431898ee788c6011febba7f779c85845ae87 26 | ``` 27 | 28 | -------------------------------------------------------------------------------- /docs/troubleshooting.zh.md: -------------------------------------------------------------------------------- 1 | ## archive.today不可用？ 2 | 3 | 有时，archive.today会实施严格的验证码策略，这可能会导致请求异常。为了解决这个问题，您可以在浏览器中手动绕过验证码并检索`cf_clearance` cookie项。然后，将此项设置为名为`ARCHIVE_COOKIE`的系统环境变量。例如，您可以将`ARCHIVE_COOKIE`的值设置为`cf_clearance=ab170e4acc49bbnsaff8687212d2cdb987e5b798-1234542375-KDUKCHU`。 4 | 5 | ## 在使用IPFS存档时如何为特定的URI禁用JavaScript？ 6 | 7 | 要在保存网页时禁用JavaScript，您可以设置环境变量`DISABLEJS_URIS`。值应按照以下格式： 8 | 9 | ```sh 10 | export DISABLEJS_URIS=wikipedia.org|eff.org/tags 11 | ``` 12 | 13 | 这将为整个网站`wikipedia.org`禁用JavaScript，或者仅在匹配时为路径`eff.org/tags`禁用。 14 | 15 | ## 如何保留Tor隐藏服务主机名？ 16 | 17 | 第一次运行`wayback`服务时，保留来自输出消息的私钥非常重要（私钥是`private key:`之后的部分）。下次运行wayback服务时，您可以通过将其提供给`--tor-key`选项或将其设置为`WAYBACK_ONION_PRIVKEY`环境变量来使用密钥。 18 | 19 | ```text 20 | [INFO] Web: Important: remember to keep the private key: d005473a611d2b23e54d6446dfe209cb6c52ddd698818d1233b1d750f790445fcfb5ece556fe5ee3b4724ac6bea7431898ee788c6011febba7f779c85845ae87 21 | ``` 22 | 23 | -------------------------------------------------------------------------------- /entity/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package entity contains all data structures used by the application. 7 | */ 8 | package entity // import "github.com/wabarc/wayback/entity" 9 | -------------------------------------------------------------------------------- /entity/entity.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package entity // import "github.com/wabarc/entity" 6 | -------------------------------------------------------------------------------- /entity/playback.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package entity // import "github.com/wabarc/entity" 6 | 7 | // EntityPlayback represents a keyword for playback entity. 8 | const EntityPlayback = "playback" 9 | 10 | // Playback represents a Playback in the application. 11 | type Playback struct { 12 | ID uint64 13 | Source string 14 | } 15 | -------------------------------------------------------------------------------- /errors/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package errors handles errors. 7 | */ 8 | package errors // import "github.com/wabarc/wayback/errors" 9 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package errors // import "github.com/wabarc/wayback/errors" 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // Error represents an error 14 | type Error struct { 15 | message string 16 | args []interface{} 17 | } 18 | 19 | // Error returns the error message. 20 | func (e Error) Error() string { 21 | return fmt.Sprintf(e.message, e.args...) 22 | } 23 | 24 | // New returns error handler. 25 | func New(message string, args ...interface{}) *Error { 26 | return &Error{message: message, args: args} 27 | } 28 | 29 | // Wrap returns an error annotating err with a stack trace at the point Wrap is called. 30 | func Wrap(err error, message string) error { 31 | return errors.Wrap(err, message) 32 | } 33 | 34 | // Is reports whether any error in err's chain matches target. 35 | func Is(err, target error) bool { 36 | return errors.Is(err, target) 37 | } 38 | -------------------------------------------------------------------------------- /ingress/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package ingress provides functionality for registering services. 7 | 8 | The ingress package allows you to register wayback services. 9 | 10 | To use the ingress package, import the package and its dependencies, 11 | such as register, as shown in the following example: 12 | 13 | package main 14 | 15 | import ( 16 | "github.com/wabarc/wayback/ingress" 17 | "github.com/wabarc/wayback/publish" 18 | "github.com/wabarc/wayback/service" 19 | _ "github.com/wabarc/wayback/ingress/register" 20 | ) 21 | 22 | func main() { 23 | // Initialize the publish service with configuration options and a context. 24 | opts := &config.Options{} 25 | ctx := context.Background() 26 | pub := publish.New(ctx, opts) 27 | go pub.Start() 28 | defer pub.Stop() 29 | 30 | ingress.Init(opts) 31 | 32 | // Use the publish service to publish data. 33 | // ... 34 | 35 | // Initialize services with configuration options and a context. 36 | err := service.Serve(ctx, service.Options{}) 37 | // ... 38 | } 39 | */ 40 | package ingress // import "github.com/wabarc/wayback/ingress" 41 | -------------------------------------------------------------------------------- /ingress/init.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package ingress // import "github.com/wabarc/wayback/ingress" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | ) 10 | 11 | // Init initializes functions with the given configuration options. 12 | func Init(opts *config.Options) { 13 | initClient(opts) 14 | } 15 | -------------------------------------------------------------------------------- /ingress/register/publish.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package register // import "github.com/wabarc/wayback/ingress/register" 6 | 7 | import ( 8 | _ "github.com/wabarc/wayback/publish/discord" 9 | _ "github.com/wabarc/wayback/publish/github" 10 | _ "github.com/wabarc/wayback/publish/mastodon" 11 | _ "github.com/wabarc/wayback/publish/matrix" 12 | _ "github.com/wabarc/wayback/publish/meili" 13 | _ "github.com/wabarc/wayback/publish/nostr" 14 | _ "github.com/wabarc/wayback/publish/notion" 15 | _ "github.com/wabarc/wayback/publish/omnivore" 16 | _ "github.com/wabarc/wayback/publish/relaychat" 17 | _ "github.com/wabarc/wayback/publish/slack" 18 | _ "github.com/wabarc/wayback/publish/telegram" 19 | _ "github.com/wabarc/wayback/publish/twitter" 20 | ) 21 | -------------------------------------------------------------------------------- /ingress/register/service.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package register // import "github.com/wabarc/wayback/ingress/register" 6 | 7 | import ( 8 | _ "github.com/wabarc/wayback/service/discord" 9 | _ "github.com/wabarc/wayback/service/httpd" 10 | _ "github.com/wabarc/wayback/service/mastodon" 11 | _ "github.com/wabarc/wayback/service/matrix" 12 | _ "github.com/wabarc/wayback/service/relaychat" 13 | _ "github.com/wabarc/wayback/service/slack" 14 | _ "github.com/wabarc/wayback/service/telegram" 15 | _ "github.com/wabarc/wayback/service/twitter" 16 | _ "github.com/wabarc/wayback/service/xmpp" 17 | ) 18 | -------------------------------------------------------------------------------- /metrics/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package metrics exposes wayback service status. 7 | */ 8 | package metrics // import "github.com/wabarc/wayback/metrics" 9 | -------------------------------------------------------------------------------- /metrics/exports.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package metrics // import "github.com/wabarc/wayback/metrics" 6 | 7 | import ( 8 | "bytes" 9 | "strings" 10 | 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/prometheus/common/expfmt" 13 | "github.com/wabarc/logger" 14 | ) 15 | 16 | // Gather holds configured collector. 17 | var Gather *Collector 18 | 19 | // Export export metrics by the given labels for matching. 20 | func (c *Collector) Export(labels ...string) string { 21 | logger.Debug("export metrics family: %#v", prometheus.DefaultRegisterer) 22 | 23 | var gatherer = prometheus.DefaultGatherer 24 | var protobufs, err = gatherer.Gather() 25 | if err != nil { 26 | logger.Error("gather metrics family failed: %v", err) 27 | } 28 | 29 | var s string 30 | for _, pb := range protobufs { 31 | var buf bytes.Buffer 32 | if _, err := expfmt.MetricFamilyToText(&buf, pb); err != nil { 33 | logger.Error("export to text failed: %v", err) 34 | } 35 | logger.Debug("string: %v\nname: %v\nhelp: %v\ntype: %v\nmetric: %v\nvalue: %v", 36 | buf.String(), pb.GetName(), pb.GetHelp(), pb.GetType(), pb.GetMetric(), pb.GetMetric()[0].GetGauge().GetValue()) 37 | if match(pb.GetName(), labels...) { 38 | s += buf.String() 39 | } 40 | } 41 | 42 | return s 43 | } 44 | 45 | func match(name string, labels ...string) bool { 46 | for _, label := range labels { 47 | if strings.HasPrefix(name, label) { 48 | return true 49 | } 50 | } 51 | return false 52 | } 53 | -------------------------------------------------------------------------------- /pooling/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package pooling implements the wayback workers pool. 7 | */ 8 | package pooling // import "github.com/wabarc/wayback/pooling" 9 | -------------------------------------------------------------------------------- /pooling/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package pooling // import "github.com/wabarc/wayback/pooling" 6 | 7 | import ( 8 | "time" 9 | ) 10 | 11 | // Options represents configuration for pooling. 12 | type Options struct { 13 | Timeout time.Duration // Timeout specifies the maximum amount of time to wait for an operation to complete. 14 | MaxRetries uint64 // MaxRetries specifies the maximum number of times to retry the operation in case of failure. 15 | Capacity int // Capacity specifies the maximum number of items that can be processed simultaneously. 16 | } 17 | 18 | // Option is a function that modifies the provided Options instance. 19 | type Option func(*Options) 20 | 21 | // Timeout returns an Option function that sets the timeout value for the Options. 22 | // The given timeout value will be applied when the returned function is called. 23 | func Timeout(t time.Duration) Option { 24 | return func(opts *Options) { 25 | opts.Timeout = t 26 | } 27 | } 28 | 29 | // MaxRetries returns an Option function that sets the maximum retries value for the Options. 30 | // The given maximum retries value will be applied when the returned function is called. 31 | func MaxRetries(r uint64) Option { 32 | return func(opts *Options) { 33 | opts.MaxRetries = r 34 | } 35 | } 36 | 37 | // Capacity returns an Option function that sets the capacity value for the Options. 38 | // The given capacity value will be applied when the returned function is called. 39 | func Capacity(c int) Option { 40 | return func(opts *Options) { 41 | opts.Capacity = c 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pooling/options_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package pooling // import "github.com/wabarc/wayback/pooling" 6 | 7 | import ( 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestTimeoutOption(t *testing.T) { 13 | opts := &Options{} 14 | 15 | timeout := time.Second 16 | Timeout(timeout)(opts) 17 | 18 | if opts.Timeout != timeout { 19 | t.Errorf("Expected timeout to be seconds, but got %v", opts.Timeout) 20 | } 21 | } 22 | 23 | func TestMaxRetriesOption(t *testing.T) { 24 | opts := &Options{} 25 | 26 | MaxRetries(3)(opts) 27 | 28 | if opts.MaxRetries != 3 { 29 | t.Errorf("Expected max retries to be 3, but got %v", opts.MaxRetries) 30 | } 31 | } 32 | 33 | func TestCapacityOption(t *testing.T) { 34 | opts := &Options{} 35 | 36 | Capacity(100)(opts) 37 | 38 | if opts.Capacity != 100 { 39 | t.Errorf("Expected capacity to be 100, but got %v", opts.Capacity) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /publish/discord/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package discord // import "github.com/wabarc/wayback/publish/discord" 6 | -------------------------------------------------------------------------------- /publish/discord/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package discord // import "github.com/wabarc/wayback/publish/discord" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagDiscord, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToDiscordChannel() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | The publish package provides a publishing service and requires initialization by the caller. 7 | 8 | To use the publish package, import the package and its dependencies, such as register, as shown in the following example: 9 | 10 | package main 11 | 12 | import ( 13 | _ "github.com/wabarc/wayback/ingress" 14 | "github.com/wabarc/wayback/publish" 15 | ) 16 | 17 | func main() { 18 | // Initialize the publish service with configuration options and a context. 19 | opts := &config.Options{} 20 | ctx := context.Background() 21 | pub := publish.New(ctx, opts) 22 | go pub.Start() 23 | defer pub.Stop() 24 | 25 | // Use the publish service to publish data. 26 | // ... 27 | } 28 | */ 29 | package publish // import "github.com/wabarc/wayback/publish" 30 | -------------------------------------------------------------------------------- /publish/example.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package publish // import "github.com/wabarc/wayback/publish" 6 | 7 | import ( 8 | "github.com/wabarc/wayback" 9 | "github.com/wabarc/wayback/config" 10 | ) 11 | 12 | var Collects = []wayback.Collect{ 13 | { 14 | Arc: config.SLOT_IA, 15 | Dst: "https://web.archive.org/web/20211000000001/https://example.com/", 16 | Src: "https://example.com/", 17 | Ext: config.SLOT_IA, 18 | }, 19 | { 20 | Arc: config.SLOT_IS, 21 | Dst: "http://archive.today/abcdE", 22 | Src: "https://example.com/", 23 | Ext: config.SLOT_IS, 24 | }, 25 | { 26 | Arc: config.SLOT_IP, 27 | Dst: "https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr", 28 | Src: "https://example.com/", 29 | Ext: config.SLOT_IP, 30 | }, 31 | { 32 | Arc: config.SLOT_PH, 33 | Dst: "http://telegra.ph/title-01-01", 34 | Src: "https://example.com/", 35 | Ext: config.SLOT_PH, 36 | }, 37 | } 38 | -------------------------------------------------------------------------------- /publish/github/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package github // import "github.com/wabarc/wayback/publish/github" 6 | -------------------------------------------------------------------------------- /publish/github/github_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package github // import "github.com/wabarc/wayback/publish/github" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/wabarc/helper" 16 | "github.com/wabarc/wayback/config" 17 | "github.com/wabarc/wayback/publish" 18 | "github.com/wabarc/wayback/reduxer" 19 | "github.com/wabarc/wayback/template/render" 20 | ) 21 | 22 | func TestToIssues(t *testing.T) { 23 | t.Setenv("WAYBACK_GITHUB_TOKEN", "foo") 24 | t.Setenv("WAYBACK_GITHUB_OWNER", "bar") 25 | t.Setenv("WAYBACK_GITHUB_REPO", "zoo") 26 | opts, _ := config.NewParser().ParseEnvironmentVariables() 27 | 28 | httpClient, mux, server := helper.MockServer() 29 | defer server.Close() 30 | 31 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 32 | w.Header().Set("Content-Type", "application/json") 33 | switch r.URL.Path { 34 | case "/repos/bar/zoo/issues": 35 | body, _ := io.ReadAll(r.Body) 36 | if !strings.Contains(string(body), config.SlotName(config.SLOT_IA)) { 37 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 38 | return 39 | } 40 | w.Header().Set("Status", "201 Created") 41 | fmt.Fprintln(w, `{"id": 1}`) 42 | default: 43 | fmt.Fprintln(w, `{}`) 44 | } 45 | }) 46 | 47 | gh := New(httpClient, opts) 48 | txt := render.ForPublish(&render.GitHub{Cols: publish.Collects, Data: reduxer.BundleExample()}).String() 49 | got := gh.toIssues(context.Background(), "", txt) 50 | if !got { 51 | t.Errorf("Unexpected create GitHub Issues got %t instead of %t", got, true) 52 | } 53 | } 54 | 55 | func TestShutdown(t *testing.T) { 56 | opts, _ := config.NewParser().ParseEnvironmentVariables() 57 | 58 | httpClient, _, server := helper.MockServer() 59 | defer server.Close() 60 | 61 | gh := New(httpClient, opts) 62 | err := gh.Shutdown() 63 | if err != nil { 64 | t.Errorf("Unexpected shutdown: %v", err) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /publish/github/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package github // import "github.com/wabarc/wayback/publish/github" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagGitHub, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.GitHubToken() != "" && opts.GitHubOwner() != "" { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/mastodon/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package mastodon // import "github.com/wabarc/wayback/publish/mastodon" 6 | -------------------------------------------------------------------------------- /publish/mastodon/mastodon_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package mastodon // import "github.com/wabarc/wayback/publish/mastodon" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "net/http" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/wabarc/helper" 15 | "github.com/wabarc/wayback/config" 16 | "github.com/wabarc/wayback/publish" 17 | "github.com/wabarc/wayback/template/render" 18 | ) 19 | 20 | func setMastodonEnv(t *testing.T) { 21 | t.Setenv("WAYBACK_MASTODON_KEY", "foo") 22 | t.Setenv("WAYBACK_MASTODON_SECRET", "bar") 23 | t.Setenv("WAYBACK_MASTODON_TOKEN", "zoo") 24 | } 25 | 26 | func TestToMastodon(t *testing.T) { 27 | setMastodonEnv(t) 28 | 29 | _, mux, server := helper.MockServer() 30 | defer server.Close() 31 | 32 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 33 | if r.Header.Get("Authorization") != "Bearer zoo" { 34 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 35 | return 36 | } 37 | if err := r.ParseForm(); err != nil { 38 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 39 | return 40 | } 41 | switch r.URL.Path { 42 | case "/api/v1/statuses": 43 | status := r.FormValue("status") 44 | if !strings.Contains(status, `title`) { 45 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 46 | return 47 | } 48 | fmt.Fprintln(w, `{"access_token": "zoo"}`) 49 | default: 50 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 51 | } 52 | }) 53 | 54 | t.Setenv("WAYBACK_MASTODON_SERVER", server.URL) 55 | opts, _ := config.NewParser().ParseEnvironmentVariables() 56 | 57 | mstdn := New(http.Client{}, opts) 58 | txt := render.ForPublish(&render.Telegram{Cols: publish.Collects}).String() 59 | got := mstdn.toMastodon(context.Background(), txt, "") 60 | if !got { 61 | t.Errorf("Unexpected publish toot got %t instead of %t", got, true) 62 | } 63 | } 64 | 65 | func TestShutdown(t *testing.T) { 66 | opts, _ := config.NewParser().ParseEnvironmentVariables() 67 | 68 | httpClient, _, server := helper.MockServer() 69 | defer server.Close() 70 | 71 | mstdn := New(*httpClient, opts) 72 | err := mstdn.Shutdown() 73 | if err != nil { 74 | t.Errorf("Unexpected shutdown: %v", err) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /publish/mastodon/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package mastodon // import "github.com/wabarc/wayback/publish/mastodon" 6 | 7 | import ( 8 | "net/http" 9 | 10 | "github.com/wabarc/wayback/config" 11 | "github.com/wabarc/wayback/publish" 12 | ) 13 | 14 | func init() { 15 | publish.Register(publish.FlagMastodon, setup) 16 | } 17 | 18 | func setup(opts *config.Options) *publish.Module { 19 | if opts.PublishToMastodon() { 20 | publisher := New(http.Client{}, opts) 21 | 22 | return &publish.Module{ 23 | Publisher: publisher, 24 | Opts: opts, 25 | } 26 | } 27 | 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /publish/matrix/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package matrix // import "github.com/wabarc/wayback/publish/matrix" 6 | -------------------------------------------------------------------------------- /publish/matrix/matrix_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package matrix // import "github.com/wabarc/wayback/publish/matrix" 6 | 7 | import ( 8 | "io" 9 | "net/http" 10 | "net/http/httptest" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/wabarc/helper" 15 | "github.com/wabarc/wayback/config" 16 | "github.com/wabarc/wayback/publish" 17 | "github.com/wabarc/wayback/template/render" 18 | ) 19 | 20 | func setMatrixEnv(t *testing.T) { 21 | t.Setenv("WAYBACK_MATRIX_USERID", "@foo:example.com") 22 | t.Setenv("WAYBACK_MATRIX_ROOMID", "!bar:example.com") 23 | t.Setenv("WAYBACK_MATRIX_PASSWORD", "zoo") 24 | } 25 | 26 | func matrixServer() *httptest.Server { 27 | _, mux, server := helper.MockServer() 28 | 29 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 30 | w.Header().Set("Content-Type", "application/json") 31 | switch { 32 | case r.URL.Path == "/_matrix/client/r0/login", r.URL.Path == "/_matrix/client/v3/login": 33 | w.Write([]byte(`{"access_token": "zoo"}`)) 34 | case strings.Contains(r.URL.Path, "!bar:example.com/send/m.room.message"): 35 | body, _ := io.ReadAll(r.Body) 36 | if !strings.Contains(string(body), config.SlotName(config.SLOT_IA)) { 37 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 38 | return 39 | } 40 | w.Write([]byte(`{"id": 1}`)) 41 | } 42 | }) 43 | 44 | return server 45 | } 46 | 47 | func TestToMatrixRoom(t *testing.T) { 48 | setMatrixEnv(t) 49 | 50 | server := matrixServer() 51 | defer server.Close() 52 | 53 | t.Setenv("WAYBACK_MATRIX_HOMESERVER", server.URL) 54 | opts, _ := config.NewParser().ParseEnvironmentVariables() 55 | 56 | mat := New(nil, opts) 57 | txt := render.ForPublish(&render.Mastodon{Cols: publish.Collects}).String() 58 | got := mat.toRoom(txt) 59 | if !got { 60 | t.Errorf("Unexpected publish room message got %t instead of %t", got, true) 61 | } 62 | } 63 | 64 | func TestShutdown(t *testing.T) { 65 | setMatrixEnv(t) 66 | 67 | server := matrixServer() 68 | defer server.Close() 69 | 70 | t.Setenv("WAYBACK_MATRIX_HOMESERVER", server.URL) 71 | opts, _ := config.NewParser().ParseEnvironmentVariables() 72 | 73 | mat := New(nil, opts) 74 | err := mat.Shutdown() 75 | if err != nil { 76 | t.Errorf("Unexpected shutdown: %v", err) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /publish/matrix/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package matrix // import "github.com/wabarc/wayback/publish/matrix" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagMatrix, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToMatrixRoom() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/meili/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package meili // import "github.com/wabarc/wayback/publish/meili" 6 | -------------------------------------------------------------------------------- /publish/meili/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package meili // import "github.com/wabarc/wayback/publish/meili" 6 | 7 | import ( 8 | "github.com/wabarc/logger" 9 | "github.com/wabarc/wayback/config" 10 | "github.com/wabarc/wayback/publish" 11 | ) 12 | 13 | func init() { 14 | publish.Register(publish.FlagMeili, setup) 15 | } 16 | 17 | func setup(opts *config.Options) *publish.Module { 18 | if opts.EnabledMeilisearch() { 19 | publisher := New(nil, opts) 20 | 21 | // Setup meilisearch 22 | err := publisher.setup() 23 | if err != nil { 24 | logger.Error("setup meilisearch failed: %v", err) 25 | return nil 26 | } 27 | 28 | return &publish.Module{ 29 | Publisher: publisher, 30 | Opts: opts, 31 | } 32 | } 33 | 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /publish/module.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package publish // import "github.com/wabarc/wayback/publish" 6 | 7 | import ( 8 | "fmt" 9 | "sync" 10 | 11 | "github.com/wabarc/wayback/config" 12 | ) 13 | 14 | var ( 15 | publishers = make(map[Flag]Publisher) 16 | modules = make(map[Flag]SetupFunc) 17 | mu sync.RWMutex 18 | ) 19 | 20 | // SetupFunc is a function type that takes a pointer 21 | // to a config.Options struct and returns a pointer 22 | // to a Module struct. 23 | type SetupFunc func(*config.Options) *Module 24 | 25 | // Module is a struct embeds the Publisher interface 26 | // and holds a pointer to config.Options. 27 | type Module struct { 28 | Publisher 29 | 30 | Opts *config.Options 31 | Flag Flag 32 | } 33 | 34 | // Register registers a publish client's setup function 35 | // and allows it to be called. 36 | func Register(flag Flag, action SetupFunc) { 37 | if _, exists := modules[flag]; exists { 38 | panic(fmt.Sprintf("module %s registered", flag)) 39 | } 40 | 41 | mu.Lock() 42 | modules[flag] = action 43 | mu.Unlock() 44 | } 45 | 46 | func parseModule(opts *config.Options) { 47 | for flag, setup := range modules { 48 | handler := setup(opts) 49 | if handler != nil { 50 | handler.Opts = opts 51 | handler.Flag = flag 52 | publishers[flag] = handler 53 | } 54 | } 55 | } 56 | 57 | func loadPublisher(flag Flag) (*Module, error) { 58 | mu.RLock() 59 | defer mu.RUnlock() 60 | 61 | p, ok := publishers[flag].(*Module) 62 | if !ok { 63 | return nil, fmt.Errorf("publisher %s not exists", flag) 64 | } 65 | return p, nil 66 | } 67 | -------------------------------------------------------------------------------- /publish/module_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package publish // import "github.com/wabarc/wayback/publish" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/wabarc/wayback/config" 11 | ) 12 | 13 | func TestRegister(t *testing.T) { 14 | setup := func(opts *config.Options) *Module { 15 | return &Module{ 16 | Opts: opts, 17 | Publisher: nil, 18 | Flag: FlagWeb, 19 | } 20 | } 21 | 22 | // Call Register with a valid flag and the setup function we just created 23 | Register(FlagWeb, setup) 24 | 25 | // Call Register again with the same flag, it should panic 26 | defer func() { 27 | // Clear 28 | delete(modules, FlagWeb) 29 | 30 | if r := recover(); r == nil { 31 | t.Errorf("Register should have panicked") 32 | } 33 | }() 34 | Register(FlagWeb, setup) 35 | } 36 | -------------------------------------------------------------------------------- /publish/nostr/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package nostr // import "github.com/wabarc/wayback/publish/nostr" 6 | -------------------------------------------------------------------------------- /publish/nostr/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package nostr // import "github.com/wabarc/wayback/publish/nostr" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagNostr, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToNostr() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/notion/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package notion // import "github.com/wabarc/wayback/publish/notion" 6 | -------------------------------------------------------------------------------- /publish/notion/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package notion // import "github.com/wabarc/wayback/publish/notion" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagNotion, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.NotionToken() != "" { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/omnivore/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package omnivore // import "github.com/wabarc/wayback/publish/omnivore" 6 | -------------------------------------------------------------------------------- /publish/omnivore/omnivore_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package omnivore // import "github.com/wabarc/wayback/publish/omnivore" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "net/http" 11 | "testing" 12 | "time" 13 | 14 | "github.com/wabarc/helper" 15 | "github.com/wabarc/wayback/config" 16 | "github.com/wabarc/wayback/publish" 17 | "github.com/wabarc/wayback/reduxer" 18 | ) 19 | 20 | const saveURLResp = `{"data":{"saveUrl":{"url":"https://omnivore.app/repo/links/cff02ab5-c36e-4efe-a976-2de32dc1685d","clientRequestId":"cff02ab5-c36e-4efe-a976-2de32dc1685d"}}}` 21 | 22 | func TestPublish(t *testing.T) { 23 | t.Setenv("WAYBACK_OMNIVORE_APIKEY", "foo") 24 | opts, _ := config.NewParser().ParseEnvironmentVariables() 25 | 26 | httpClient, mux, server := helper.MockServer() 27 | defer server.Close() 28 | 29 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 30 | w.Header().Set("Content-Type", "application/json") 31 | switch r.URL.Path { 32 | case "/api/graphql": 33 | fmt.Fprintln(w, saveURLResp) 34 | default: 35 | fmt.Fprintln(w, `{}`) 36 | } 37 | }) 38 | 39 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 40 | defer cancel() 41 | 42 | o := New(httpClient, opts) 43 | got := o.Publish(ctx, reduxer.BundleExample(), publish.Collects) 44 | if got != nil { 45 | t.Errorf("unexpected save url got %v", got) 46 | } 47 | } 48 | 49 | func TestShutdown(t *testing.T) { 50 | opts, _ := config.NewParser().ParseEnvironmentVariables() 51 | 52 | httpClient, _, server := helper.MockServer() 53 | defer server.Close() 54 | 55 | no := New(httpClient, opts) 56 | err := no.Shutdown() 57 | if err != nil { 58 | t.Errorf("Unexpected shutdown: %v", err) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /publish/omnivore/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package omnivore // import "github.com/wabarc/wayback/publish/omnivore" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagOmnivore, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.EnabledOmnivore() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/relaychat/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package relaychat // import "github.com/wabarc/wayback/publish/relaychat" 6 | -------------------------------------------------------------------------------- /publish/relaychat/relaychat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package relaychat // import "github.com/wabarc/wayback/publish/relaychat" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/wabarc/helper" 11 | "github.com/wabarc/wayback/config" 12 | ) 13 | 14 | const server = "irc.libera.chat:6697" 15 | 16 | func setIRCEnv(t *testing.T) { 17 | t.Setenv("WAYBACK_IRC_NICK", helper.RandString(6, "")) 18 | t.Setenv("WAYBACK_IRC_CHANNEL", "bar") 19 | t.Setenv("WAYBACK_IRC_SERVER", server) 20 | } 21 | func TestToIRCChannel(t *testing.T) { 22 | } 23 | 24 | func TestShutdown(t *testing.T) { 25 | setIRCEnv(t) 26 | opts, _ := config.NewParser().ParseEnvironmentVariables() 27 | 28 | irc := New(nil, opts) 29 | err := irc.Shutdown() 30 | if err != nil { 31 | t.Errorf("Unexpected shutdown: %v", err) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /publish/relaychat/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package relaychat // import "github.com/wabarc/wayback/publish/relaychat" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagIRC, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToIRCChannel() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/slack/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package slack // import "github.com/wabarc/wayback/publish/slack" 6 | -------------------------------------------------------------------------------- /publish/slack/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package slack // import "github.com/wabarc/wayback/publish/slack" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagSlack, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToSlackChannel() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/telegram/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package telegram // import "github.com/wabarc/wayback/publish/telegram" 6 | -------------------------------------------------------------------------------- /publish/telegram/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package telegram // import "github.com/wabarc/wayback/publish/telegram" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagTelegram, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToChannel() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/twitter/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package twitter // import "github.com/wabarc/wayback/publish/twitter" 6 | -------------------------------------------------------------------------------- /publish/twitter/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package twitter // import "github.com/wabarc/wayback/publish/twitter" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/publish" 10 | ) 11 | 12 | func init() { 13 | publish.Register(publish.FlagTwitter, setup) 14 | } 15 | 16 | func setup(opts *config.Options) *publish.Module { 17 | if opts.PublishToTwitter() { 18 | publisher := New(nil, opts) 19 | 20 | return &publish.Module{ 21 | Publisher: publisher, 22 | Opts: opts, 23 | } 24 | } 25 | 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /publish/twitter/twitter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package twitter // import "github.com/wabarc/wayback/publish/twitter" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "net/http" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/wabarc/helper" 15 | "github.com/wabarc/wayback/config" 16 | "github.com/wabarc/wayback/publish" 17 | "github.com/wabarc/wayback/template/render" 18 | ) 19 | 20 | func setTwitterEnv(t *testing.T) { 21 | t.Setenv("WAYBACK_TWITTER_CONSUMER_KEY", "foo") 22 | t.Setenv("WAYBACK_TWITTER_CONSUMER_SECRET", "foo") 23 | t.Setenv("WAYBACK_TWITTER_ACCESS_TOKEN", "foo") 24 | t.Setenv("WAYBACK_TWITTER_ACCESS_SECRET", "foo") 25 | } 26 | 27 | func TestToTwitter(t *testing.T) { 28 | setTwitterEnv(t) 29 | 30 | client, mux, server := helper.MockServer() 31 | defer server.Close() 32 | 33 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 34 | w.Header().Set("Content-Type", "application/json") 35 | switch r.URL.Path { 36 | case "/1.1/statuses/update.json": 37 | status := r.FormValue("status") 38 | if !strings.Contains(status, config.SlotName(config.SLOT_IA)) { 39 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 40 | return 41 | } 42 | fmt.Fprintln(w, `{"id": 1}`) 43 | default: 44 | fmt.Fprintln(w, `{}`) 45 | } 46 | }) 47 | 48 | opts, _ := config.NewParser().ParseEnvironmentVariables() 49 | twitt := New(client, opts) 50 | txt := render.ForPublish(&render.Twitter{Cols: publish.Collects}).String() 51 | got := twitt.ToTwitter(context.Background(), txt) 52 | if !got { 53 | t.Errorf("Unexpected create GitHub Issues got %t instead of %t", got, true) 54 | } 55 | } 56 | 57 | func TestShutdown(t *testing.T) { 58 | setTwitterEnv(t) 59 | 60 | opts, _ := config.NewParser().ParseEnvironmentVariables() 61 | 62 | httpClient, _, server := helper.MockServer() 63 | defer server.Close() 64 | 65 | tw := New(httpClient, opts) 66 | err := tw.Shutdown() 67 | if err != nil { 68 | t.Errorf("Unexpected shutdown: %v", err) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /publish/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package publish // import "github.com/wabarc/wayback/publish" 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/wabarc/wayback" 11 | "github.com/wabarc/wayback/errors" 12 | "github.com/wabarc/wayback/reduxer" 13 | ) 14 | 15 | // Artifact returns an artifact from the reduxer that corresponds to the first 16 | // collect in the provided slice of collects. If the artifact is not found in 17 | // the reduxer, an error is returned. 18 | func Artifact(ctx context.Context, rdx reduxer.Reduxer, cols []wayback.Collect) (art reduxer.Artifact, err error) { 19 | if len(cols) == 0 { 20 | return art, errors.New("no collect") 21 | } 22 | 23 | var uri = cols[0].Src 24 | if bundle, ok := rdx.Load(reduxer.Src(uri)); ok { 25 | return bundle.Artifact(), nil 26 | } 27 | return art, errors.New("reduxer data not found") 28 | } 29 | -------------------------------------------------------------------------------- /publish/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package publish // import "github.com/wabarc/wayback/publish" 6 | 7 | import ( 8 | "context" 9 | "reflect" 10 | "testing" 11 | 12 | "github.com/wabarc/wayback" 13 | "github.com/wabarc/wayback/errors" 14 | "github.com/wabarc/wayback/reduxer" 15 | ) 16 | 17 | func TestArtifact(t *testing.T) { 18 | // create a mock reduxer 19 | rdx := reduxer.BundleExample() 20 | src := "https://example.com/" 21 | example := reduxer.Src(src) 22 | bundle, ok := rdx.Load(example) 23 | if !ok { 24 | t.Fatalf("unexpected load artifat: not found") 25 | } 26 | 27 | // create some collects 28 | col1 := wayback.Collect{Src: src} 29 | col2 := wayback.Collect{Src: "https://example.org/"} 30 | 31 | tests := []struct { 32 | name string 33 | cols []wayback.Collect 34 | exp reduxer.Artifact 35 | err error 36 | }{ 37 | { 38 | name: "valid", 39 | cols: []wayback.Collect{col1}, 40 | exp: bundle.Artifact(), 41 | err: nil, 42 | }, 43 | { 44 | name: "no collect", 45 | cols: []wayback.Collect{}, 46 | exp: reduxer.Artifact{}, 47 | err: errors.New("no collect"), 48 | }, 49 | { 50 | name: "not found", 51 | cols: []wayback.Collect{col2}, 52 | exp: reduxer.Artifact{}, 53 | err: errors.New("reduxer data not found"), 54 | }, 55 | } 56 | 57 | for _, test := range tests { 58 | t.Run(test.name, func(t *testing.T) { 59 | art, err := Artifact(context.Background(), rdx, test.cols) 60 | 61 | if !reflect.DeepEqual(err, test.err) { 62 | t.Errorf("expected error %v, but got %v", test.err, err) 63 | } 64 | 65 | if art != test.exp { 66 | t.Errorf("expected artifact %v, but got %v", test.exp, art) 67 | } 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /reduxer/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package reduxer implements a set of functions to transform webpage to various formats. 7 | */ 8 | package reduxer // import "github.com/wabarc/wayback/reduxer" 9 | -------------------------------------------------------------------------------- /reduxer/media_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package reduxer // import "github.com/wabarc/wayback/reduxer" 6 | 7 | import ( 8 | "net/url" 9 | "os" 10 | "testing" 11 | ) 12 | 13 | const ( 14 | host = `https://www.youtube.com` 15 | domain = `youtube.com` 16 | ) 17 | 18 | var ( 19 | validURL, _ = url.Parse(host) 20 | invalidURL = &url.URL{Host: `invalid-tld`} 21 | ) 22 | 23 | func TestBaseHost(t *testing.T) { 24 | var tests = []struct { 25 | url *url.URL 26 | exp string 27 | }{ 28 | {validURL, domain}, 29 | {invalidURL, ``}, 30 | } 31 | 32 | for _, test := range tests { 33 | t.Run("", func(t *testing.T) { 34 | dom, _ := baseHost(test.url) 35 | if dom != test.exp { 36 | t.Errorf(`Unexpected extract base host, got %v instead of %v`, dom, test.exp) 37 | } 38 | }) 39 | } 40 | } 41 | 42 | func TestSupportedMediaSite(t *testing.T) { 43 | extraDomain := "https://extra-domain.com" 44 | missing, _ := url.Parse("https://missing.com") 45 | extraURL, _ := url.Parse(extraDomain) 46 | 47 | var tests = []struct { 48 | url *url.URL 49 | testname string 50 | filename string 51 | extra string 52 | supported bool 53 | }{ 54 | {validURL, `test with valid url`, filename, ``, true}, 55 | {invalidURL, `test with invalid url`, filename, ``, false}, 56 | {missing, `test not found`, filename, ``, false}, 57 | {extraURL, `test extra sites`, filename, extraDomain, true}, 58 | {invalidURL, `test extra invalid sites`, filename, extraDomain, false}, 59 | {invalidURL, `test sites configuration file not exists`, `/path/not/exists`, extraDomain, false}, 60 | } 61 | 62 | for _, test := range tests { 63 | t.Run(test.testname, func(t *testing.T) { 64 | os.Setenv("WAYBACK_MEDIA_SITES", test.extra) 65 | parseMediaSites(test.filename) 66 | supported := supportedMediaSite(test.url) 67 | if supported != test.supported { 68 | t.Errorf(`Unexpected check download media supported, got %v instead of %v`, supported, test.supported) 69 | } 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /reduxer/sites: -------------------------------------------------------------------------------- 1 | 365yg.com 2 | 56.com 3 | 7gogo.jp 4 | acfun.cn 5 | alive.in.th 6 | archive.org 7 | baidu.com 8 | bandcamp.com 9 | baomihua.com 10 | bilibili.com 11 | cbs.com 12 | cntv.cn 13 | coub.com 14 | dailymotion.com 15 | douban.com 16 | douyin.com 17 | douyutv.com 18 | ehow.com 19 | facebook.com 20 | fc2.com 21 | flickr.com 22 | freesound.org 23 | fun.tv 24 | heavy-music.ru 25 | huomao.com 26 | imgur.com 27 | infoq.com 28 | instagram.com 29 | interest.me 30 | iqiyi.com 31 | isuntv.com 32 | ixigua.com 33 | joy.cn 34 | khanacademy.org 35 | ku6.com 36 | kuaishou.com 37 | kugou.com 38 | kuwo.cn 39 | le.com 40 | lizhi.fm 41 | lrts.me 42 | magisto.com 43 | metacafe.com 44 | mgtv.com 45 | miaopai.com 46 | miomio.tv 47 | missevan.com 48 | mixcloud.com 49 | mtv81.com 50 | naver.com 51 | nicovideo.jp 52 | pinterest.com 53 | pixnet.net 54 | pptv.com 55 | v.qq.com 56 | reddit.com 57 | showroom-live.com 58 | sina.com.cn 59 | slideshare.net 60 | sohu.com 61 | soundcloud.com 62 | ted.com 63 | tiktok.com 64 | tudou.com 65 | tumblr.com 66 | twitch.tv 67 | twitter.com 68 | veoh.com 69 | v.ifeng.com 70 | vimeo.com 71 | vine.co 72 | v.iqilu.com 73 | vk.com 74 | weibo.cn 75 | weibo.com 76 | xiami.com 77 | xinpianchang.com 78 | youku.com 79 | youtube.com 80 | zhanqi.tv 81 | zhibo.tv 82 | zhihu.com 83 | -------------------------------------------------------------------------------- /reduxer/video.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !with_lux 6 | 7 | package reduxer // import "github.com/wabarc/wayback/reduxer" 8 | 9 | import ( 10 | "context" 11 | 12 | "github.com/wabarc/wayback/config" 13 | ) 14 | 15 | func (m media) viaLux(ctx context.Context, cfg *config.Options) string { 16 | return "" 17 | } 18 | -------------------------------------------------------------------------------- /render.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | # More details see https://render.com/docs/blueprint-spec 6 | 7 | services: 8 | - type: web 9 | name: wayback-starter 10 | plan: free 11 | env: docker 12 | autoDeploy: false 13 | healthCheckPath: /healthcheck 14 | dockerfilePath: ./build/docker/Dockerfile.render 15 | envVars: 16 | - key: PORT 17 | value: 80 18 | - key: WAYBACK_IPFS_APIKEY 19 | sync: false 20 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs-material==9.5.19 2 | mkdocs==1.6.0 3 | 4 | # Python Markdown Extensions 5 | pygments==2.17.2 6 | pymdown-extensions==10.8.1 7 | 8 | # MkDocs Plugins 9 | mkdocs-exclude-search==0.6.6 10 | mkdocs-git-revision-date-localized-plugin==1.2.4 11 | mkdocs-static-i18n==1.2.2 12 | -------------------------------------------------------------------------------- /service/discord/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package discord implements the discord bot daemon service. 7 | */ 8 | package discord // import "github.com/wabarc/wayback/service/discord" 9 | -------------------------------------------------------------------------------- /service/discord/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package discord // import "github.com/wabarc/wayback/service/discord" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceDiscord, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.DiscordEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("discord service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package service implements the common utils function for daemon services. 7 | 8 | To use the service package, import the package and its dependencies, as shown in the following example: 9 | 10 | package main 11 | 12 | import ( 13 | _ "github.com/wabarc/wayback/ingress" 14 | "github.com/wabarc/wayback/service" 15 | ) 16 | 17 | func main() { 18 | // Initialize services with configuration options and a context. 19 | opts := service.Options{} 20 | ctx := context.Background() 21 | err := service.Serve(ctx, opts) 22 | // ... 23 | } 24 | */ 25 | package service // import "github.com/wabarc/wayback/service" 26 | -------------------------------------------------------------------------------- /service/httpd/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package httpd implements the tor network service. 7 | */ 8 | package httpd // import "github.com/wabarc/wayback/service/httpd" 9 | -------------------------------------------------------------------------------- /service/httpd/libtor.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build with_tor 6 | 7 | package httpd // import "github.com/wabarc/wayback/service/httpd" 8 | 9 | import ( 10 | "github.com/cretz/bine/process" 11 | "github.com/ipsn/go-libtor" 12 | ) 13 | 14 | var creator process.Creator = libtor.Creator 15 | -------------------------------------------------------------------------------- /service/httpd/libtor_stub.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !with_tor 6 | 7 | package httpd // import "github.com/wabarc/wayback/service/httpd" 8 | 9 | import ( 10 | "github.com/cretz/bine/process" 11 | ) 12 | 13 | var creator process.Creator 14 | -------------------------------------------------------------------------------- /service/httpd/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package httpd // import "github.com/wabarc/wayback/service/httpd" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceHTTPd, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.HTTPdEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("httpd service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/mastodon/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package mastodon implements the mastodon daemon service. 7 | */ 8 | package mastodon // import "github.com/wabarc/wayback/service/mastodon" 9 | -------------------------------------------------------------------------------- /service/mastodon/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package mastodon // import "github.com/wabarc/wayback/service/mastodon" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceMastodon, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.MastodonEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("mastodon service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/matrix/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package matrix implements the matrix daemon service. 7 | Spec: https://matrix.org/docs/spec/ 8 | */ 9 | package matrix // import "github.com/wabarc/wayback/service/matrix" 10 | -------------------------------------------------------------------------------- /service/matrix/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package matrix // import "github.com/wabarc/wayback/service/matrix" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceMatrix, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.MatrixEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("matrix service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/module.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package service // import "github.com/wabarc/wayback/service" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "sync" 11 | 12 | "github.com/wabarc/wayback/config" 13 | ) 14 | 15 | var ( 16 | services = make(map[config.Flag]Servicer) 17 | modules = make(map[config.Flag]SetupFunc) 18 | mu sync.RWMutex 19 | ) 20 | 21 | // SetupFunc is a function type that takes a pointer 22 | // to a Options struct and returns a pointer 23 | // to a Module struct. 24 | type SetupFunc func(context.Context, Options) (*Module, error) 25 | 26 | // Module is a struct embeds the Servicer interface 27 | // and holds a pointer to Options. 28 | type Module struct { 29 | Servicer 30 | 31 | Opts Options 32 | Flag config.Flag 33 | } 34 | 35 | // Register registers a service instance's setup function 36 | // and allows it to be called. 37 | func Register(srv config.Flag, action SetupFunc) { 38 | if _, exists := modules[srv]; exists { 39 | panic(fmt.Sprintf("module %s registered", srv)) 40 | } 41 | 42 | mu.Lock() 43 | modules[srv] = action 44 | mu.Unlock() 45 | } 46 | 47 | func parseModule(ctx context.Context, opts Options) { 48 | for flag, setup := range modules { 49 | mod, err := setup(ctx, opts) 50 | if err == nil && mod != nil { 51 | mod.Opts = opts 52 | mod.Flag = flag 53 | services[flag] = mod 54 | } 55 | } 56 | } 57 | 58 | func loadServicer(flag config.Flag) (*Module, error) { 59 | mu.RLock() 60 | defer mu.RUnlock() 61 | 62 | p, ok := services[flag].(*Module) 63 | if !ok { 64 | return nil, fmt.Errorf("servicer %s not exists", flag) 65 | } 66 | return p, nil 67 | } 68 | -------------------------------------------------------------------------------- /service/module_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package service // import "github.com/wabarc/wayback/service" 6 | 7 | import ( 8 | "context" 9 | "testing" 10 | 11 | "github.com/wabarc/wayback/config" 12 | ) 13 | 14 | func TestRegister(t *testing.T) { 15 | setup := func(ctx context.Context, opts Options) (*Module, error) { 16 | return &Module{ 17 | Opts: opts, 18 | Servicer: nil, 19 | Flag: config.ServiceHTTPd, 20 | }, nil 21 | } 22 | // Call Register with a valid flag and the setup function we just created 23 | Register(config.ServiceHTTPd, setup) 24 | 25 | // Call Register again with the same flag, it should panic 26 | 27 | defer func() { 28 | // Clear 29 | delete(modules, config.ServiceHTTPd) 30 | 31 | if r := recover(); r == nil { 32 | t.Errorf("Register should have panicked") 33 | } 34 | }() 35 | 36 | Register(config.ServiceHTTPd, setup) 37 | } 38 | -------------------------------------------------------------------------------- /service/options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package service // import "github.com/wabarc/wayback/service" 6 | 7 | import ( 8 | "github.com/wabarc/wayback/config" 9 | "github.com/wabarc/wayback/pooling" 10 | "github.com/wabarc/wayback/publish" 11 | "github.com/wabarc/wayback/storage" 12 | ) 13 | 14 | // Options represents the configuration for services. 15 | type Options struct { 16 | // Config holds the configuration options. 17 | Config *config.Options 18 | 19 | // Pool holds the connection pool to be used. 20 | Pool *pooling.Pool 21 | 22 | // Publish holds the publish service to be used. 23 | Publish *publish.Publish 24 | 25 | // Storage holds the storage service to be used. 26 | Storage *storage.Storage 27 | } 28 | 29 | // Option is a function that modifies the provided Options instance. 30 | type Option func(*Options) 31 | 32 | // ParseOptions returns the Options instance with modifications applied using the provided Option functions. 33 | func ParseOptions(opts ...Option) (o Options) { 34 | for _, fn := range opts { 35 | fn(&o) 36 | } 37 | return o 38 | } 39 | 40 | // Config returns an Option function that sets the Config field of Options. 41 | func Config(c *config.Options) Option { 42 | return func(opts *Options) { 43 | opts.Config = c 44 | } 45 | } 46 | 47 | // Pool returns an Option function that sets the Pool field of Options. 48 | func Pool(p *pooling.Pool) Option { 49 | return func(opts *Options) { 50 | opts.Pool = p 51 | } 52 | } 53 | 54 | // Publish returns an Option function that sets the Publish field of Options. 55 | func Publish(p *publish.Publish) Option { 56 | return func(opts *Options) { 57 | opts.Publish = p 58 | } 59 | } 60 | 61 | // Storage returns an Option function that sets the Storage field of Options. 62 | func Storage(s *storage.Storage) Option { 63 | return func(opts *Options) { 64 | opts.Storage = s 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /service/options_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package service // import "github.com/wabarc/wayback/service" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/wabarc/wayback/config" 11 | "github.com/wabarc/wayback/pooling" 12 | "github.com/wabarc/wayback/publish" 13 | "github.com/wabarc/wayback/storage" 14 | ) 15 | 16 | func TestParseOptions(t *testing.T) { 17 | configOpts := &config.Options{} 18 | pool := &pooling.Pool{} 19 | publish := &publish.Publish{} 20 | storage := &storage.Storage{} 21 | 22 | opts := ParseOptions( 23 | Config(configOpts), 24 | Pool(pool), 25 | Publish(publish), 26 | Storage(storage), 27 | ) 28 | 29 | if opts.Config != configOpts { 30 | t.Errorf("Unexpected config options: expected=%v, got=%v", configOpts, opts.Config) 31 | } 32 | 33 | if opts.Pool != pool { 34 | t.Errorf("Unexpected pooling options: expected=%v, got=%v", pool, opts.Pool) 35 | } 36 | 37 | if opts.Publish != publish { 38 | t.Errorf("Unexpected publish options: expected=%v, got=%v", publish, opts.Publish) 39 | } 40 | 41 | if opts.Storage != storage { 42 | t.Errorf("Unexpected storage options: expected=%v, got=%v", storage, opts.Storage) 43 | } 44 | } 45 | 46 | func TestConfig(t *testing.T) { 47 | configOpts := &config.Options{} 48 | option := Config(configOpts) 49 | 50 | opts := &Options{} 51 | option(opts) 52 | 53 | if opts.Config != configOpts { 54 | t.Errorf("Unexpected config options: expected=%v, got=%v", configOpts, opts.Config) 55 | } 56 | } 57 | 58 | func TestPool(t *testing.T) { 59 | pool := &pooling.Pool{} 60 | option := Pool(pool) 61 | 62 | opts := &Options{} 63 | option(opts) 64 | 65 | if opts.Pool != pool { 66 | t.Errorf("Unexpected pooling options: expected=%v, got=%v", pool, opts.Pool) 67 | } 68 | } 69 | 70 | func TestPublish(t *testing.T) { 71 | publish := &publish.Publish{} 72 | option := Publish(publish) 73 | 74 | opts := &Options{} 75 | option(opts) 76 | 77 | if opts.Publish != publish { 78 | t.Errorf("Unexpected publish options: expected=%v, got=%v", publish, opts.Publish) 79 | } 80 | } 81 | 82 | func TestStorage(t *testing.T) { 83 | storage := &storage.Storage{} 84 | option := Storage(storage) 85 | 86 | opts := &Options{} 87 | option(opts) 88 | 89 | if opts.Storage != storage { 90 | t.Errorf("Unexpected storage options: expected=%v, got=%v", storage, opts.Storage) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /service/relaychat/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package relaychat implements the internet relay chat daemon service. 7 | RFC: https://tools.ietf.org/html/rfc1459 8 | */ 9 | package relaychat // import "github.com/wabarc/wayback/service/relaychat" 10 | -------------------------------------------------------------------------------- /service/relaychat/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package relaychat // import "github.com/wabarc/wayback/service/relaychat" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceIRC, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.IRCEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("irc service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/slack/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package slack implements the slack bot daemon service. 7 | */ 8 | package slack // import "github.com/wabarc/wayback/service/slack" 9 | -------------------------------------------------------------------------------- /service/slack/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package slack // import "github.com/wabarc/wayback/service/slack" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceSlack, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.SlackEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("slack service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/telegram/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package telegram implements the telegram bot daemon service. 7 | */ 8 | package telegram // import "github.com/wabarc/wayback/service/telegram" 9 | -------------------------------------------------------------------------------- /service/telegram/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package telegram // import "github.com/wabarc/wayback/service/telegram" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceTelegram, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.TelegramEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("telegram service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/twitter/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package twitter implements the twitter daemon service. 7 | */ 8 | package twitter // import "github.com/wabarc/wayback/service/twitter" 9 | -------------------------------------------------------------------------------- /service/twitter/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package twitter // import "github.com/wabarc/wayback/service/twitter" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceTwitter, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.TwitterEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("twitter service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /service/utils_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package service // import "github.com/wabarc/wayback/service" 6 | 7 | import ( 8 | "net/url" 9 | "reflect" 10 | "strconv" 11 | "testing" 12 | 13 | "github.com/wabarc/helper" 14 | "github.com/wabarc/wayback/config" 15 | ) 16 | 17 | func TestMatchURL(t *testing.T) { 18 | defer helper.CheckTest(t) 19 | 20 | parser := config.NewParser() 21 | opts, err := parser.ParseEnvironmentVariables() 22 | if err != nil { 23 | t.Fatalf("Parse environment variables or flags failed, error: %v", err) 24 | } 25 | 26 | var ( 27 | u = "http://example.org" 28 | x = "http://example.com" 29 | y = "https://example.com/" 30 | z = "https://example.com/path" 31 | ) 32 | 33 | var tests = []struct { 34 | text string 35 | leng int 36 | }{ 37 | { 38 | text: "", 39 | leng: 0, 40 | }, 41 | { 42 | text: "foo " + x, 43 | leng: 1, 44 | }, 45 | { 46 | text: x + " foo " + y, 47 | leng: 1, 48 | }, 49 | { 50 | text: y + " foo " + z, 51 | leng: 2, 52 | }, 53 | { 54 | text: u + " foo " + x, 55 | leng: 2, 56 | }, 57 | } 58 | 59 | for i, test := range tests { 60 | t.Run(strconv.Itoa(i), func(t *testing.T) { 61 | got := len(MatchURL(opts, test.text)) 62 | if got != test.leng { 63 | t.Fatalf(`Unexpected extract URLs number from text got %d instead of %d`, got, test.leng) 64 | } 65 | }) 66 | } 67 | } 68 | 69 | func TestExcludeURL(t *testing.T) { 70 | defer helper.CheckTest(t) 71 | 72 | var ( 73 | u, _ = url.Parse("http://example.org") 74 | m, _ = url.Parse("http://t.me/s/foo") 75 | host = "t.me" 76 | ) 77 | 78 | var tests = []struct { 79 | urls []*url.URL 80 | want []*url.URL 81 | }{ 82 | { 83 | urls: []*url.URL{u}, 84 | want: []*url.URL{u}, 85 | }, 86 | { 87 | urls: []*url.URL{u, m}, 88 | want: []*url.URL{u}, 89 | }, 90 | { 91 | urls: []*url.URL{m}, 92 | want: []*url.URL{m}, 93 | }, 94 | } 95 | 96 | for i, test := range tests { 97 | t.Run(strconv.Itoa(i), func(t *testing.T) { 98 | got := ExcludeURL(test.urls, host) 99 | if !reflect.DeepEqual(got, test.want) { 100 | t.Fatalf(`Unexpected exclude URLs number, got %v instead of %v`, got, test.want) 101 | } 102 | }) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /service/xmpp/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package xmpp implements the xmpp daemon service. 7 | */ 8 | package xmpp // import "github.com/wabarc/wayback/service/xmpp" 9 | -------------------------------------------------------------------------------- /service/xmpp/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package xmpp // import "github.com/wabarc/wayback/service/xmpp" 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/service" 13 | ) 14 | 15 | func init() { 16 | service.Register(config.ServiceXMPP, setup) 17 | } 18 | 19 | func setup(ctx context.Context, opts service.Options) (*service.Module, error) { 20 | if opts.Config.XMPPEnabled() { 21 | mod, err := New(ctx, opts) 22 | 23 | return &service.Module{ 24 | Servicer: mod, 25 | Opts: opts, 26 | }, err 27 | } 28 | 29 | return nil, fmt.Errorf("xmpp service disabled") 30 | } 31 | -------------------------------------------------------------------------------- /snapcraft.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Wayback Archiver. All rights reserved. 2 | # Use of this source code is governed by the GNU GPL v3 3 | # license that can be found in the LICENSE file. 4 | # 5 | name: wayback 6 | 7 | version: 'git' 8 | 9 | summary: A toolkit to upload files to IPFS pinning services. 10 | 11 | description: | 12 | A toolkit for snapshot webpage to the Wayback Machine. 13 | Website https://github.com/wabarc/wayback 14 | 15 | base: core20 16 | grade: stable 17 | confinement: strict 18 | compression: lzo 19 | 20 | architectures: 21 | - amd64 22 | - arm64 23 | - armhf 24 | - i386 25 | 26 | parts: 27 | wayback: 28 | plugin: go 29 | source: https://github.com/wabarc/wayback.git 30 | override-build: | 31 | make build 32 | install $SNAPCRAFT_PART_BUILD/build/binary/wayback -D $SNAPCRAFT_PART_INSTALL/bin/wayback 33 | mkdir -p $SNAPCRAFT_PART_INSTALL/bin 34 | if [ ! -e $SNAPCRAFT_PART_INSTALL/bin/wayback ]; then 35 | ln -s $SNAPCRAFT_PART_INSTALL/bin/wayback $SNAPCRAFT_PART_INSTALL/bin/wayback 36 | fi 37 | build-packages: 38 | - build-essential 39 | 40 | apps: 41 | wayback: 42 | command: bin/wayback 43 | plugs: 44 | - home 45 | - network 46 | - network-bind 47 | -------------------------------------------------------------------------------- /storage/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package storage implements a set of functions to interact with the database. 7 | */ 8 | package storage // import "github.com/wabarc/wayback/storage" 9 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage // import "github.com/wabarc/wayback/storage" 6 | 7 | import ( 8 | "encoding/binary" 9 | 10 | "github.com/wabarc/logger" 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/errors" 13 | 14 | bolt "go.etcd.io/bbolt" 15 | ) 16 | 17 | var ErrDatabaseNotFound = errors.New("database not found") 18 | 19 | // Storage handles all operations related to the database. 20 | type Storage struct { 21 | db *bolt.DB 22 | } 23 | 24 | // Open a bolt database on current directory in given path. 25 | // It is the caller's responsibility to close it. 26 | func Open(opts *config.Options, path string) (*Storage, error) { 27 | if path == "" { 28 | path = opts.BoltPathname() 29 | } 30 | db, err := bolt.Open(path, 0600, nil) 31 | if err != nil { 32 | logger.Fatal("open bolt database failed: %v", err) 33 | return nil, err 34 | } 35 | return &Storage{db: db}, nil 36 | } 37 | 38 | // Close the bolt database 39 | func (s *Storage) Close() error { 40 | if s.db != nil { 41 | return s.db.Close() 42 | } 43 | return ErrDatabaseNotFound 44 | } 45 | 46 | func itob(v uint64) []byte { 47 | b := make([]byte, 8) 48 | binary.BigEndian.PutUint64(b, v) 49 | return b 50 | } 51 | -------------------------------------------------------------------------------- /storage/storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage // import "github.com/wabarc/wayback/storage" 6 | 7 | import ( 8 | "os" 9 | "path" 10 | "testing" 11 | 12 | "github.com/wabarc/wayback/config" 13 | ) 14 | 15 | func TestOpen(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | path string 19 | }{ 20 | {"empty path", ""}, 21 | {"exist path", "bolt.db"}, 22 | } 23 | 24 | for _, test := range tests { 25 | t.Run(test.name, func(t *testing.T) { 26 | file := path.Join(t.TempDir(), test.path) 27 | if test.path == "" { 28 | file = test.path 29 | } 30 | 31 | opts, err := config.NewParser().ParseEnvironmentVariables() 32 | if err != nil { 33 | t.Fatalf("Parse environment variables or flags failed, error: %v", err) 34 | } 35 | defer os.Remove(opts.BoltPathname()) 36 | 37 | s, err := Open(opts, file) 38 | if err != nil { 39 | t.Fatalf("failed to open database: %v", err) 40 | } 41 | defer s.db.Close() 42 | 43 | if s == nil { 44 | t.Fatalf("Storage instance is nil") 45 | } 46 | if s.db == nil { 47 | t.Fatalf("bolt.DB instance is nil") 48 | } 49 | }) 50 | } 51 | } 52 | 53 | func TestClose(t *testing.T) { 54 | file := path.Join(t.TempDir(), "bolt.db") 55 | opts := &config.Options{} 56 | s, err := Open(opts, file) 57 | if err != nil { 58 | t.Fatalf("failed to open database: %v", err) 59 | } 60 | 61 | err = s.Close() 62 | if err != nil { 63 | t.Fatalf("failed to close database: %v", err) 64 | } 65 | 66 | if s.db.String() != `DB<"">` { 67 | t.Fatalf("failed to close database: %s", s.db) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /storage/telegram.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage // import "github.com/wabarc/wayback/storage" 6 | 7 | import ( 8 | "bytes" 9 | 10 | "github.com/wabarc/helper" 11 | "github.com/wabarc/logger" 12 | "github.com/wabarc/wayback/entity" 13 | bolt "go.etcd.io/bbolt" 14 | ) 15 | 16 | func (s *Storage) createPlaybackBucket() error { 17 | tx, err := s.db.Begin(true) 18 | if err != nil { 19 | return err 20 | } 21 | defer tx.Rollback() //nolint:errcheck 22 | 23 | _, err = tx.CreateBucketIfNotExists(helper.String2Byte(entity.EntityPlayback)) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | if err := tx.Commit(); err != nil { 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | 35 | // Playback returns playback data of the given id. 36 | func (s *Storage) Playback(id uint64) (*entity.Playback, error) { 37 | var pb entity.Playback 38 | 39 | err := s.db.View(func(tx *bolt.Tx) error { 40 | b := tx.Bucket(helper.String2Byte(entity.EntityPlayback)) 41 | v := b.Get(itob(id)) 42 | pb.Source = helper.Byte2String(v) 43 | pb.ID = id 44 | return nil 45 | }) 46 | 47 | return &pb, err 48 | } 49 | 50 | // CreatePlayback creates a playback callback data. 51 | func (s *Storage) CreatePlayback(pb *entity.Playback) error { 52 | if err := s.createPlaybackBucket(); err != nil { 53 | logger.Error("create playback buckte failed: %v", err) 54 | return err 55 | } 56 | 57 | return s.db.Update(func(tx *bolt.Tx) (err error) { 58 | b := tx.Bucket(helper.String2Byte(entity.EntityPlayback)) 59 | id, err := b.NextSequence() 60 | if err != nil { 61 | logger.Error("generate id for playback failed: %v", err) 62 | return err 63 | } 64 | logger.Debug("putting data to bucket, id: %d, value: %s", id, pb.Source) 65 | 66 | pb.ID = id 67 | buf := bytes.NewBufferString(pb.Source).Bytes() 68 | 69 | return b.Put(itob(pb.ID), buf) 70 | }) 71 | } 72 | 73 | // RemovePlayback removes a playback callback entry by id. 74 | func (s *Storage) RemovePlayback(id uint64) error { 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /storage/telegram_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package storage // import "github.com/wabarc/wayback/storage" 6 | 7 | import ( 8 | "path" 9 | "testing" 10 | 11 | "github.com/wabarc/wayback/config" 12 | "github.com/wabarc/wayback/entity" 13 | ) 14 | 15 | func TestCreatePlayback(t *testing.T) { 16 | parser := config.NewParser() 17 | opts, err := parser.ParseEnvironmentVariables() 18 | if err != nil { 19 | t.Fatalf("Parse environment variables or flags failed, error: %v", err) 20 | } 21 | 22 | s, err := Open(opts, path.Join(t.TempDir(), "wayback.db")) 23 | if err != nil { 24 | t.Fatalf("Unexpected open a bolt db: %v", err) 25 | } 26 | defer s.Close() 27 | 28 | pb := &entity.Playback{Source: ":wayback https://example.com"} 29 | err = s.CreatePlayback(pb) 30 | if err != nil { 31 | t.Fatalf("Unexpected create playback, error: %v", err) 32 | } 33 | } 34 | 35 | func TestPlayback(t *testing.T) { 36 | parser := config.NewParser() 37 | opts, err := parser.ParseEnvironmentVariables() 38 | if err != nil { 39 | t.Fatalf("Parse environment variables or flags failed, error: %v", err) 40 | } 41 | 42 | s, err := Open(opts, path.Join(t.TempDir(), "wayback.db")) 43 | if err != nil { 44 | t.Fatalf("Unexpected open a bolt db: %v", err) 45 | } 46 | defer s.Close() 47 | 48 | dt := ":wayback https://example.com" 49 | pb := &entity.Playback{Source: dt} 50 | err = s.CreatePlayback(pb) 51 | if err != nil { 52 | t.Fatalf("Unexpected create playback, error: %v", err) 53 | } 54 | 55 | pb, err = s.Playback(pb.ID) 56 | if err != nil { 57 | t.Fatalf("Unexpected query playback, error: %v", err) 58 | } 59 | if pb.ID == 0 { 60 | t.Errorf("Unexpected query playback, got %d instead of grather than 0", pb.ID) 61 | } 62 | if pb.Source != dt { 63 | t.Errorf("Unexpected query playback, got %s instead of %s", pb.Source, dt) 64 | } 65 | } 66 | 67 | func TestRemovePlayback(t *testing.T) { 68 | parser := config.NewParser() 69 | opts, err := parser.ParseEnvironmentVariables() 70 | if err != nil { 71 | t.Fatalf("Parse environment variables or flags failed, error: %v", err) 72 | } 73 | 74 | s, err := Open(opts, path.Join(t.TempDir(), "wayback.db")) 75 | if err != nil { 76 | t.Fatalf("Unexpected open a bolt db: %v", err) 77 | } 78 | defer s.Close() 79 | 80 | if s.RemovePlayback(0) != nil { 81 | t.Error("Unexpected remove playback data") 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /systemd/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package systemd provides a Go implementation of the sd_notify protocol. 7 | It can be used to inform systemd of service start-up completion. 8 | */ 9 | package systemd // import "github.com/wabarc/wayback/systemd" 10 | -------------------------------------------------------------------------------- /systemd/systemd.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !windows 6 | // +build !windows 7 | 8 | package systemd // import "github.com/wabarc/wayback/systemd" 9 | 10 | import ( 11 | "net" 12 | "os" 13 | ) 14 | 15 | // SdNotifyReady tells the service manager that service startup is 16 | // finished, or the service finished loading its configuration. 17 | // https://www.freedesktop.org/software/systemd/man/sd_notify.html#READY=1 18 | const SdNotifyReady = "READY=1" 19 | 20 | // HasNotifySocket checks if the process is supervised by Systemd and has the notify socket. 21 | func HasNotifySocket() bool { 22 | return os.Getenv("NOTIFY_SOCKET") != "" 23 | } 24 | 25 | // SdNotify sends a message to systemd using the sd_notify protocol. 26 | // See https://www.freedesktop.org/software/systemd/man/sd_notify.html. 27 | func SdNotify(state string) error { 28 | addr := &net.UnixAddr{ 29 | Net: "unixgram", 30 | Name: os.Getenv("NOTIFY_SOCKET"), 31 | } 32 | 33 | // We're not running under systemd (NOTIFY_SOCKET is not set). 34 | if addr.Name == "" { 35 | return nil 36 | } 37 | 38 | conn, err := net.DialUnix(addr.Net, nil, addr) 39 | if err != nil { 40 | return err 41 | } 42 | defer conn.Close() 43 | 44 | if _, err = conn.Write([]byte(state)); err != nil { 45 | return err 46 | } 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /systemd/systemd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !windows 6 | // +build !windows 7 | 8 | package systemd // import "github.com/wabarc/wayback/systemd" 9 | 10 | import ( 11 | "fmt" 12 | "net" 13 | "os" 14 | "testing" 15 | ) 16 | 17 | func TestSdNotify(t *testing.T) { 18 | testDir := t.TempDir() 19 | 20 | notifySocket := testDir + "/notify-socket.sock" 21 | laddr := net.UnixAddr{ 22 | Name: notifySocket, 23 | Net: "unixgram", 24 | } 25 | if _, err := net.ListenUnixgram("unixgram", &laddr); err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | tests := []struct { 30 | envSocket string 31 | 32 | werr bool 33 | }{ 34 | // (nil) - notification supported, data has been sent 35 | {notifySocket, false}, 36 | // (err) - notification supported, but failure happened 37 | {testDir + "/missing.sock", true}, 38 | // (nil) - notification not supported 39 | {"", false}, 40 | } 41 | 42 | for i, tt := range tests { 43 | os.Unsetenv("NOTIFY_SOCKET") 44 | 45 | if tt.envSocket != "" { 46 | os.Setenv("NOTIFY_SOCKET", tt.envSocket) 47 | } 48 | err := SdNotify(fmt.Sprintf("TestSdNotify test message #%d", i)) 49 | 50 | if tt.werr && err == nil { 51 | t.Errorf("#%d: want non-nil err, got nil", i) 52 | } else if !tt.werr && err != nil { 53 | t.Errorf("#%d: want nil err, got %v", i, err) 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /systemd/systemd_windows.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package systemd // import "github.com/wabarc/wayback/systemd" 6 | 7 | const SdNotifyReady = "READY=1" 8 | 9 | func HasNotifySocket() bool { 10 | return false 11 | } 12 | 13 | func SdNotify(state string) error { 14 | return nil 15 | } 16 | -------------------------------------------------------------------------------- /template/assets/image/favicon-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/favicon-16.png -------------------------------------------------------------------------------- /template/assets/image/favicon-32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/favicon-32.png -------------------------------------------------------------------------------- /template/assets/image/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/favicon.ico -------------------------------------------------------------------------------- /template/assets/image/icon-120.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-120.png -------------------------------------------------------------------------------- /template/assets/image/icon-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-128.png -------------------------------------------------------------------------------- /template/assets/image/icon-152.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-152.png -------------------------------------------------------------------------------- /template/assets/image/icon-167.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-167.png -------------------------------------------------------------------------------- /template/assets/image/icon-180.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-180.png -------------------------------------------------------------------------------- /template/assets/image/icon-192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-192.png -------------------------------------------------------------------------------- /template/assets/image/icon-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wabarc/wayback/1a7f080703db6e77d6a46419574caae014c89ef9/template/assets/image/icon-512.png -------------------------------------------------------------------------------- /template/assets/image/search.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /template/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package template handles template parsing and execution. 7 | */ 8 | package template // import "github.com/wabarc/wayback/template" 9 | -------------------------------------------------------------------------------- /template/render/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package render handles template parsing and execution for services. 7 | */ 8 | package render // import "github.com/wabarc/wayback/template/render" 9 | -------------------------------------------------------------------------------- /template/render/mastodon.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "bytes" 9 | "text/template" 10 | 11 | "github.com/wabarc/logger" 12 | "github.com/wabarc/wayback" 13 | "github.com/wabarc/wayback/reduxer" 14 | ) 15 | 16 | var _ Renderer = (*Mastodon)(nil) 17 | 18 | // Mastodon represents a Mastodon template data for render. 19 | type Mastodon struct { 20 | Cols []wayback.Collect 21 | Data reduxer.Reduxer 22 | } 23 | 24 | // ForReply implements the standard Renderer interface: 25 | // it returns a Render from the ForPublish. 26 | func (m *Mastodon) ForReply() *Render { 27 | return m.ForPublish() 28 | } 29 | 30 | // ForPublish implements the standard Renderer interface: 31 | // it reads `[]wayback.Collect` and `reduxer.Reduxer` from 32 | // the Mastodon and returns a *Render. 33 | func (m *Mastodon) ForPublish() *Render { 34 | var tmplBytes bytes.Buffer 35 | 36 | if title := Title(m.Cols, m.Data); title != "" { 37 | tmplBytes.WriteString(`‹ `) 38 | tmplBytes.WriteString(title) 39 | tmplBytes.WriteString(" ›\n\n") 40 | } 41 | 42 | const tmpl = `{{range $ := .}} 43 | • {{ $.Arc | name }} 44 | > {{ $.Dst }} 45 | {{end}}` 46 | 47 | tpl, err := template.New("mastodon").Funcs(funcMap()).Parse(tmpl) 48 | if err != nil { 49 | logger.Error("[masatodon] parse Mastodon template failed, %v", err) 50 | return new(Render) 51 | } 52 | 53 | tmplBytes.WriteString(original(m.Cols)) 54 | err = tpl.Execute(&tmplBytes, m.Cols) 55 | if err != nil { 56 | logger.Error("[masatodon] execute Mastodon template failed, %v", err) 57 | return new(Render) 58 | } 59 | tmplBytes.WriteString("\n#wayback #存档") 60 | tmplBytes = *bytes.NewBuffer(bytes.TrimSpace(tmplBytes.Bytes())) 61 | 62 | return &Render{buf: tmplBytes} 63 | } 64 | -------------------------------------------------------------------------------- /template/render/mastodon_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package render handles template parsing and execution for services. 7 | */ 8 | 9 | package render // import "github.com/wabarc/wayback/template/render" 10 | 11 | import ( 12 | "testing" 13 | ) 14 | 15 | func TestRenderMastodon(t *testing.T) { 16 | const toot = `‹ Example › 17 | 18 | • Source 19 | > https://example.com/ 20 | 21 | ———— 22 | 23 | • Internet Archive 24 | > https://web.archive.org/web/20211000000001/https://example.com/ 25 | 26 | • archive.today 27 | > http://archive.today/abcdE 28 | 29 | • IPFS 30 | > https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr 31 | 32 | • Telegraph 33 | > http://telegra.ph/title-01-01 34 | 35 | #wayback #存档` 36 | 37 | got := ForPublish(&Mastodon{Cols: collects, Data: bundleExample}).String() 38 | if got != toot { 39 | t.Fatalf("Unexpected render template for Mastodon, got \n%s\ninstead of \n%s", got, toot) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /template/render/nostr.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All ritts reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "bytes" 9 | "text/template" 10 | 11 | "github.com/wabarc/logger" 12 | "github.com/wabarc/wayback" 13 | "github.com/wabarc/wayback/reduxer" 14 | ) 15 | 16 | var _ Renderer = (*Nostr)(nil) 17 | 18 | // Nostr represents a Nostr template data for render. 19 | type Nostr struct { 20 | Cols []wayback.Collect 21 | Data reduxer.Reduxer 22 | } 23 | 24 | // ForReply implements the standard Renderer interface: 25 | // it reads `[]wayback.Collect` from the Nostr and returns a *Render. 26 | func (n *Nostr) ForReply() *Render { 27 | return n.ForPublish() 28 | } 29 | 30 | // ForPublish implements the standard Renderer interface: 31 | // it reads `[]wayback.Collect` and `reduxer.Reduxer` from 32 | // the Nostr and returns a *Render. 33 | // 34 | // ForPublish generate tweet of given wayback collects in Nostr struct. 35 | // It excluded telegra.ph, because this link has been identified by Nostr. 36 | func (n *Nostr) ForPublish() *Render { 37 | var tmplBytes bytes.Buffer 38 | 39 | if title := Title(n.Cols, n.Data); title != "" { 40 | tmplBytes.WriteString(`‹ `) 41 | tmplBytes.WriteString(title) 42 | tmplBytes.WriteString(" ›\n\n") 43 | } 44 | 45 | const tmpl = `{{range $ := .}} 46 | • {{ $.Arc | name }} 47 | > {{ $.Dst }} 48 | {{end}}` 49 | 50 | tpl, err := template.New("nostr").Funcs(funcMap()).Parse(tmpl) 51 | if err != nil { 52 | logger.Error("parse Nostr template failed: %v", err) 53 | return new(Render) 54 | } 55 | 56 | tmplBytes.WriteString(original(n.Cols)) 57 | if err := tpl.Execute(&tmplBytes, n.Cols); err != nil { 58 | logger.Error("execute Nostr template failed: %v", err) 59 | return new(Render) 60 | } 61 | 62 | return &Render{buf: tmplBytes} 63 | } 64 | -------------------------------------------------------------------------------- /template/render/nostr_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestRenderNostrForReply(t *testing.T) { 12 | TestRenderNostrForPublish(t) 13 | } 14 | 15 | func TestRenderNostrForPublish(t *testing.T) { 16 | expected := `‹ Example › 17 | 18 | • Source 19 | > https://example.com/ 20 | 21 | ———— 22 | 23 | • Internet Archive 24 | > https://web.archive.org/web/20211000000001/https://example.com/ 25 | 26 | • archive.today 27 | > http://archive.today/abcdE 28 | 29 | • IPFS 30 | > https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr 31 | 32 | • Telegraph 33 | > http://telegra.ph/title-01-01` 34 | 35 | got := ForPublish(&Nostr{Cols: collects, Data: bundleExample}).String() 36 | if got != expected { 37 | t.Errorf("Unexpected render template for IRC, got \n%s\ninstead of \n%s", got, expected) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /template/render/notion.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "bytes" 9 | 10 | "github.com/wabarc/logger" 11 | "github.com/wabarc/wayback" 12 | "github.com/wabarc/wayback/reduxer" 13 | ) 14 | 15 | var _ Renderer = (*Notion)(nil) 16 | 17 | // Notion represents a Notion template data for render. 18 | type Notion struct { 19 | Cols []wayback.Collect 20 | Data reduxer.Reduxer 21 | } 22 | 23 | // ForReply implements the standard Renderer interface: 24 | // it returns a Render from the ForPublish. 25 | func (no *Notion) ForReply() *Render { 26 | return no.ForPublish() 27 | } 28 | 29 | // ForPublish implements the standard Renderer interface: 30 | // it reads `[]wayback.Collect` and `reduxer.Reduxer` from 31 | // the Notion and returns a *Render. 32 | func (no *Notion) ForPublish() *Render { 33 | var tmplBytes bytes.Buffer 34 | 35 | rdx := no.Data 36 | for uri := range deDepURI(no.Cols) { 37 | if bundle, ok := rdx.Load(reduxer.Src(uri)); ok { 38 | if html := bundle.Article().Content; html != "" { 39 | logger.Debug("generate digest from article content: %s", html) 40 | tmplBytes.WriteString(html) 41 | } 42 | } 43 | } 44 | 45 | return &Render{buf: tmplBytes} 46 | } 47 | -------------------------------------------------------------------------------- /template/render/notion_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/wabarc/wayback" 11 | "github.com/wabarc/wayback/config" 12 | ) 13 | 14 | func TestRenderNotion(t *testing.T) { 15 | collects := []wayback.Collect{ 16 | { 17 | Arc: config.SLOT_IA, 18 | Dst: "https://web.archive.org/web/20211000000001/https://example.com/", 19 | Src: "https://example.com/", 20 | Ext: config.SLOT_IA, 21 | }, 22 | { 23 | Arc: config.SLOT_IS, 24 | Dst: "http://archive.today/abcdE", 25 | Src: "https://example.com/", 26 | Ext: config.SLOT_IS, 27 | }, 28 | { 29 | Arc: config.SLOT_IP, 30 | Dst: "https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr", 31 | Src: "https://example.com/", 32 | Ext: config.SLOT_IP, 33 | }, 34 | { 35 | Arc: config.SLOT_PH, 36 | Dst: "http://telegra.ph/title-01-01", 37 | Src: "https://example.com/", 38 | Ext: config.SLOT_PH, 39 | }, 40 | } 41 | 42 | expected := ` 43 | 44 | 45 | Example Domain 46 | 47 | 48 | 49 |
50 |

Example Domain

51 |

This domain is for use in illustrative examples in documents. You may use this 52 | domain in literature without prior coordination or asking for permission.

53 |

More information...

54 |
55 | 56 | ` 57 | 58 | got := ForPublish(&Notion{Cols: collects, Data: bundleExample}).String() 59 | if got != expected { 60 | t.Errorf("Unexpected render template for Notion Issues, got \n%s\ninstead of \n%s", got, expected) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /template/render/relaychat.go: -------------------------------------------------------------------------------- 1 | // Copyriit 2021 Wayback Archiver. All riits reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "bytes" 9 | "text/template" 10 | 11 | "github.com/wabarc/logger" 12 | "github.com/wabarc/wayback" 13 | "github.com/wabarc/wayback/reduxer" 14 | ) 15 | 16 | var _ Renderer = (*Relaychat)(nil) 17 | 18 | // Relaychat represents a Relaychat template data for render. 19 | type Relaychat struct { 20 | Cols []wayback.Collect 21 | Data reduxer.Reduxer 22 | } 23 | 24 | // ForReply implements the standard Renderer interface: 25 | // it returns a Render from the ForPublish. 26 | func (i *Relaychat) ForReply() *Render { 27 | buf := i.join(i.main()) 28 | 29 | return &Render{buf: *buf} 30 | } 31 | 32 | // ForPublish implements the standard Renderer interface: 33 | // it reads `[]wayback.Collect` from the Relaychat and returns a *Render. 34 | func (i *Relaychat) ForPublish() *Render { 35 | tmplBytes := &bytes.Buffer{} 36 | 37 | if title := Title(i.Cols, i.Data); title != "" { 38 | tmplBytes.WriteString(`‹ `) 39 | tmplBytes.WriteString(title) 40 | tmplBytes.WriteString(" ›") 41 | tmplBytes.WriteString("\n \n") 42 | } 43 | // tmplBytes.WriteString("Source:\n") 44 | tmplBytes.WriteString(original(i.Cols)) 45 | tmplBytes.WriteString(" \n") 46 | tmplBytes.Write(i.main().Bytes()) 47 | 48 | return &Render{buf: *i.join(tmplBytes)} 49 | } 50 | 51 | func (i *Relaychat) main() *bytes.Buffer { 52 | tmplBytes := new(bytes.Buffer) 53 | 54 | const tmpl = "{{range $ := .}}• {{ $.Arc | name }}:\n> {{ $.Dst }}\n{{end}}" 55 | 56 | tpl, err := template.New("relaychat").Funcs(funcMap()).Parse(tmpl) 57 | if err != nil { 58 | logger.Error("parse IRC template failed, %v", err) 59 | return new(bytes.Buffer) 60 | } 61 | 62 | if err := tpl.Execute(tmplBytes, i.Cols); err != nil { 63 | logger.Error("execute IRC template failed, %v", err) 64 | return new(bytes.Buffer) 65 | } 66 | 67 | return bytes.NewBuffer(bytes.TrimRight(tmplBytes.Bytes(), "\n")) 68 | } 69 | 70 | func (i *Relaychat) join(buf *bytes.Buffer) *bytes.Buffer { 71 | tmplBytes := &bytes.Buffer{} 72 | tmplBytes.WriteString("***** List of Archives *****") 73 | tmplBytes.WriteString("\n") 74 | tmplBytes.Write(buf.Bytes()) 75 | tmplBytes.WriteString("\n") 76 | tmplBytes.WriteString("***** End of Archives *****") 77 | return tmplBytes 78 | } 79 | -------------------------------------------------------------------------------- /template/render/relaychat_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestRenderForIRC(t *testing.T) { 12 | expected := `***** List of Archives ***** 13 | ‹ Example › 14 | 15 | • Source 16 | > https://example.com/ 17 | 18 | ———— 19 | 20 | • Internet Archive: 21 | > https://web.archive.org/web/20211000000001/https://example.com/ 22 | • archive.today: 23 | > http://archive.today/abcdE 24 | • IPFS: 25 | > https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr 26 | • Telegraph: 27 | > http://telegra.ph/title-01-01 28 | ***** End of Archives *****` 29 | 30 | got := ForPublish(&Relaychat{Cols: collects, Data: bundleExample}).String() 31 | if got != expected { 32 | t.Errorf("Unexpected render template for IRC, got \n%s\ninstead of \n%s", got, expected) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /template/render/twitter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestRenderTwitterForReply(t *testing.T) { 13 | const tweet = `• Internet Archive 14 | > https://web.archive.org/123/https://example.com/ 15 | > https://web.archive.org/123/https://example.org/ 16 | 17 | • archive.today 18 | > http://archive.today/abcdE 19 | > http://archive.today/abc 20 | 21 | #wayback #存档` 22 | 23 | got := ForReply(&Twitter{Cols: multi, Data: bundleExample}).String() 24 | if !strings.Contains(got, tweet) { 25 | t.Errorf("Unexpected render template for Twitter, got \n%s\ninstead of \n%s", got, tweet) 26 | } 27 | } 28 | 29 | func TestRenderTwitterForPublish(t *testing.T) { 30 | const tweet = `‹ Example › 31 | 32 | • Source 33 | > https://example.com/ 34 | 35 | ———— 36 | 37 | • Internet Archive 38 | > https://web.archive.org/web/20211000000001/https://example.com/ 39 | 40 | • archive.today 41 | > http://archive.today/abcdE 42 | 43 | • IPFS 44 | > https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr 45 | 46 | #wayback #存档` 47 | 48 | got := ForPublish(&Twitter{Cols: collects, Data: bundleExample}).String() 49 | if got != tweet { 50 | t.Errorf("Unexpected render template for Twitter got \n%s\ninstead of \n%s", got, tweet) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /template/render/xmpp.go: -------------------------------------------------------------------------------- 1 | // Copyriit 2023 Wayback Archiver. All riits reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "bytes" 9 | "text/template" 10 | 11 | "github.com/wabarc/logger" 12 | "github.com/wabarc/wayback" 13 | ) 14 | 15 | var _ Renderer = (*XMPP)(nil) 16 | 17 | // XMPP represents a XMPP template data for render. 18 | type XMPP struct { 19 | Cols []wayback.Collect 20 | Data interface{} 21 | } 22 | 23 | // ForReply implements the standard Renderer interface: 24 | // it returns a Render from the ForPublish. 25 | func (x *XMPP) ForReply() *Render { 26 | return x.ForPublish() 27 | } 28 | 29 | // ForPublish implements the standard Renderer interface: 30 | // it reads `[]wayback.Collect` from the XMPP and returns a *Render. 31 | func (x *XMPP) ForPublish() (r *Render) { 32 | var tmplBytes bytes.Buffer 33 | 34 | const tmpl = `{{range $ := .}}{{ $.Arc | name }}: 35 | {{ range $map := $.Dst -}} 36 | {{ range $src, $dst := $map -}} 37 | • {{ if $dst | isURL }}{{ $dst }}{{ else }}{{ $dst | escapeString }}{{ end }} 38 | {{ end }}{{ end }} 39 | {{ end }}` 40 | 41 | tpl, err := template.New("message").Funcs(funcMap()).Parse(tmpl) 42 | if err != nil { 43 | logger.Error("parse Telegram template failed, %v", err) 44 | return r 45 | } 46 | 47 | groups := groupBySlot(x.Cols) 48 | logger.Debug("for reply telegram: %#v", groups) 49 | if err = tpl.Execute(&tmplBytes, groups); err != nil { 50 | logger.Error("execute Telegram template failed, %v", err) 51 | return r 52 | } 53 | tmplBytes = *bytes.NewBuffer(bytes.TrimSpace(tmplBytes.Bytes())) 54 | tmplBytes.WriteString("\n") 55 | 56 | return &Render{buf: tmplBytes} 57 | } 58 | -------------------------------------------------------------------------------- /template/render/xmpp_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package render // import "github.com/wabarc/wayback/template/render" 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func TestRenderForXMPP(t *testing.T) { 12 | expected := `Internet Archive: 13 | • https://web.archive.org/web/20211000000001/https://example.com/ 14 | 15 | IPFS: 16 | • https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr 17 | 18 | archive.today: 19 | • http://archive.today/abcdE 20 | 21 | Telegraph: 22 | • http://telegra.ph/title-01-01` 23 | 24 | got := ForPublish(&XMPP{Cols: collects, Data: bundleExample}).String() 25 | if got != expected { 26 | t.Errorf("Unexpected render template for XMPP, got \n%s\ninstead of \n%s", got, expected) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /template/views/offline.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Wayback Archiver 10 | 66 | 67 | 68 | 69 |

You are offline. Quit and reopen try again.

70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /version/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Package version contains application and build information. 7 | */ 8 | package version // import "github.com/wabarc/wayback/version" 9 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Wayback Archiver. All rights reserved. 2 | // Use of this source code is governed by the GNU GPL v3 3 | // license that can be found in the LICENSE file. 4 | 5 | package version // import "github.com/wabarc/wayback/version" 6 | 7 | // Variables populated at build time. 8 | var ( 9 | Version = "dev" 10 | Commit = "HEAD" 11 | BuildDate = "undefined" 12 | ) 13 | --------------------------------------------------------------------------------