├── .dockerignore ├── .env-sample ├── .github ├── FUNDING.yml └── workflows │ ├── clippy-fmt.yml │ ├── coverage.yml │ └── linux.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE.md ├── Makefile ├── README.md ├── api_routes ├── Cargo.toml └── src │ └── lib.rs ├── build.rs ├── config └── default.toml ├── db ├── db-core │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ │ ├── errors.rs │ │ ├── lib.rs │ │ ├── ops.rs │ │ └── tests.rs ├── db-sqlx-sqlite │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ ├── migrations │ │ ├── 20220405113942_world_forges.sql │ │ ├── 20230211075309_starchart_import.sql │ │ ├── 20230223063159_starchart_fts_repository.sql │ │ ├── 20230228083200_starchart_introducer.sql │ │ ├── 20230302072543_starchart_mini_index.sql │ │ ├── 20230302094417_starchart_federated_mini_index.sql │ │ └── 20230302132435_starchart_imported_starcharts.sql │ ├── sqlx-data.json │ └── src │ │ ├── errors.rs │ │ ├── lib.rs │ │ └── tests.rs └── migrator │ ├── .gitignore │ ├── Cargo.lock │ ├── Cargo.toml │ └── src │ └── main.rs ├── docker-compose-dev-deps.yml ├── docs ├── ARCHITECTURE.md ├── HACKING.md └── published-crwaled-data.md ├── federate ├── federate-core │ ├── Cargo.toml │ └── src │ │ ├── errors.rs │ │ ├── lib.rs │ │ └── tests.rs └── publiccodeyml │ ├── .gitignore │ ├── Cargo.toml │ └── src │ ├── errors.rs │ ├── lib.rs │ ├── schema.rs │ └── tests.rs ├── forge ├── forge-core │ ├── Cargo.toml │ └── src │ │ └── lib.rs └── gitea │ ├── Cargo.toml │ ├── src │ ├── lib.rs │ └── schema.rs │ └── tests │ └── schema │ └── gitea │ └── git.batsense.net.json ├── scripts ├── entrypoint.sh └── gitea.py ├── sqlx-data.json ├── src ├── api.rs ├── counter.rs ├── ctx.rs ├── db.rs ├── dns │ └── mod.rs ├── errors.rs ├── federate.rs ├── introduce.rs ├── main.rs ├── master.rs ├── pages │ ├── auth │ │ ├── add.rs │ │ ├── mod.rs │ │ └── verify.rs │ ├── chart │ │ ├── home.rs │ │ ├── mod.rs │ │ └── search.rs │ ├── errors.rs │ ├── mod.rs │ └── routes.rs ├── routes.rs ├── search.rs ├── settings.rs ├── spider.rs ├── static_assets │ ├── filemap.rs │ ├── mod.rs │ └── static_files.rs ├── tests.rs ├── utils.rs └── verify.rs ├── static └── cache │ └── css │ └── main.css ├── templates ├── components │ ├── base.html │ ├── error.html │ ├── footer.html │ └── nav │ │ ├── base.html │ │ ├── pub.html │ │ └── search.html ├── index.html └── pages │ ├── auth │ ├── add.html │ └── challenge.html │ └── chart │ ├── components │ └── repo_info.html │ ├── index.html │ └── search.html └── utils └── cache-bust ├── .gitignore ├── Cargo.lock ├── Cargo.toml └── src └── main.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | **/target 2 | tarpaulin-report.html 3 | .env 4 | cobertura.xml 5 | prod/ 6 | node_modules/ 7 | /static-assets/bundle 8 | ./templates/**/*.js 9 | /static/cache/bundle/* 10 | src/cache_buster_data.json 11 | 12 | browser/target 13 | browser/cobertura.xml 14 | browser/docs 15 | **/tmp/ 16 | -------------------------------------------------------------------------------- /.env-sample: -------------------------------------------------------------------------------- 1 | export POSTGRES_DATABASE_URL="postgres://postgres:password@localhost:5432/postgres" 2 | export SQLITE_TMP="$(pwd)/db/db-sqlx-sqlite/tmp" 3 | export SQLITE_DATABASE_URL="sqlite://$SQLITE_TMP/admin.db" 4 | export STARCHART__CRAWLER__WAIT_BEFORE_NEXT_API_CALL=0 5 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | # github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | # patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | # ko_fi: # Replace with a single Ko-fi username 7 | # tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | # community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: realaravinth 10 | issuehunt: # Replace with a single IssueHunt username 11 | # otechie: # Replace with a single Otechie username 12 | custom: ['https://batsense.net/donate'] 13 | -------------------------------------------------------------------------------- /.github/workflows/clippy-fmt.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened] 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | fmt: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Install Rust 17 | uses: actions-rs/toolchain@v1 18 | with: 19 | toolchain: stable 20 | components: rustfmt 21 | - name: Check with rustfmt 22 | uses: actions-rs/cargo@v1 23 | with: 24 | command: fmt 25 | args: --all -- --check 26 | 27 | clippy: 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v2 31 | 32 | - name: Install Rust 33 | uses: actions-rs/toolchain@v1 34 | with: 35 | toolchain: stable 36 | components: clippy 37 | override: true 38 | 39 | - name: Check with Clippy 40 | uses: actions-rs/clippy-check@v1 41 | with: 42 | token: ${{ secrets.GITHUB_TOKEN }} 43 | args: --workspace --tests --all-features 44 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Coverage 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened] 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | build_and_test: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | version: 16 | - stable 17 | 18 | # services: 19 | # postgres: 20 | # image: postgres 21 | # env: 22 | # POSTGRES_PASSWORD: password 23 | # POSTGRES_USER: postgres 24 | # POSTGRES_DB: postgres 25 | # options: >- 26 | # --health-cmd pg_isready 27 | # --health-interval 10s 28 | # --health-timeout 5s 29 | # --health-retries 5 30 | # ports: 31 | # - 5432:5432 32 | 33 | name: ${{ matrix.version }} - x86_64-unknown-linux-gnu 34 | runs-on: ubuntu-latest 35 | 36 | steps: 37 | - uses: actions/checkout@v2 38 | # - name: ⚡ Cache 39 | # uses: actions/cache@v3 40 | # with: 41 | # path: | 42 | # ~/.cargo/registry 43 | # ~/.cargo/git 44 | # target 45 | # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 46 | 47 | - name: Install ${{ matrix.version }} 48 | uses: actions-rs/toolchain@v1 49 | with: 50 | toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu 51 | profile: minimal 52 | override: true 53 | 54 | - name: load env 55 | run: | 56 | mkdir -p db/db-sqlx-sqlite/tmp && 57 | source .env-sample \ 58 | && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \ 59 | && echo "SQLITE_DATABASE_URL=$SQLITE_DATABASE_URL" >> $GITHUB_ENV 60 | 61 | # usually run as part of `make test` but because this workflow doesn't run 62 | # that command, `make dev-env` is used 63 | - name: setup dev environment 64 | run: make dev-env 65 | env: 66 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value 67 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 68 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 69 | 70 | - name: run migrations 71 | run: make migrate 72 | env: 73 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value 74 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 75 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 76 | 77 | - name: Generate coverage file 78 | if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'pull_request') 79 | uses: actions-rs/tarpaulin@v0.1 80 | env: 81 | # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin 82 | # execution so this value is required for preventing meta tests from 83 | # panicking 84 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 85 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 86 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 87 | with: 88 | args: "--all-features --no-fail-fast --workspace=db/db-sqlx-sqlite,. -t 1200" 89 | # args: "--all-features --no-fail-fast --workspace=database/db-sqlx-postgres,database/db-sqlx-sqlite,. -t 1200" 90 | 91 | - name: Upload to Codecov 92 | if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'pull_request') 93 | uses: codecov/codecov-action@v2 94 | -------------------------------------------------------------------------------- /.github/workflows/linux.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | pull_request: 5 | types: [opened, synchronize, reopened] 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | build_and_test: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | version: 16 | - stable 17 | # - nightly 18 | 19 | name: ${{ matrix.version }} - x86_64-unknown-linux-gnu 20 | runs-on: 21 | ubuntu-latest 22 | 23 | # services: 24 | # postgres: 25 | # image: postgres 26 | # env: 27 | # POSTGRES_PASSWORD: password 28 | # POSTGRES_USER: postgres 29 | # POSTGRES_DB: postgres 30 | # options: >- 31 | # --health-cmd pg_isready 32 | # --health-interval 10s 33 | # --health-timeout 5s 34 | # --health-retries 5 35 | # ports: 36 | # - 5432:5432 37 | # 38 | steps: 39 | - uses: actions/checkout@v2 40 | 41 | # - name: ⚡ Cache 42 | # uses: actions/cache@v3 43 | # with: 44 | # path: | 45 | # /var/lib/docker 46 | # ~/.cargo/registry 47 | # ~/.cargo/git 48 | # target 49 | # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 50 | 51 | 52 | - name: Cache 53 | uses: Swatinem/rust-cache@v1 54 | 55 | - name: Login to DockerHub 56 | if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' 57 | uses: docker/login-action@v1 58 | with: 59 | username: ${{ secrets.DOCKERHUB_USERNAME }} 60 | password: ${{ secrets.DOCKERHUB_TOKEN }} 61 | 62 | - name: Install ${{ matrix.version }} 63 | uses: actions-rs/toolchain@v1 64 | with: 65 | toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu 66 | profile: minimal 67 | override: true 68 | 69 | - name: load env 70 | run: | 71 | mkdir -p db/db-sqlx-sqlite/tmp && 72 | source .env-sample \ 73 | && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \ 74 | && echo "SQLITE_DATABASE_URL=$SQLITE_DATABASE_URL" >> $GITHUB_ENV 75 | 76 | - name: run migrations 77 | run: make migrate 78 | env: 79 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value 80 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 81 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 82 | 83 | - name: build 84 | run: 85 | make 86 | env: 87 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 88 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 89 | 90 | - name: build docker images 91 | if: matrix.version == 'stable' 92 | run: make docker 93 | 94 | - name: publish docker images 95 | if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' 96 | run: make docker-publish 97 | 98 | - name: run tests 99 | timeout-minutes: 40 100 | run: 101 | make test 102 | env: 103 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value 104 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 105 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 106 | 107 | - name: generate documentation 108 | if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' 109 | run: 110 | make doc 111 | env: 112 | GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61 # dummy value 113 | POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}" 114 | SQLITE_DATABASE_URL: "${{ env.SQLITE_DATABASE_URL }}" 115 | 116 | - name: Deploy to GitHub Pages 117 | if: matrix.version == 'stable' && (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'forgeflux-org/starchart' 118 | uses: JamesIves/github-pages-deploy-action@3.7.1 119 | with: 120 | branch: gh-pages 121 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 122 | FOLDER: 123 | ./target/doc/ 124 | 125 | # - name: deploy 126 | # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'realaravinth/realaravinth' }} 127 | # run: >- 128 | # curl --location --request POST "https://deploy.batsense.net/api/v1/update" --header 'Content-Type: application/json' --data-raw "{ \"secret\": \"${{ secrets.DEPLOY_TOKEN }}\", \"branch\": \"gh-pages\" }" 129 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env 3 | **/target/ 4 | tmp 5 | tarpaulin-report.html 6 | src/cache_buster_data.json 7 | assets 8 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "starchart" 3 | repository = "https://github.com/forgeflux-org/starchart" 4 | version = "0.1.0" 5 | authors = ["realaravinth "] 6 | description = "ForgeFlux StarChart - Federated forge spider" 7 | documentation = "https://forgeflux.org/" 8 | edition = "2021" 9 | license = "AGPLv3 or later version" 10 | build = "build.rs" 11 | 12 | [workspace] 13 | exclude = ["db/migrator", "utils/cache-bust"] 14 | members = [ 15 | ".", 16 | "db/db-core", 17 | "db/db-sqlx-sqlite", 18 | "forge/forge-core", 19 | "forge/gitea", 20 | "federate/federate-core", 21 | "federate/publiccodeyml" 22 | ] 23 | 24 | [dependencies] 25 | actix-rt = "2.7" 26 | actix-web = "4.0.1" 27 | actix-identity = "0.4.0" 28 | actix-files = "0.6.0" 29 | async-trait = "0.1.51" 30 | config = "0.13.0" 31 | lazy_static = "1.4.0" 32 | mime = "0.3.16" 33 | mime_guess = "2.0.3" 34 | rand = "0.8.5" 35 | tera = "1.15" 36 | tokio = { version = "1.17", features = ["fs", "time", "sync"] } 37 | url = { version = "2.2.2", features = ["serde"] } 38 | validator = { version = "0.15", features = ["derive"]} 39 | derive_more = "0.99.17" 40 | log = "0.4.16" 41 | pretty_env_logger = "0.4" 42 | rust-embed = "6.3.0" 43 | urlencoding = "2.1.0" 44 | clap = { version = "4.0.32", features = ["derive"] } 45 | api_routes = { path ="./api_routes/"} 46 | actix = "0.13.0" 47 | derive_builder = "0.12.0" 48 | 49 | [dependencies.cache-buster] 50 | git = "https://github.com/realaravinth/cache-buster" 51 | 52 | [dependencies.actix-web-codegen-const-routes] 53 | git = "https://github.com/realaravinth/actix-web-codegen-const-routes" 54 | 55 | [dependencies.reqwest] 56 | features = ["rustls-tls-native-roots", "gzip", "deflate", "brotli", "json"] 57 | version = "0.11.10" 58 | 59 | [dependencies.serde] 60 | features = ["derive"] 61 | version = "1" 62 | 63 | [dependencies.serde_json] 64 | version = "1" 65 | 66 | [dependencies.trust-dns-resolver] 67 | features = ["tokio-runtime", "dns-over-tls", "dns-over-rustls"] 68 | version = "0.21.1" 69 | 70 | [dependencies.db-core] 71 | path = "./db/db-core" 72 | 73 | [dependencies.db-sqlx-sqlite] 74 | path = "./db/db-sqlx-sqlite" 75 | 76 | [dependencies.gitea] 77 | path = "./forge/gitea" 78 | 79 | [dependencies.forge-core] 80 | path = "./forge/forge-core" 81 | 82 | [dependencies.federate-core] 83 | path = "./federate/federate-core" 84 | 85 | [dependencies.publiccodeyml] 86 | path = "./federate/publiccodeyml" 87 | 88 | [dependencies.sqlx] 89 | features = ["runtime-actix-rustls", "uuid", "postgres", "time", "offline", "sqlite"] 90 | version = "0.6.2" 91 | 92 | [dev-dependencies] 93 | mktemp = "0.4.1" 94 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:latest as planner 2 | RUN cargo install cargo-chef 3 | WORKDIR /src 4 | COPY . /src/ 5 | RUN cargo chef prepare --recipe-path recipe.json 6 | 7 | 8 | FROM rust:latest as cacher 9 | WORKDIR /src/ 10 | RUN cargo install cargo-chef 11 | COPY --from=planner /src/recipe.json recipe.json 12 | RUN cargo chef cook --release --recipe-path recipe.json 13 | 14 | 15 | FROM rust:latest as builder 16 | WORKDIR /src/ 17 | COPY . . 18 | COPY --from=cacher /src/target target 19 | RUN make release 20 | 21 | FROM debian:bullseye-slim 22 | LABEL org.opencontainers.image.source https://github.com/forgeflux-org/starchart 23 | RUN apt-get update && apt-get install -y ca-certificates 24 | COPY --from=builder /src/target/release/starchart /usr/local/bin/ 25 | COPY --from=builder /src/config/default.toml /etc/starchart/config.toml 26 | COPY scripts/entrypoint.sh /usr/local/bin 27 | RUN chmod +x /usr/local/bin/entrypoint.sh 28 | CMD [ "/usr/local/bin/entrypoint.sh" ] 29 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | define launch_test_env 2 | docker-compose -f docker-compose-dev-deps.yml up --detach 3 | python ./scripts/gitea.py 4 | endef 5 | 6 | define test_databases 7 | cd db/db-core &&\ 8 | cargo test --no-fail-fast 9 | cd db/db-sqlx-sqlite &&\ 10 | DATABASE_URL=${SQLITE_DATABASE_URL}\ 11 | cargo test --no-fail-fast 12 | endef 13 | 14 | define test_forges 15 | cd forge/forge-core && \ 16 | cargo test --no-fail-fast 17 | cd forge/gitea && \ 18 | cargo test --no-fail-fast 19 | endef 20 | 21 | define test_federation 22 | cd federate/federate-core && \ 23 | cargo test --no-fail-fast 24 | cd federate/publiccodeyml && \ 25 | cargo test --no-fail-fast 26 | endef 27 | 28 | define cache_bust ## run cache_busting program 29 | cd utils/cache-bust && cargo run 30 | endef 31 | 32 | define test_workspaces 33 | $(call test_databases) 34 | $(call test_forges) 35 | $(call test_federation) 36 | DATABASE_URL=${SQLITE_DATABASE_URL}\ 37 | cargo test --no-fail-fast 38 | endef 39 | 40 | default: ## Debug build 41 | $(call cache_bust) 42 | cargo build 43 | 44 | env.serve: 45 | STARCHART__LOG=info \ 46 | STARCHART__SOURCE_CODE="https://github.com/forgeflux-org/starchart" \ 47 | STARCHART__ALLOW_NEW_INDEX=true \ 48 | STARCHART__ADMIN_EMAIL=realaravinth@batsense.net \ 49 | STARCHART__SERVER__IP=0.0.0.0 \ 50 | STARCHART__SERVER__PORT=7000 \ 51 | STARCHART__SERVER__DOMAIN=localhost \ 52 | STARCHART__SERVER__PROXY_HAS_TLS=false \ 53 | STARCHART__SERVER__COOKIE_SECRET=7514316e58bfdb2eb2d71bf4af40827a \ 54 | STARCHART__DATABASE__POOL=5 STARCHART__DATABASE__TYPE=sqlite \ 55 | STARCHART__CRAWLER__TTL=3600 \ 56 | STARCHART__CRAWLER__WAIT_BEFORE_NEXT_API_CALL=2 \ 57 | STARCHART__CRAWLER__CLIENT_TIMEOUT=60 \ 58 | STARCHART__CRAWLER__ITEMS_PER_API_CALL=20 \ 59 | STARCHART__INTRODUCER__PUBLIC_URL="http://localhost:7000" \ 60 | STARCHART__INTRODUCER__NODES=http://localhost:7001,http://localhost:7002 \ 61 | STARCHART__REPOSITORY__ROOT=/tmp/starchart.forgeflux.org \ 62 | cargo run 63 | 64 | cache-bust: ## Run cache buster on static assets 65 | $(call cache_bust) 66 | 67 | clean: ## Clean all build artifacts and dependencies 68 | @-/bin/rm -rf target/ 69 | @-/bin/rm -rf database/migrator/target/ 70 | @-/bin/rm -rf database/*/target/ 71 | @-/bin/rm -rf database/*/tmp/ 72 | @cargo clean 73 | 74 | coverage: migrate ## Generate coverage report in HTML format 75 | $(call launch_test_env) 76 | $(call cache_bust) 77 | cargo tarpaulin -t 1200 --out Html --skip-clean --all-features --no-fail-fast --workspace=db/db-sqlx-sqlite,forge/gitea,federate/publiccodeyml,. 78 | 79 | check: ## Check for syntax errors on all workspaces 80 | cargo check --workspace --tests --all-features 81 | cd db/db-sqlx-sqlite &&\ 82 | DATABASE_URL=${SQLITE_DATABASE_URL}\ 83 | cargo check 84 | cd db/db-core/ && cargo check 85 | cd db/migrator && cargo check --tests --all-features 86 | cd forge/forge-core && cargo check --tests --all-features 87 | cd forge/gitea && cargo check --tests --all-features 88 | cd federate/federate-core && cargo check --tests --all-features 89 | cd federate/publiccodeyml && cargo check --tests --all-features 90 | cd utils/cache-bust && cargo check --tests --all-features 91 | cd api_routes && cargo check --tests --all-features 92 | 93 | dev-env: ## Download development dependencies 94 | $(call launch_test_env) 95 | cargo fetch 96 | 97 | doc: ## Prepare documentation 98 | cargo doc --no-deps --workspace --all-features 99 | 100 | docker: ## Build docker images 101 | docker build -t forgedfed/starchart:master -t forgedfed/starchart:latest . 102 | 103 | docker-publish: docker ## Build and publish docker images 104 | docker push forgedfed/starchart:master 105 | docker push forgedfed/starchart:latest 106 | 107 | lint: ## Lint codebase 108 | cargo fmt -v --all -- --emit files 109 | cargo clippy --workspace --tests --all-features 110 | 111 | release: ## Release build 112 | $(call cache_bust) 113 | cargo build --release 114 | 115 | run: default ## Run debug build 116 | cargo run 117 | 118 | migrate: ## run migrations 119 | @-rm -rf db/db-sqlx-sqlite/tmp && mkdir db/db-sqlx-sqlite/tmp 120 | @-rm -rf db/migrator/target/ 121 | cd db/migrator && cargo run 122 | # echo TODO: add migrations 123 | 124 | sqlx-offline-data: ## prepare sqlx offline data 125 | cd db/db-sqlx-sqlite/ \ 126 | && DATABASE_URL=${SQLITE_DATABASE_URL} cargo sqlx prepare 127 | # cargo sqlx prepare --database-url=${POSTGRES_DATABASE_URL} -- --bin starchart \ 128 | --all-features 129 | test: migrate ## Run tests 130 | $(call launch_test_env) 131 | $(call cache_bust) 132 | $(call test_workspaces) 133 | 134 | # cd database/db-sqlx-postgres &&\ 135 | # DATABASE_URL=${POSTGRES_DATABASE_URL}\ 136 | # cargo test --no-fail-fast 137 | 138 | xml-test-coverage: migrate ## Generate cobertura.xml test coverage 139 | $(call launch_test_env) 140 | $(call cache_bust) 141 | cargo tarpaulin -t 1200 --out XMl --skip-clean --all-features --no-fail-fast --workspace=db/db-sqlx-sqlite,forge/gitea,federate/publiccodeyml,. 142 | 143 | help: ## Prints help for targets with comments 144 | @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # ForgeFlux StarChart 4 | 5 | [![Documentation](https://img.shields.io/badge/docs-master-blue?style=flat-square)](https://forgeflux-org.github.io/starchart/starchart/) 6 | [![Build](https://github.com/forgeflux-org/starchart/actions/workflows/linux.yml/badge.svg)](https://github.com/forgeflux-org/starchart/actions/workflows/linux.yml) 7 | [![dependency status](https://deps.rs/repo/github/forgeflux-org/starchart/status.svg?style=flat-square)](https://deps.rs/repo/github/forgeflux-org/starchart) 8 | [![codecov](https://codecov.io/gh/forgeflux-org/starchart/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/forgeflux-org/starchart) 9 |
10 | [![AGPL License](https://img.shields.io/badge/license-AGPL-blue.svg?style=flat-square)](http://www.gnu.org/licenses/agpl-3.0) 11 | [![Chat](https://img.shields.io/badge/matrix-+forgefederation:matrix.batsense.net-purple?style=flat-square)](https://matrix.to/#/#forgefederation:matrix.batsense.net) 12 | 13 |
14 | 15 | ## Why 16 | 17 | There are several small, private forges that host Free Software projects. 18 | Some of these Forges might one day participate in the federated 19 | ecosystem. So it would make sense to have a system(see 20 | [spider mechanism](#consensual-spidering)) that would map and advertise these instances 21 | and the projects that they host. 22 | 23 | ## Consensual Spidering 24 | 25 | We are aware that spiders some [very 26 | aggressive](https://git.sr.ht/~sircmpwn/sr.ht-nginx/commit/d8b0bd6aa514a23f5dd3c29168dac7f89f5b64e7) 27 | and small forges are often running on resource-constrained environments. 28 | Therefore, StarChart(this spider) will only crawl a service if the crawl is 29 | requested by the admin of the forge(more accurately, folks that have 30 | access to the DNS associated with the forge's hostname though). 31 | 32 | StarChart will rate limit API calls to one call every 10 seconds. For 33 | instance, a Gitea API call would resemble: 34 | 35 | ```bash 36 | curl -X 'GET' \ 37 | 'https://gitea.example.org/api/v1/repos/search?page=2&limit=20' \ 38 | -H 'accept: application/json' 39 | ``` 40 | 41 | ## Contributing 42 | 43 | Thanks for considering contributing on GitHub. If you are not an GitHub 44 | but would like to contribute to ForgeFlux sub-projects(all repositories 45 | under this organisation), I would be happy to manually mirror this 46 | repository on my [Gitea instance](https://git.batsense.net), which has a 47 | much [more respectful privacy policy](https://batsense.net/privacy-policy) 48 | -------------------------------------------------------------------------------- /api_routes/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "api_routes" 3 | version = "0.1.0" 4 | edition = "2021" 5 | repository = "https://github.com/forgeflux-org/starchart" 6 | authors = ["realaravinth "] 7 | description = "ForgeFlux StarChart - Federated forge spider" 8 | documentation = "https://forgeflux.org/" 9 | license = "AGPLv3 or later version" 10 | 11 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 12 | 13 | [dependencies] 14 | 15 | [dependencies.serde] 16 | features = ["derive"] 17 | version = "1" 18 | 19 | [dependencies.db-core] 20 | path = "../db/db-core" 21 | -------------------------------------------------------------------------------- /api_routes/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2023 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use serde::{Deserialize, Serialize}; 19 | 20 | use db_core::Repository; 21 | 22 | pub const ROUTES: Api = Api::new(); 23 | 24 | #[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] 25 | pub struct Search { 26 | pub repository: &'static str, 27 | } 28 | 29 | impl Search { 30 | const fn new() -> Search { 31 | let repository = "/api/v1/search/repository"; 32 | Search { repository } 33 | } 34 | } 35 | 36 | #[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] 37 | pub struct Introducer { 38 | pub list: &'static str, 39 | pub introduce: &'static str, 40 | pub get_mini_index: &'static str, 41 | } 42 | 43 | impl Introducer { 44 | const fn new() -> Introducer { 45 | let list = "/api/v1/introducer/list"; 46 | let introduce = "/api/v1/introducer/new"; 47 | let get_mini_index = "/api/v1/introducer/mini-index"; 48 | Introducer { 49 | list, 50 | introduce, 51 | get_mini_index, 52 | } 53 | } 54 | } 55 | 56 | #[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] 57 | pub struct MiniIndex { 58 | pub mini_index: String, 59 | } 60 | 61 | #[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] 62 | pub struct Api { 63 | pub get_latest: &'static str, 64 | pub forges: &'static str, 65 | pub search: Search, 66 | pub introducer: Introducer, 67 | } 68 | 69 | impl Api { 70 | const fn new() -> Api { 71 | let get_latest = "/api/v1/federated/latest"; 72 | let forges = "/api/v1/forges/list"; 73 | let search = Search::new(); 74 | let introducer = Introducer::new(); 75 | Api { 76 | get_latest, 77 | search, 78 | forges, 79 | introducer, 80 | } 81 | } 82 | } 83 | 84 | #[derive(Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] 85 | pub struct LatestResp { 86 | pub latest: String, 87 | } 88 | 89 | #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] 90 | pub struct SearchRepositoryReq { 91 | pub query: String, 92 | } 93 | 94 | #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, Eq)] 95 | pub struct SearchRepositoryResp { 96 | pub repositories: Vec, 97 | } 98 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::process::Command; 18 | 19 | //use cache_buster::{BusterBuilder, NoHashCategory}; 20 | 21 | fn main() { 22 | let output = Command::new("git") 23 | .args(["rev-parse", "HEAD"]) 24 | .output() 25 | .expect("error in git command, is git installed?"); 26 | let git_hash = String::from_utf8(output.stdout).unwrap(); 27 | println!("cargo:rustc-env=GIT_HASH={}", git_hash); 28 | 29 | // cache_bust(); 30 | } 31 | 32 | //fn cache_bust() { 33 | // // until APPLICATION_WASM gets added to mime crate 34 | // // PR: https://github.com/hyperium/mime/pull/138 35 | // // let types = vec![ 36 | // // mime::IMAGE_PNG, 37 | // // mime::IMAGE_SVG, 38 | // // mime::IMAGE_JPEG, 39 | // // mime::IMAGE_GIF, 40 | // // mime::APPLICATION_JAVASCRIPT, 41 | // // mime::TEXT_CSS, 42 | // // ]; 43 | // 44 | // println!("cargo:rerun-if-changed=static/cache"); 45 | // let no_hash = vec![NoHashCategory::FileExtentions(vec!["wasm"])]; 46 | // 47 | // let config = BusterBuilder::default() 48 | // .source("./static/cache/") 49 | // .result("./assets") 50 | // .no_hash(no_hash) 51 | // .follow_links(true) 52 | // .build() 53 | // .unwrap(); 54 | // 55 | // config.process().unwrap(); 56 | //} 57 | -------------------------------------------------------------------------------- /config/default.toml: -------------------------------------------------------------------------------- 1 | log = "info" # possible values: "info", "warn", "trace", "error", "debug" 2 | source_code = "https://github.com/forgeflux-org/starchart" 3 | allow_new_index = true # allow registration on server 4 | admin_email = "admin@starchart.example.com" 5 | 6 | [server] 7 | # The port at which you want authentication to listen to 8 | # takes a number, choose from 1000-10000 if you dont know what you are doing 9 | port = 7000 10 | #IP address. Enter 0.0.0.0 to listen on all availale addresses 11 | ip= "0.0.0.0" 12 | # enter your hostname, eg: example.com 13 | domain = "localhost" 14 | proxy_has_tls = false 15 | cookie_secret = "f12d9adf4e364648664442b8f50bf478e748e1d77c4797b2ec1f56803278" 16 | #workers = 2 17 | 18 | [database] 19 | # This section deals with the database location and how to access it 20 | # Please note that at the moment, we have support for only postgresqa. 21 | # Example, if you are Batman, your config would be: 22 | # hostname = "batcave.org" 23 | # port = "5432" 24 | # username = "batman" 25 | # password = "somereallycomplicatedBatmanpassword" 26 | hostname = "localhost" 27 | port = 5432 28 | username = "postgres" 29 | password = "password" 30 | name = "postgres" 31 | pool = 4 32 | database_type = "postgres" 33 | 34 | [crawler] 35 | ttl = 432000 # of crawled records / how often the instance must be polled. In seconds. 36 | items_per_api_call = 20 37 | client_timeout = 60 # of HTTP client involved in crawling. In seconds. 38 | wait_before_next_api_call = 2 # in seconds 39 | 40 | [introducer] 41 | #nodes = ["http://localhost:7000"] 42 | public_url = "http://localhost:7000" 43 | nodes = [] 44 | wait=1 45 | 46 | [repository] 47 | root = "/tmp/starchart.forgeflux.org" 48 | -------------------------------------------------------------------------------- /db/db-core/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env 3 | -------------------------------------------------------------------------------- /db/db-core/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "async-trait" 7 | version = "0.1.53" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" 10 | dependencies = [ 11 | "proc-macro2", 12 | "quote", 13 | "syn", 14 | ] 15 | 16 | [[package]] 17 | name = "db-core" 18 | version = "0.1.0" 19 | dependencies = [ 20 | "async-trait", 21 | "serde", 22 | "serde_json", 23 | "thiserror", 24 | ] 25 | 26 | [[package]] 27 | name = "itoa" 28 | version = "1.0.1" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" 31 | 32 | [[package]] 33 | name = "proc-macro2" 34 | version = "1.0.37" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" 37 | dependencies = [ 38 | "unicode-xid", 39 | ] 40 | 41 | [[package]] 42 | name = "quote" 43 | version = "1.0.17" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" 46 | dependencies = [ 47 | "proc-macro2", 48 | ] 49 | 50 | [[package]] 51 | name = "ryu" 52 | version = "1.0.9" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" 55 | 56 | [[package]] 57 | name = "serde" 58 | version = "1.0.136" 59 | source = "registry+https://github.com/rust-lang/crates.io-index" 60 | checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" 61 | dependencies = [ 62 | "serde_derive", 63 | ] 64 | 65 | [[package]] 66 | name = "serde_derive" 67 | version = "1.0.136" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" 70 | dependencies = [ 71 | "proc-macro2", 72 | "quote", 73 | "syn", 74 | ] 75 | 76 | [[package]] 77 | name = "serde_json" 78 | version = "1.0.79" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" 81 | dependencies = [ 82 | "itoa", 83 | "ryu", 84 | "serde", 85 | ] 86 | 87 | [[package]] 88 | name = "syn" 89 | version = "1.0.91" 90 | source = "registry+https://github.com/rust-lang/crates.io-index" 91 | checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" 92 | dependencies = [ 93 | "proc-macro2", 94 | "quote", 95 | "unicode-xid", 96 | ] 97 | 98 | [[package]] 99 | name = "thiserror" 100 | version = "1.0.30" 101 | source = "registry+https://github.com/rust-lang/crates.io-index" 102 | checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" 103 | dependencies = [ 104 | "thiserror-impl", 105 | ] 106 | 107 | [[package]] 108 | name = "thiserror-impl" 109 | version = "1.0.30" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" 112 | dependencies = [ 113 | "proc-macro2", 114 | "quote", 115 | "syn", 116 | ] 117 | 118 | [[package]] 119 | name = "unicode-xid" 120 | version = "0.2.2" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 123 | -------------------------------------------------------------------------------- /db/db-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "db-core" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/forgeflux-org/starchart" 6 | repository = "https://github.com/forgeflux-org/starchart" 7 | documentation = "https://github.con/forgeflux-org/starchart" 8 | readme = "https://github.com/forgeflux-org/starchart/blob/master/README.md" 9 | license = "AGPLv3 or later version" 10 | authors = ["Aravinth Manivannan "] 11 | 12 | [lib] 13 | name = "db_core" 14 | path = "src/lib.rs" 15 | 16 | 17 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 18 | 19 | [dependencies] 20 | async-trait = "0.1.51" 21 | thiserror = "1.0.30" 22 | serde = { version = "1", features = ["derive"]} 23 | url = { version = "2.2.2", features = ["serde"] } 24 | 25 | [features] 26 | default = [] 27 | test = [] 28 | 29 | [dev-dependencies] 30 | serde_json = "1" 31 | -------------------------------------------------------------------------------- /db/db-core/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | //! represents all the ways a trait can fail using this crate 19 | use std::error::Error as StdError; 20 | 21 | //use derive_more::{error, Error as DeriveError}; 22 | use thiserror::Error; 23 | 24 | /// Error data structure grouping various error subtypes 25 | #[derive(Debug, Error)] 26 | pub enum DBError { 27 | /// DNS challenge value is already taken 28 | #[error("DNS challenge is already taken")] 29 | DuplicateChallengeText, 30 | 31 | /// DNS challenge hostname is already taken 32 | #[error("DNS challenge hostname is already taken")] 33 | DuplicateChallengeHostname, 34 | 35 | /// Hostname is already taken 36 | #[error("Hostname is already taken")] 37 | DuplicateHostname, 38 | 39 | /// Forge Type is already taken 40 | #[error("Forge Type is already taken")] 41 | DuplicateForgeType, 42 | 43 | /// HTML link Type is already taken 44 | #[error("User HTML link is already taken")] 45 | DuplicateUserLink, 46 | 47 | /// Topic is already taken 48 | #[error("Topic is already taken")] 49 | DuplicateTopic, 50 | 51 | /// Repository link is already taken 52 | #[error("Repository link is already taken")] 53 | DuplicateRepositoryLink, 54 | 55 | /// forge instance type is unknown 56 | #[error("Unknown forge instance specifier {}", _0)] 57 | UnknownForgeType(String), 58 | 59 | /// errors that are specific to a database implementation 60 | #[error("{0}")] 61 | DBError(#[source] BoxDynError), 62 | } 63 | 64 | /// Convenience type alias for grouping driver-specific errors 65 | pub type BoxDynError = Box; 66 | 67 | /// Generic result data structure 68 | pub type DBResult = std::result::Result; 69 | -------------------------------------------------------------------------------- /db/db-core/src/ops.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | //! meta operations like migration and connecting to a database 19 | use crate::dev::*; 20 | 21 | /// Database operations trait(migrations, pool creation and fetching connection from pool) 22 | pub trait DBOps: GetConnection + Migrate {} 23 | 24 | /// Get database connection 25 | #[async_trait] 26 | pub trait GetConnection { 27 | /// database connection type 28 | type Conn; 29 | /// database specific error-type 30 | /// get connection from connection pool 31 | async fn get_conn(&self) -> DBResult; 32 | } 33 | 34 | /// Create databse connection 35 | #[async_trait] 36 | pub trait Connect { 37 | /// database specific pool-type 38 | type Pool: SCDatabase; 39 | /// database specific error-type 40 | /// create connection pool 41 | async fn connect(self) -> DBResult; 42 | } 43 | 44 | /// database migrations 45 | #[async_trait] 46 | pub trait Migrate: SCDatabase { 47 | /// database specific error-type 48 | /// run migrations 49 | async fn migrate(&self) -> DBResult<()>; 50 | } 51 | -------------------------------------------------------------------------------- /db/db-core/src/tests.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | //! Test utilities 19 | use crate::prelude::*; 20 | 21 | /// adding forge works 22 | pub async fn adding_forge_works<'a, T: SCDatabase>( 23 | db: &T, 24 | create_forge_msg: CreateForge<'a>, 25 | add_user_msg: AddUser<'a>, 26 | add_user_msg2: AddUser<'a>, 27 | add_repo_msg: AddRepository<'a>, 28 | ) { 29 | let _ = db.delete_forge_instance(&create_forge_msg.url).await; 30 | db.create_forge_instance(&create_forge_msg).await.unwrap(); 31 | assert!( 32 | db.forge_exists(&create_forge_msg.url).await.unwrap(), 33 | "forge creation failed, forge existence check failure" 34 | ); 35 | 36 | { 37 | let forge = db.get_forge(&create_forge_msg.url).await.unwrap(); 38 | let forges = db.get_all_forges(true, 0, 100).await.unwrap(); 39 | assert!(forges 40 | .iter() 41 | .any(|f| f.url == create_forge_msg.url.to_string())); 42 | 43 | assert_eq!(forge.forge_type, create_forge_msg.forge_type); 44 | assert_eq!(forge.url, crate::clean_url(&create_forge_msg.url)); 45 | } 46 | 47 | // add user 48 | db.add_user(&add_user_msg).await.unwrap(); 49 | db.add_user(&add_user_msg2).await.unwrap(); 50 | { 51 | let db_user = db 52 | .get_user(add_user_msg.username, &add_user_msg.url) 53 | .await 54 | .unwrap(); 55 | assert_eq!(db_user.url, crate::clean_url(&add_user_msg.url)); 56 | assert_eq!(db_user.username, add_user_msg.username); 57 | assert_eq!(db_user.html_link, add_user_msg.html_link); 58 | assert_eq!( 59 | db_user.profile_photo, 60 | add_user_msg.profile_photo.map(|s| s.to_owned()) 61 | ); 62 | } 63 | // verify user exists 64 | assert!(db.user_exists(add_user_msg.username, None).await.unwrap()); 65 | assert!(db 66 | .user_exists(add_user_msg.username, Some(&add_user_msg.url)) 67 | .await 68 | .unwrap()); 69 | assert!(db 70 | .is_word_mini_indexed(add_user_msg2.username) 71 | .await 72 | .unwrap()); 73 | 74 | // add repository 75 | db.create_repository(&add_repo_msg).await.unwrap(); 76 | // verify repo exists 77 | assert!(db 78 | .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url) 79 | .await 80 | .unwrap()); 81 | assert!(db.is_word_mini_indexed(add_repo_msg.owner).await.unwrap()); 82 | assert!(db.is_word_mini_indexed(add_repo_msg.name).await.unwrap()); 83 | assert!(db 84 | .is_word_mini_indexed(add_repo_msg.description.unwrap()) 85 | .await 86 | .unwrap()); 87 | assert!(db 88 | .is_word_mini_indexed(add_repo_msg.website.unwrap()) 89 | .await 90 | .unwrap()); 91 | 92 | assert!(!db.get_all_repositories(00, 1000).await.unwrap().is_empty()); 93 | let repo_search = db.search_repository(add_repo_msg.name).await.unwrap(); 94 | 95 | assert!(!repo_search.is_empty()); 96 | assert_eq!(repo_search.first().unwrap().url, add_repo_msg.url.as_str()); 97 | 98 | // delete repository 99 | db.delete_repository(add_repo_msg.owner, add_repo_msg.name, &add_repo_msg.url) 100 | .await 101 | .unwrap(); 102 | assert!(!db 103 | .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url) 104 | .await 105 | .unwrap()); 106 | 107 | // delete user 108 | db.delete_user(add_user_msg.username, &add_user_msg.url) 109 | .await 110 | .unwrap(); 111 | assert!(!db 112 | .user_exists(add_user_msg.username, Some(&add_user_msg.url)) 113 | .await 114 | .unwrap()); 115 | } 116 | 117 | /// test if all forge type implementations are loaded into DB 118 | pub async fn forge_type_exists_helper(db: &T) { 119 | //for f in [ForgeImplementation::Gitea].iter() { 120 | //let f = For 121 | let f = ForgeImplementation::Gitea; 122 | println!("Testing forge implementation exists for: {}", f.to_str()); 123 | assert!(db.forge_type_exists(&f).await.unwrap()); 124 | } 125 | 126 | /// test if all instance introducer methods work 127 | pub async fn instance_introducer_helper(db: &T, instance_url: &Url) { 128 | const MINI_INDEX: &str = "instance_introducer_helper test mini index uniquerq2342"; 129 | db.add_starchart_to_introducer(instance_url).await.unwrap(); 130 | let instances = db 131 | .get_all_introduced_starchart_instances(0, 100) 132 | .await 133 | .unwrap(); 134 | assert!(instances 135 | .iter() 136 | .any(|i| i.instance_url == instance_url.as_str())); 137 | 138 | db.import_mini_index(instance_url, MINI_INDEX) 139 | .await 140 | .unwrap(); 141 | let matching_instances = db.search_mini_index("uniquerq2342").await.unwrap(); 142 | assert_eq!(matching_instances.len(), 1); 143 | assert_eq!(matching_instances.first().unwrap(), instance_url.as_str()); 144 | 145 | db.rm_starchart_import(instance_url).await.unwrap(); 146 | assert!(!db.is_starchart_imported(instance_url).await.unwrap()); 147 | db.record_starchart_imports(instance_url).await.unwrap(); 148 | assert!(db.is_starchart_imported(instance_url).await.unwrap()); 149 | db.rm_starchart_import(instance_url).await.unwrap(); 150 | assert!(!db.is_starchart_imported(instance_url).await.unwrap()); 151 | } 152 | 153 | /// test if all instance introducer methods work 154 | pub async fn mini_index_helper(db: &T) { 155 | // batman is repeated twice but mini-index should contain it only once 156 | // Batman is different from Batman; mini-index is case-sensitive 157 | const WORDS: [&str; 5] = ["batman", "superman", "aquaman", "Batman", "batman"]; 158 | 159 | let expected_mini_index = "superman aquaman Batman batman"; 160 | 161 | for w in WORDS.iter() { 162 | db.rm_word_from_mini_index(w).await.unwrap(); 163 | assert!(!db.is_word_mini_indexed(w).await.unwrap()); 164 | db.add_word_to_mini_index(w).await.unwrap(); 165 | assert!(db.is_word_mini_indexed(w).await.unwrap()); 166 | } 167 | 168 | let mini_index = db.export_mini_index().await.unwrap(); 169 | assert!(mini_index.contains(expected_mini_index)); 170 | } 171 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env 3 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "db-sqlx-sqlite" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/forgeflux-org/starchart" 6 | repository = "https://github.com/forgeflux-org/starchart" 7 | documentation = "https://github.con/forgeflux-org/starchart" 8 | readme = "https://github.com/forgeflux-org/starchart/blob/master/README.md" 9 | license = "AGPLv3 or later version" 10 | authors = ["realaravinth "] 11 | include = ["./mgrations/"] 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | 15 | [lib] 16 | name = "db_sqlx_sqlite" 17 | path = "src/lib.rs" 18 | 19 | [dependencies] 20 | sqlx = { version = "0.6.2", features = [ "sqlite", "time", "offline", "runtime-actix-rustls" ] } 21 | db-core = {path = "../db-core"} 22 | async-trait = "0.1.51" 23 | url = { version = "2.2.2", features = ["serde"] } 24 | 25 | [dev-dependencies] 26 | actix-rt = "2" 27 | sqlx = { version = "0.6.2", features = [ "runtime-actix-rustls", "postgres", "time", "offline" ] } 28 | db-core = {path = "../db-core", features = ["test"]} 29 | url = { version = "2.2.2", features = ["serde"] } 30 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20220405113942_world_forges.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS starchart_forge_type ( 2 | name VARCHAR(30) NOT NULL UNIQUE, 3 | ID INTEGER PRIMARY KEY NOT NULL 4 | ); 5 | 6 | INSERT OR IGNORE INTO starchart_forge_type (name) VALUES('gitea'); 7 | 8 | CREATE TABLE IF NOT EXISTS starchart_forges ( 9 | forge_type INTEGER NOT NULL REFERENCES starchart_forge_type(ID) ON DELETE CASCADE, 10 | hostname TEXT NOT NULL UNIQUE, 11 | verified_on INTEGER NOT NULL, 12 | last_crawl_on INTEGER DEFAULT NULL, 13 | ID INTEGER PRIMARY KEY NOT NULL 14 | ); 15 | 16 | CREATE TABLE IF NOT EXISTS starchart_dns_challenges ( 17 | hostname TEXT NOT NULL UNIQUE, 18 | key TEXT NOT NULL UNIQUE, 19 | value TEXT NOT NULL UNIQUE, 20 | created INTEGER NOT NULL, 21 | ID INTEGER PRIMARY KEY NOT NULL 22 | ); 23 | 24 | CREATE TABLE IF NOT EXISTS starchart_users ( 25 | hostname_id INTEGER NOT NULL REFERENCES starchart_forges(ID) ON DELETE CASCADE, 26 | username TEXT NOT NULL, 27 | html_url TEXT NOT NULL UNIQUE, 28 | profile_photo_html_url TEXT DEFAULT NULL, 29 | added_on INTEGER NOT NULL, 30 | last_crawl_on INTEGER NOT NULL, 31 | ID INTEGER PRIMARY KEY NOT NULL 32 | ); 33 | 34 | CREATE TABLE IF NOT EXISTS starchart_project_topics ( 35 | name VARCHAR(50) NOT NULL UNIQUE, 36 | ID INTEGER PRIMARY KEY NOT NULL 37 | ); 38 | 39 | CREATE TABLE IF NOT EXISTS starchart_repositories ( 40 | ID INTEGER PRIMARY KEY NOT NULL, 41 | hostname_id INTEGER NOT NULL REFERENCES starchart_forges(ID) ON DELETE CASCADE, 42 | owner_id INTEGER NOT NULL REFERENCES starchart_users(ID) ON DELETE CASCADE, 43 | name TEXT NOT NULL, 44 | description TEXT DEFAULT NULL, 45 | website TEXT DEFAULT NULL, 46 | html_url TEXT NOT NULL UNIQUE, 47 | created INTEGER NOT NULL, 48 | last_crawl INTEGER NOT NULL 49 | ); 50 | 51 | CREATE TABLE IF NOT EXISTS starchart_repository_topic_mapping ( 52 | repository_id INTEGER NOT NULL REFERENCES starchart_repositories(ID) ON DELETE CASCADE, 53 | topic_id INTEGER NOT NULL REFERENCES starchart_project_topics(ID) ON DELETE CASCADE 54 | ); 55 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230211075309_starchart_import.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE starchart_forges ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); 2 | 3 | ALTER TABLE starchart_users ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); 4 | 5 | ALTER TABLE starchart_repositories ADD COLUMN imported BOOLEAN NOT NULL DEFAULT(0); 6 | 7 | CREATE TABLE IF NOT EXISTS foo ( 8 | ID INTEGER PRIMARY KEY NOT NULL 9 | ); 10 | 11 | DROP TABLE foo; 12 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230223063159_starchart_fts_repository.sql: -------------------------------------------------------------------------------- 1 | CREATE VIRTUAL TABLE IF NOT EXISTS fts_repositories USING fts4( 2 | name TEXT NOT NULL, 3 | description TEXT DEFAULT NULL, 4 | website TEXT DEFAULT NULL, 5 | html_url TEXT NOT NULL UNIQUE, 6 | ); 7 | 8 | 9 | CREATE VIRTUAL TABLE IF NOT EXISTS fts_project_topics USING fts4( 10 | name VARCHAR(50) NOT NULL UNIQUE 11 | ); 12 | 13 | 14 | CREATE VIRTUAL TABLE IF NOT EXISTS fts_users USING fts4( 15 | username TEXT NOT NULL 16 | ); 17 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230228083200_starchart_introducer.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS starchart_introducer ( 2 | ID INTEGER PRIMARY KEY NOT NULL, 3 | instance_url TEXT NOT NULL UNIQUE 4 | ); 5 | 6 | 7 | ALTER TABLE starchart_forges ADD COLUMN starchart_instance INTEGER REFERENCES 8 | starchart_introducer(ID) ON DELETE CASCADE DEFAULT NULL; 9 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230302072543_starchart_mini_index.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS starchart_mini_index ( 2 | word TEXT NOT NULL UNIQUE, 3 | ID INTEGER PRIMARY KEY NOT NULL 4 | ); 5 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230302094417_starchart_federated_mini_index.sql: -------------------------------------------------------------------------------- 1 | CREATE VIRTUAL TABLE IF NOT EXISTS starchart_federated_mini_index USING fts4 ( 2 | starchart_instance INTEGER REFERENCES starchart_introducer(ID) ON DELETE CASCADE, 3 | mini_index TEXT NOT NULL, 4 | ID INTEGER PRIMARY KEY NOT NULL 5 | ); 6 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/migrations/20230302132435_starchart_imported_starcharts.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS starchart_imported_starcharts ( 2 | starchart_instance INTEGER REFERENCES starchart_introducer(ID) ON DELETE CASCADE, 3 | ID INTEGER PRIMARY KEY NOT NULL 4 | ); 5 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::borrow::Cow; 18 | 19 | use db_core::dev::*; 20 | use sqlx::Error; 21 | 22 | pub fn map_register_err(e: Error) -> DBError { 23 | if let Error::Database(err) = e { 24 | if err.code() == Some(Cow::from("2067")) { 25 | let msg = err.message(); 26 | println!("db err: {msg}"); 27 | if msg.contains("starchart_dns_challenges.hostname") { 28 | DBError::DuplicateChallengeHostname 29 | } else if msg.contains("starchart_forges.hostname") { 30 | DBError::DuplicateHostname 31 | } else if msg.contains("starchart_dns_challenges.challenge") { 32 | DBError::DuplicateChallengeText 33 | } else if msg.contains("starchart_users.html_url") { 34 | DBError::DuplicateUserLink 35 | } else if msg.contains("starchart_project_topics.name") { 36 | DBError::DuplicateTopic 37 | } else if msg.contains("starchart_repositories.html_url") { 38 | DBError::DuplicateRepositoryLink 39 | } else if msg.contains("starchart_forge_type.name") { 40 | DBError::DuplicateForgeType 41 | } else if msg.contains("starchart_users.html_url") { 42 | DBError::DuplicateUserLink 43 | } else if msg.contains("starchart_project_topics.name") { 44 | DBError::DuplicateTopic 45 | } else if msg.contains("starchart_repositories.name") { 46 | DBError::DuplicateRepositoryLink 47 | } else { 48 | DBError::DBError(Box::new(Error::Database(err)).into()) 49 | } 50 | } else { 51 | DBError::DBError(Box::new(Error::Database(err)).into()) 52 | } 53 | } else { 54 | DBError::DBError(Box::new(e).into()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /db/db-sqlx-sqlite/src/tests.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::env; 18 | 19 | use sqlx::sqlite::SqlitePoolOptions; 20 | use url::Url; 21 | 22 | use crate::*; 23 | 24 | use db_core::tests::*; 25 | 26 | #[actix_rt::test] 27 | async fn everything_works() { 28 | const URL: &str = "https://test-gitea.example.com"; 29 | const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1"; 30 | const HTML_PROFILE_PHOTO_URL_2: &str = "https://test-gitea.example.com/profile-photo/user2"; 31 | const USERNAME: &str = "user1"; 32 | const USERNAME2: &str = "user2"; 33 | 34 | const REPO_NAME: &str = "starchart"; 35 | const HTML_REPO_URL: &str = "https://test-gitea.example.com/user1/starchart"; 36 | const TAGS: [&str; 3] = ["test", "starchart", "spider"]; 37 | 38 | let url = Url::parse(URL).unwrap(); 39 | 40 | let create_forge_msg = CreateForge { 41 | url: url.clone(), 42 | forge_type: ForgeImplementation::Gitea, 43 | starchart_url: None, 44 | }; 45 | 46 | let add_user_msg = AddUser { 47 | url: url.clone(), 48 | html_link: HTML_PROFILE_URL, 49 | profile_photo: None, 50 | username: USERNAME, 51 | import: false, 52 | }; 53 | 54 | let add_user_msg_2 = AddUser { 55 | url: url.clone(), 56 | html_link: HTML_PROFILE_PHOTO_URL_2, 57 | profile_photo: Some(HTML_PROFILE_PHOTO_URL_2), 58 | username: USERNAME2, 59 | import: false, 60 | }; 61 | 62 | let db = { 63 | let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); 64 | let pool_options = SqlitePoolOptions::new().max_connections(2); 65 | let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); 66 | let db = connection_options.connect().await.unwrap(); 67 | db.migrate().await.unwrap(); 68 | db 69 | }; 70 | 71 | let add_repo_msg = AddRepository { 72 | html_link: HTML_REPO_URL, 73 | name: REPO_NAME, 74 | tags: Some(TAGS.into()), 75 | owner: USERNAME, 76 | website: "https://starcahrt-sqlite-test.example.org".into(), 77 | description: "starchart sqlite test repo sescription".into(), 78 | url, 79 | import: false, 80 | }; 81 | 82 | adding_forge_works( 83 | &db, 84 | create_forge_msg, 85 | add_user_msg, 86 | add_user_msg_2, 87 | add_repo_msg, 88 | ) 89 | .await; 90 | } 91 | 92 | #[actix_rt::test] 93 | async fn introducer_works() { 94 | let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); 95 | let pool_options = SqlitePoolOptions::new().max_connections(2); 96 | let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); 97 | let db = connection_options.connect().await.unwrap(); 98 | 99 | let instance_url = Url::parse("https://introducer_works_sqlite_sqlx.example.com").unwrap(); 100 | instance_introducer_helper(&db, &instance_url).await; 101 | } 102 | 103 | #[actix_rt::test] 104 | async fn forge_type_exists() { 105 | let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); 106 | let pool_options = SqlitePoolOptions::new().max_connections(2); 107 | let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); 108 | let db = connection_options.connect().await.unwrap(); 109 | 110 | db.migrate().await.unwrap(); 111 | forge_type_exists_helper(&db).await; 112 | } 113 | 114 | #[actix_rt::test] 115 | async fn mini_index_test() { 116 | let url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); 117 | let pool_options = SqlitePoolOptions::new().max_connections(2); 118 | let connection_options = ConnectionOptions::Fresh(Fresh { pool_options, url }); 119 | let db = connection_options.connect().await.unwrap(); 120 | 121 | db.migrate().await.unwrap(); 122 | mini_index_helper(&db).await; 123 | } 124 | -------------------------------------------------------------------------------- /db/migrator/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .env 3 | -------------------------------------------------------------------------------- /db/migrator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "migrator" 3 | version = "0.1.0" 4 | edition = "2021" 5 | homepage = "https://github.com/forgeflux-org/starchart" 6 | repository = "https://github.com/forgeflux-org/starchart" 7 | documentation = "https://github.con/forgeflux-org/starchart" 8 | readme = "https://github.com/forgeflux-org/starchart/blob/master/README.md" 9 | license = "AGPLv3 or later version" 10 | authors = ["Aravinth Manivannan "] 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [workspace] 15 | 16 | [dependencies] 17 | sqlx = { version = "0.5.11", features = [ "runtime-actix-rustls", "sqlite", "postgres", "time", "offline" ] } 18 | actix-rt = "2" 19 | -------------------------------------------------------------------------------- /db/migrator/src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::env; 19 | 20 | use sqlx::migrate::MigrateDatabase; 21 | //use sqlx::postgres::PgPoolOptions; 22 | use sqlx::sqlite::SqlitePoolOptions; 23 | use sqlx::Sqlite; 24 | 25 | #[cfg(not(tarpaulin_include))] 26 | #[actix_rt::main] 27 | async fn main() { 28 | //TODO featuregate sqlite and postgres 29 | // postgres_migrate().await; 30 | sqlite_migrate().await; 31 | } 32 | 33 | //async fn postgres_migrate() { 34 | // let db_url = env::var("POSTGRES_DATABASE_URL").expect("set POSTGRES_DATABASE_URL env var"); 35 | // let db = PgPoolOptions::new() 36 | // .max_connections(2) 37 | // .connect(&db_url) 38 | // .await 39 | // .expect("Unable to form database pool"); 40 | // 41 | // todo!("unimplemented"); 42 | // 43 | //// sqlx::migrate!("../db-sqlx-postgres/migrations/") 44 | //// .run(&db) 45 | //// .await 46 | //// .unwrap(); 47 | //} 48 | 49 | async fn sqlite_migrate() { 50 | let db_url = env::var("SQLITE_DATABASE_URL").expect("Set SQLITE_DATABASE_URL env var"); 51 | 52 | if !matches!(Sqlite::database_exists(&db_url).await, Ok(true)) { 53 | Sqlite::create_database(&db_url).await.unwrap(); 54 | } 55 | 56 | let db = SqlitePoolOptions::new() 57 | .max_connections(2) 58 | .connect(&db_url) 59 | .await 60 | .expect("Unable to form database pool"); 61 | 62 | sqlx::migrate!("../db-sqlx-sqlite/migrations/") 63 | .run(&db) 64 | .await 65 | .unwrap(); 66 | } 67 | -------------------------------------------------------------------------------- /docker-compose-dev-deps.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | networks: 4 | gitea: 5 | external: false 6 | 7 | services: 8 | server: 9 | image: gitea/gitea:1.16.5 10 | container_name: gitea 11 | environment: 12 | - USER_UID=1000 13 | - USER_GID=1000 14 | restart: always 15 | networks: 16 | - gitea 17 | volumes: 18 | - ./tmp/gitea:/data 19 | - /etc/timezone:/etc/timezone:ro 20 | - /etc/localtime:/etc/localtime:ro 21 | ports: 22 | - "8080:3000" 23 | - "2221:22" 24 | -------------------------------------------------------------------------------- /docs/ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | Starchart is designed with maximum flexibility in mind and so it is 4 | highly extensible. Support for new forges, federation formats and 5 | databases can be implemented with ease and this document intends to 6 | document how to do just that. 7 | 8 | 1. [`db-core`](../db/db-core): Contains traits(Rust-speak for 9 | interfaces) to implement support for new databases. Support for 10 | SQLite via [sqlx](https://crates.io/crates/sqlx) is implemented in 11 | [`db-sqlx-sqlite`](../db/db-sqlx-sqlite) 12 | 13 | 2. [`forge-core`](../forge/forge-core): Contains traits for implementing 14 | spidering support for a new forge type. Support for Gitea is 15 | implemented in [`gitea`](../forge/forge-core). 16 | 17 | 3. [`federation-core`](../federate/federate-core): Contains traits to 18 | implement support for new federation file formats. Support for 19 | [publiccodeyml](https://yml.publiccode.tools/) is implemented in 20 | [publiccodeyml](../federate/publiccodeyml). 21 | -------------------------------------------------------------------------------- /docs/HACKING.md: -------------------------------------------------------------------------------- 1 | # Hacking 2 | 3 | Instructions WIP. Kindly give feedback :) 4 | 5 | ## Development dependencies 6 | 7 | 1. [pkg-config](https://packages.debian.org/bullseye/pkg-config) 8 | 2. [GNU make](https://packages.debian.org/bullseye/make) 9 | 3. [libssl-dev](https://packages.debian.org/bullseye/libssl-dev) 10 | 4. Rust(see [installation instructions](#install-rust)) 11 | 5. Docker-compose 12 | 13 | ### Install Rust 14 | 15 | Install Rust using [rustup](https://rustup.rs/). 16 | 17 | `rustup` is the official Rust installation tool. It enables installation 18 | of multiple versions of `rustc` for different architectures across 19 | multiple release channels(stable, nightly, etc.). 20 | 21 | Rust undergoes [six-week release 22 | cycles](https://doc.rust-lang.org/book/appendix-05-editions.html#appendix-e---editions) 23 | and some of the dependencies that are used in this program have often 24 | relied on cutting edge features of the Rust compiler. OS Distribution 25 | packaging teams don't often track the latest releases. For this reason, 26 | we encourage managing your Rust installation with `rustup`. 27 | 28 | **`rustup` is the officially supported Rust installation method of this 29 | project.** 30 | 31 | ```bash 32 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh 33 | ``` 34 | 35 | ### Setting up the workspace 36 | After installing rust, the database schema needs to be migrated, 37 | we use `sqlx` in this project to handle this. However, before running 38 | this application you might end up finding yourself stuck, here are a 39 | few things that you might come across. 40 | 41 | #### Environment variables 42 | > thread 'main' panicked at 'called `Result::unwrap()` on an `Err` 43 | > value: missing field `url`' 44 | 45 | Please ensure that you have the `.env` stored in the root of the 46 | repository, you can copy this from the `.env-sample` present in 47 | the root of the repository. 48 | 49 | [temporary fix] There is also a need for the `DATABASE_URL` to 50 | be defined, so add that in too. 51 | 52 | Next up, run the following commands to have the project compile and run, 53 | ```bash 54 | source .env 55 | make migrate 56 | make 57 | ``` 58 | 59 | This should ideally get your instance of Starchart running, and if 60 | you face any issues at this point, it's a good idea to check your 61 | environment variables once more, and review the dependencies for 62 | the project. 63 | 64 | ## Implementing Support for $FORGE 65 | 66 | > In the future, Starchart will be modified to talk forge federation 67 | > ActivityPub protocol(general term, not referring to 68 | > [forgefed](https://forgefed.peers.community/)), so implementing support 69 | > for $FORGE would mean implementing that protocol for $FORGE. 70 | 71 | **TODO** 72 | 73 | ### Testing 74 | 75 | **2022-04-13:** Support for [Gitea](https://gitea.io) is WIP and because 76 | Gitea is Free Software and light-weight to run within CI/CD environment, 77 | we are able to run a Gitea instate and run tests against it. See 78 | [docker-compose-dev-deps.yml](../docker-compose-dev-deps.yml). 79 | 80 | ## Implementing Support for $DATABASE 81 | 82 | > Thank you for your interest in adding support for a new database. Please let us know about your effort 83 | > so that we can link to it on this repository :) 84 | 85 | Starchart defines all database operations in [`db-core`](../db/db-core]) 86 | local crate. Implementing `SCDatabase` from the same crate will add 87 | support for your database. 88 | 89 | ### Testing 90 | 91 | Tests are generic over all database support implementations, so tests 92 | are implemented as part of the core package at 93 | [db-core/tests.rs](../db/db-core/src/tests.rs) and re-exported for use 94 | within tests. 95 | 96 | Please see 97 | [SQLite tests implementation](../db/db-sqlx-sqlite/src/tests.rs) for 98 | inspiration. 99 | -------------------------------------------------------------------------------- /docs/published-crwaled-data.md: -------------------------------------------------------------------------------- 1 | # Published Crawled Data 2 | 3 | Starchart publishes all crawled data. This document explains the 4 | format(s) and the directory structure of the published data. 5 | 6 | ## Directory Structure 7 | 8 | ```bash 9 | (lab)➜ starchart tree data 10 | data 11 | └── git.batsense.net 12 | ├── instance.yml 13 | └── realaravinth 14 | ├── analysis-of-captcha-systems 15 | │   └── publiccode.yml 16 | └── user.yml 17 | ``` 18 | 19 | > Snippet of data crawled from git.batsense.net 20 | 21 | ## Forge 22 | 23 | Each forge instance gets its own directory in the repository root path 24 | specified in the [configuration](../config/default.toml). All data 25 | crawled from an instance will be stored in the instance's directory 26 | only. 27 | 28 | Each forge instance directory contains an `instance.yml` file that 29 | describes the instance. The schema of `instance.yml` might change as 30 | starchart is currently under development. 31 | 32 | ```yml 33 | --- 34 | hostname: git.batsense.net 35 | forge_type: gitea 36 | ``` 37 | 38 | > example instance.yml 39 | 40 | ## User 41 | 42 | A forge instance's user gets their own subdirectory in starchart and an 43 | `user.yml` to describe them. Information on all their repositories will be stored under 44 | this subdirectory. 45 | 46 | Like `instance.yml`, `user.yml` schema is not finalized too. 47 | 48 | ```yml 49 | --- 50 | hostname: git.batsense.net 51 | username: realaravinth 52 | html_link: "https://git.batsense.net/realaravinth" 53 | profile_photo: "https://git.batsense.net/avatars/bc11e95d9356ac4bdc035964be00ff0d" 54 | ``` 55 | 56 | > example user.yml 57 | 58 | ## Repository 59 | 60 | Repository information is stored under the owner's subdirectory. 61 | Currently, partial support for 62 | [publiccodeyml](https://yml.publiccode.tools/) is implemented. So all 63 | repository information is stored in `publiccode.yml` under the 64 | repository subdirectory. 65 | 66 | ```yml 67 | --- 68 | publiccodeYmlVersion: "0.2" 69 | name: git.batsense.net 70 | url: "https://git.batsense.net/realaravinth/git.batsense.net" 71 | description: 72 | en: 73 | shortDescription: "Instance administration logs and discussions pertaining to this Gitea instance. Have a question about git.batsense.net? Please create an issue on this repository! :)" 74 | ``` 75 | 76 | > example publiccode.yml implemented by starchart 77 | 78 | See 79 | [forgeflux-org/starchart#3](https://github.com/forgeflux-org/starchart/issues/3) and 80 | [publiccodeyml/publiccodeyml/discussions](https://github.com/publiccodeyml/publiccode.yml/discussions/157) for more information. 81 | -------------------------------------------------------------------------------- /federate/federate-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "federate-core" 3 | version = "0.1.0" 4 | authors = ["realaravinth "] 5 | description = "ForgeFlux StarChart - Federated forge spider" 6 | documentation = "https://forgeflux.org/" 7 | edition = "2021" 8 | license = "AGPLv3 or later version" 9 | 10 | [lib] 11 | name = "federate_core" 12 | path = "src/lib.rs" 13 | 14 | 15 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 16 | 17 | [dependencies] 18 | async-trait = "0.1.51" 19 | thiserror = "1.0.30" 20 | serde = { version = "1", features = ["derive"]} 21 | url = { version = "2.2.2", features = ["serde"] } 22 | api_routes = { path = "../../api_routes/" } 23 | 24 | [dependencies.reqwest] 25 | version = "0.11.10" 26 | 27 | [dependencies.db-core] 28 | path = "../../db/db-core" 29 | 30 | [features] 31 | default = [] 32 | test = [] 33 | -------------------------------------------------------------------------------- /federate/federate-core/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | -------------------------------------------------------------------------------- /federate/federate-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::path::Path; 19 | use std::path::PathBuf; 20 | use std::result::Result; 21 | 22 | use async_trait::async_trait; 23 | use reqwest::Client; 24 | use url::Url; 25 | 26 | use db_core::prelude::*; 27 | 28 | #[cfg(feature = "test")] 29 | pub mod tests; 30 | 31 | pub use api_routes::*; 32 | 33 | #[async_trait] 34 | pub trait Federate: Sync + Send { 35 | type Error: std::error::Error + std::fmt::Debug; 36 | 37 | /// utility method to create dir if not exists 38 | async fn create_dir_if_not_exists(&self, path: &Path) -> Result<(), Self::Error>; 39 | 40 | /// utility method to remove file/dir 41 | async fn rm_util(&self, path: &Path) -> Result<(), Self::Error>; 42 | 43 | /// create forge instance 44 | async fn create_forge_instance(&self, f: &CreateForge) -> Result<(), Self::Error>; 45 | 46 | /// delete forge instance 47 | async fn delete_forge_instance(&self, url: &Url) -> Result<(), Self::Error>; 48 | 49 | /// check if a forge instance exists 50 | async fn forge_exists(&self, url: &Url) -> Result; 51 | 52 | /// check if an user exists. 53 | async fn user_exists(&self, username: &str, url: &Url) -> Result; 54 | 55 | /// create user instance 56 | async fn create_user(&self, f: &AddUser<'_>) -> Result<(), Self::Error>; 57 | 58 | /// add repository instance 59 | async fn create_repository(&self, f: &AddRepository<'_>) -> Result<(), Self::Error>; 60 | 61 | /// check if a repository exists. 62 | async fn repository_exists( 63 | &self, 64 | name: &str, 65 | owner: &str, 66 | url: &Url, 67 | ) -> Result; 68 | 69 | /// delete user 70 | async fn delete_user(&self, username: &str, url: &Url) -> Result<(), Self::Error>; 71 | 72 | /// delete repository 73 | async fn delete_repository( 74 | &self, 75 | owner: &str, 76 | name: &str, 77 | url: &Url, 78 | ) -> Result<(), Self::Error>; 79 | 80 | /// publish results in tar ball 81 | async fn tar(&self) -> Result; 82 | 83 | /// get latest tar ball 84 | async fn latest_tar(&self) -> Result; 85 | 86 | /// import archive from another Starchart instance 87 | async fn import( 88 | &self, 89 | mut starchart_url: Url, 90 | client: &Client, 91 | db: &Box, 92 | ) -> Result<(), Self::Error>; 93 | 94 | async fn latest_tar_json(&self) -> Result { 95 | let latest = self.latest_tar().await?; 96 | Ok(LatestResp { latest }) 97 | } 98 | } 99 | 100 | pub fn get_hostname(url: &Url) -> &str { 101 | url.host_str().unwrap() 102 | } 103 | -------------------------------------------------------------------------------- /federate/federate-core/src/tests.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | //! Test utilities 19 | use crate::*; 20 | 21 | /// adding forge works 22 | pub async fn adding_forge_works<'a, T: Federate>( 23 | ff: &T, 24 | create_forge_msg: CreateForge<'a>, 25 | create_user_msg: AddUser<'a>, 26 | add_repo_msg: AddRepository<'a>, 27 | ) { 28 | let _ = ff.delete_forge_instance(&create_forge_msg.url).await; 29 | assert!(!ff.forge_exists(&create_forge_msg.url).await.unwrap()); 30 | ff.create_forge_instance(&create_forge_msg).await.unwrap(); 31 | assert!(ff.forge_exists(&create_forge_msg.url).await.unwrap()); 32 | 33 | // add user 34 | assert!(!ff 35 | .user_exists(create_user_msg.username, &create_user_msg.url) 36 | .await 37 | .unwrap()); 38 | ff.create_user(&create_user_msg).await.unwrap(); 39 | assert!(ff 40 | .user_exists(create_user_msg.username, &create_user_msg.url) 41 | .await 42 | .unwrap()); 43 | 44 | // add repository 45 | assert!(!ff 46 | .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url) 47 | .await 48 | .unwrap()); 49 | ff.create_repository(&add_repo_msg).await.unwrap(); 50 | assert!(ff 51 | .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url) 52 | .await 53 | .unwrap()); 54 | 55 | // tar() 56 | let tar = ff.tar().await.unwrap().to_str().unwrap().to_string(); 57 | let latest = ff.latest_tar().await.unwrap(); 58 | assert!(tar.contains(&latest)); 59 | 60 | // delete repository 61 | ff.delete_repository(add_repo_msg.owner, add_repo_msg.name, &add_repo_msg.url) 62 | .await 63 | .unwrap(); 64 | 65 | // delete user 66 | ff.delete_user(create_user_msg.username, &create_user_msg.url) 67 | .await 68 | .unwrap(); 69 | 70 | // delete user 71 | ff.delete_forge_instance(&create_forge_msg.url) 72 | .await 73 | .unwrap(); 74 | } 75 | -------------------------------------------------------------------------------- /federate/publiccodeyml/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /federate/publiccodeyml/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "publiccodeyml" 3 | version = "0.1.0" 4 | authors = ["realaravinth "] 5 | description = "ForgeFlux StarChart - Federated forge spider" 6 | documentation = "https://forgeflux.org/" 7 | edition = "2021" 8 | license = "AGPLv3 or later version" 9 | 10 | [lib] 11 | name = "publiccodeyml" 12 | path = "src/lib.rs" 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | async-trait = "0.1.51" 18 | serde = { version = "1", features = ["derive"]} 19 | serde_yaml = "0.9" 20 | tokio = { version = "1.18.2", features = ["fs"]} 21 | thiserror = "1.0.30" 22 | url = { version = "2.2.2", features = ["serde"] } 23 | tar = "0.4.38" 24 | log = "0.4.16" 25 | mktemp = "0.4.1" 26 | 27 | [dependencies.reqwest] 28 | features = ["rustls-tls-native-roots", "gzip", "deflate", "brotli", "json"] 29 | version = "0.11.10" 30 | 31 | 32 | [dependencies.db-core] 33 | path = "../../db/db-core" 34 | 35 | [dependencies.federate-core] 36 | path = "../federate-core" 37 | 38 | [dev-dependencies] 39 | actix-rt = "2" 40 | mktemp = "0.4.1" 41 | federate-core = { path = "../federate-core", features = ["test"] } 42 | -------------------------------------------------------------------------------- /federate/publiccodeyml/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | //! represents all the ways a trait can fail using this crate 19 | use std::error::Error as StdError; 20 | 21 | use serde_yaml::Error as YamlError; 22 | use thiserror::Error; 23 | use tokio::io::Error as IOError; 24 | 25 | use db_core::errors::DBError; 26 | 27 | /// Error data structure grouping various error subtypes 28 | #[derive(Debug, Error)] 29 | pub enum FederateErorr { 30 | /// serialization error 31 | #[error("Serialization error: {0}")] 32 | SerializationError(YamlError), 33 | /// database errors 34 | #[error("{0}")] 35 | DBError(DBError), 36 | 37 | /// IO Error 38 | #[error("{0}")] 39 | IOError(IOError), 40 | } 41 | 42 | impl From for FederateErorr { 43 | fn from(e: DBError) -> Self { 44 | Self::DBError(e) 45 | } 46 | } 47 | 48 | impl From for FederateErorr { 49 | fn from(e: IOError) -> Self { 50 | Self::IOError(e) 51 | } 52 | } 53 | 54 | impl From for FederateErorr { 55 | fn from(e: YamlError) -> Self { 56 | Self::SerializationError(e) 57 | } 58 | } 59 | 60 | /// Convenience type alias for grouping driver-specific errors 61 | pub type BoxDynError = Box; 62 | 63 | /// Generic result data structure 64 | pub type FResult = std::result::Result; 65 | -------------------------------------------------------------------------------- /federate/publiccodeyml/src/schema.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::collections::HashMap; 19 | 20 | use db_core::AddRepository; 21 | use serde::{Deserialize, Serialize}; 22 | use url::Url; 23 | 24 | const PUBLIC_CODE_VERSION: &str = "0.2"; 25 | 26 | #[derive(Serialize, Deserialize)] 27 | #[serde(rename_all = "camelCase")] 28 | pub struct Repository { 29 | pub publiccode_yml_version: String, 30 | pub name: String, 31 | pub url: Url, 32 | #[serde(skip_serializing_if = "Option::is_none")] 33 | pub landing_url: Option, 34 | #[serde(skip_serializing_if = "Option::is_none")] 35 | pub is_based_on: Option, 36 | #[serde(skip_serializing_if = "HashMap::is_empty")] 37 | pub description: HashMap, 38 | pub legal: Legal, 39 | #[serde(skip_serializing_if = "IntendedAudience::is_none")] 40 | pub intended_audience: IntendedAudience, 41 | } 42 | 43 | #[derive(Serialize, Deserialize)] 44 | #[serde(rename_all = "camelCase")] 45 | pub struct Description { 46 | #[serde(skip_serializing_if = "Option::is_none")] 47 | pub short_description: Option, 48 | #[serde(skip_serializing_if = "Option::is_none")] 49 | pub long_description: Option, 50 | #[serde(skip_serializing_if = "Option::is_none")] 51 | pub documentation: Option, 52 | } 53 | 54 | #[derive(Serialize, Deserialize)] 55 | #[serde(rename_all = "camelCase")] 56 | pub struct Legal { 57 | #[serde(skip_serializing_if = "Option::is_none")] 58 | pub license: Option, 59 | pub repo_owner: String, 60 | } 61 | 62 | #[derive(Serialize, Deserialize)] 63 | #[serde(rename_all = "camelCase")] 64 | pub struct Maintenance { 65 | #[serde( 66 | skip_serializing_if = "Option::is_none", 67 | rename(serialize = "type", deserialize = "m_type") 68 | )] 69 | pub m_type: Option, 70 | pub contacts: Vec, 71 | } 72 | 73 | #[derive(Serialize, Deserialize)] 74 | #[serde(rename_all = "camelCase")] 75 | pub struct Contacts { 76 | pub name: String, 77 | } 78 | 79 | #[derive(Serialize, Deserialize)] 80 | #[serde(rename_all = "camelCase")] 81 | pub struct IntendedAudience { 82 | #[serde( 83 | skip_serializing_if = "Option::is_none", 84 | rename(serialize = "type", deserialize = "m_type") 85 | )] 86 | pub scope: Option>, 87 | } 88 | 89 | impl IntendedAudience { 90 | /// global is_none, to skip_serializing_if 91 | pub fn is_none(&self) -> bool { 92 | if self.scope.is_none() { 93 | true 94 | } else { 95 | self.scope.as_ref().unwrap().is_empty() 96 | } 97 | } 98 | } 99 | 100 | impl From<&db_core::AddRepository<'_>> for Repository { 101 | fn from(r: &db_core::AddRepository<'_>) -> Self { 102 | let mut description = HashMap::with_capacity(1); 103 | description.insert( 104 | "en".into(), 105 | Description { 106 | short_description: r.description.map(|d| d.into()), 107 | documentation: r.website.map(|d| d.into()), 108 | long_description: None, 109 | }, 110 | ); 111 | 112 | let legal = Legal { 113 | license: None, 114 | repo_owner: r.owner.to_string(), 115 | }; 116 | 117 | let scope = r 118 | .tags 119 | .as_ref() 120 | .map(|tags| tags.iter().map(|t| t.to_string()).collect()); 121 | let intended_audience = IntendedAudience { scope }; 122 | 123 | Self { 124 | publiccode_yml_version: PUBLIC_CODE_VERSION.into(), 125 | url: Url::parse(r.html_link).unwrap(), 126 | landing_url: r.website.map(|s| Url::parse(s).unwrap()), 127 | name: r.name.into(), 128 | is_based_on: None, // TODO collect is_fork information in forge/* 129 | description, 130 | legal, 131 | intended_audience, 132 | } 133 | } 134 | } 135 | 136 | impl Repository { 137 | pub fn to_add_repository(&self, import: bool) -> AddRepository { 138 | let tags = self 139 | .intended_audience 140 | .scope 141 | .as_ref() 142 | .map(|s| s.iter().map(|t| t.as_str()).collect()); 143 | let description = self 144 | .description 145 | .get("en") 146 | .as_ref() 147 | .unwrap() 148 | .short_description 149 | .as_deref(); 150 | let website = self.description.get("en").unwrap().documentation.as_deref(); 151 | AddRepository { 152 | html_link: self.url.as_str(), 153 | tags, 154 | url: self.url.clone(), 155 | name: &self.name, 156 | owner: &self.legal.repo_owner, 157 | description, 158 | website, 159 | import, 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /federate/publiccodeyml/src/tests.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use mktemp::Temp; 18 | use url::Url; 19 | 20 | use crate::*; 21 | use federate_core::tests; 22 | 23 | #[actix_rt::test] 24 | async fn everything_works() { 25 | const URL: &str = "https://test-gitea.example.com"; 26 | const HTML_PROFILE_URL: &str = "https://test-gitea.example.com/user1"; 27 | const USERNAME: &str = "user1"; 28 | 29 | const REPO_NAME: &str = "starchart"; 30 | const HTML_REPO_URL: &str = "https://test-gitea.example.com/user1/starchart"; 31 | const TAGS: [&str; 3] = ["test", "starchart", "spider"]; 32 | 33 | let tmp_dir = Temp::new_dir().unwrap(); 34 | 35 | let url = Url::parse(URL).unwrap(); 36 | 37 | let create_forge_msg = CreateForge { 38 | url: url.clone(), 39 | forge_type: ForgeImplementation::Gitea, 40 | starchart_url: None, 41 | }; 42 | 43 | let add_user_msg = AddUser { 44 | url: url.clone(), 45 | html_link: HTML_PROFILE_URL, 46 | profile_photo: None, 47 | username: USERNAME, 48 | import: false, 49 | }; 50 | 51 | let add_repo_msg = AddRepository { 52 | html_link: HTML_REPO_URL, 53 | name: REPO_NAME, 54 | tags: Some(TAGS.into()), 55 | owner: USERNAME, 56 | website: None, 57 | description: None, 58 | url: url.clone(), 59 | import: false, 60 | }; 61 | 62 | let pcc = PccFederate::new(tmp_dir.to_str().unwrap().to_string()) 63 | .await 64 | .unwrap(); 65 | tests::adding_forge_works(&pcc, create_forge_msg, add_user_msg, add_repo_msg).await; 66 | } 67 | -------------------------------------------------------------------------------- /forge/forge-core/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "forge-core" 3 | repository = "https://github.com/forgeflux-org/starchart" 4 | version = "0.1.0" 5 | authors = ["realaravinth "] 6 | description = "ForgeFlux StarChart - Federated forge spider" 7 | documentation = "https://forgeflux.org/" 8 | edition = "2021" 9 | license = "AGPLv3 or later version" 10 | 11 | [lib] 12 | name = "forge_core" 13 | path = "src/lib.rs" 14 | 15 | 16 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 17 | 18 | [dependencies] 19 | async-trait = "0.1.51" 20 | url = { version = "2.2.2", features = ["serde"] } 21 | 22 | [dependencies.db-core] 23 | path = "../../db/db-core" 24 | -------------------------------------------------------------------------------- /forge/forge-core/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::collections::{HashMap, HashSet}; 19 | use std::sync::Arc; 20 | 21 | use async_trait::async_trait; 22 | use db_core::prelude::*; 23 | use url::Url; 24 | 25 | pub mod prelude { 26 | pub use super::*; 27 | pub use async_trait::async_trait; 28 | } 29 | 30 | pub mod dev { 31 | pub use super::*; 32 | pub use async_trait::async_trait; 33 | pub use db_core; 34 | } 35 | 36 | #[derive(Clone, Debug)] 37 | pub struct User { 38 | /// url of the forge instance: with scheme but remove trailing slash 39 | /// url can be derived from html_link also, but used to link to user's forge instance 40 | pub url: Url, 41 | /// username of the user 42 | pub username: Arc, 43 | /// html link to the user profile 44 | pub html_link: String, 45 | /// OPTIONAL: html link to the user's profile photo 46 | pub profile_photo: Option, 47 | } 48 | 49 | impl<'a> From<&'a User> for AddUser<'a> { 50 | fn from(u: &'a User) -> Self { 51 | Self { 52 | url: u.url.clone(), 53 | username: u.username.as_str(), 54 | html_link: &u.html_link, 55 | profile_photo: u.profile_photo.as_deref(), 56 | import: false, 57 | } 58 | } 59 | } 60 | 61 | #[derive(Clone, Debug)] 62 | /// add new repository to database 63 | pub struct Repository { 64 | /// html link to the repository 65 | pub html_link: String, 66 | /// repository topic tags 67 | pub tags: Option>>, 68 | /// url of the forge instance: with scheme but remove trailing slash 69 | /// url can be deras_ref().map(|p| p.as_str()),ived from html_link also, but used to link to user's forge instance 70 | pub url: Url, 71 | /// repository name 72 | pub name: String, 73 | /// repository owner 74 | pub owner: Arc, 75 | /// repository description, if any 76 | pub description: Option, 77 | /// repository website, if any 78 | pub website: Option, 79 | } 80 | 81 | impl<'a> From<&'a Repository> for AddRepository<'a> { 82 | fn from(r: &'a Repository) -> Self { 83 | let tags = if let Some(rtags) = &r.tags { 84 | let mut tags = Vec::with_capacity(rtags.len()); 85 | for t in rtags.iter() { 86 | tags.push(t.as_str()); 87 | } 88 | Some(tags) 89 | } else { 90 | None 91 | }; 92 | Self { 93 | url: r.url.clone(), 94 | name: &r.name, 95 | description: r.description.as_deref(), 96 | owner: r.owner.username.as_str(), 97 | tags, 98 | html_link: &r.html_link, 99 | website: r.website.as_deref(), 100 | import: false, 101 | } 102 | } 103 | } 104 | 105 | pub type UserMap = HashMap, Arc>; 106 | pub type Tags = HashSet>; 107 | pub type Repositories = Vec; 108 | 109 | pub struct CrawlResp { 110 | pub repos: Repositories, 111 | pub tags: Tags, 112 | pub users: UserMap, 113 | } 114 | 115 | #[async_trait] 116 | pub trait SCForge: std::marker::Send + std::marker::Sync + CloneSPForge { 117 | async fn is_forge(&self) -> bool; 118 | async fn crawl(&self, limit: u64, page: u64, rate_limit: u64) -> CrawlResp; 119 | fn get_url(&self) -> &Url; 120 | fn forge_type(&self) -> ForgeImplementation; 121 | } 122 | 123 | /// Trait to clone SCForge 124 | pub trait CloneSPForge { 125 | /// clone Forge 126 | fn clone_forge(&self) -> Box; 127 | } 128 | 129 | impl CloneSPForge for T 130 | where 131 | T: SCForge + Clone + 'static, 132 | { 133 | fn clone_forge(&self) -> Box { 134 | Box::new(self.clone()) 135 | } 136 | } 137 | 138 | impl Clone for Box { 139 | fn clone(&self) -> Self { 140 | (**self).clone_forge() 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /forge/gitea/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "gitea" 3 | version = "0.1.0" 4 | authors = ["realaravinth "] 5 | description = "ForgeFlux StarChart - Federated forge spider" 6 | documentation = "https://forgeflux.org/" 7 | edition = "2021" 8 | license = "AGPLv3 or later version" 9 | 10 | 11 | [lib] 12 | name = "gitea" 13 | path = "src/lib.rs" 14 | 15 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 16 | 17 | [dependencies] 18 | async-trait = "0.1.51" 19 | url = { version = "2.2.2", features = ["serde"] } 20 | tokio = { version = "1.17", features = ["time"] } 21 | 22 | [dependencies.forge-core] 23 | path = "../forge-core" 24 | 25 | [dependencies.reqwest] 26 | features = ["rustls-tls-native-roots", "gzip", "deflate", "brotli", "json"] 27 | version = "0.11.10" 28 | 29 | [dependencies.serde] 30 | features = ["derive"] 31 | version = "1" 32 | 33 | [dependencies.serde_json] 34 | version = "1" 35 | 36 | [dev-dependencies] 37 | actix-rt = "2.7" 38 | -------------------------------------------------------------------------------- /forge/gitea/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::sync::Arc; 19 | use std::time::Duration; 20 | 21 | use reqwest::Client; 22 | use tokio::task::JoinHandle; 23 | use url::Url; 24 | 25 | use db_core::ForgeImplementation; 26 | use forge_core::dev::*; 27 | use forge_core::Repository; 28 | 29 | pub mod schema; 30 | 31 | const REPO_SEARCH_PATH: &str = "/api/v1/repos/search"; 32 | const GITEA_NODEINFO: &str = "/api/v1/nodeinfo"; 33 | const GITEA_IDENTIFIER: &str = "gitea"; 34 | 35 | #[derive(Clone)] 36 | pub struct Gitea { 37 | pub instance_url: Url, 38 | pub client: Client, 39 | url: Url, 40 | } 41 | 42 | impl Gitea { 43 | pub fn new(instance_url: Url, client: Client) -> Self { 44 | let url = Url::parse(&db_core::clean_url(&instance_url)).unwrap(); 45 | 46 | Self { 47 | instance_url, 48 | client, 49 | url, 50 | } 51 | } 52 | } 53 | 54 | impl PartialEq for Gitea { 55 | fn eq(&self, other: &Self) -> bool { 56 | self.url == other.url && self.instance_url == other.instance_url 57 | } 58 | } 59 | 60 | #[async_trait] 61 | impl SCForge for Gitea { 62 | async fn is_forge(&self) -> bool { 63 | true 64 | } 65 | 66 | fn get_url(&self) -> &Url { 67 | &self.url 68 | } 69 | 70 | fn forge_type(&self) -> ForgeImplementation { 71 | ForgeImplementation::Gitea 72 | } 73 | 74 | async fn crawl(&self, limit: u64, page: u64, rate_limit: u64) -> CrawlResp { 75 | fn empty_is_none(s: &str) -> Option { 76 | let s = s.trim(); 77 | if s.is_empty() { 78 | None 79 | } else { 80 | Some(s.to_owned()) 81 | } 82 | } 83 | 84 | let mut tags = Tags::default(); 85 | let mut users = UserMap::default(); 86 | let mut repos = Repositories::default(); 87 | 88 | let instance_url = self.instance_url.clone(); 89 | 90 | let mut url = instance_url.clone(); 91 | url.set_path(REPO_SEARCH_PATH); 92 | url.set_query(Some(&format!("page={page}&limit={limit}"))); 93 | let mut res: schema::SearchResults = self 94 | .client 95 | .get(url) 96 | .send() 97 | .await 98 | .unwrap() 99 | .json() 100 | .await 101 | .unwrap(); 102 | 103 | fn to_user(u: schema::User, g: &Gitea) -> Arc { 104 | let mut profile_url = g.instance_url.clone(); 105 | profile_url.set_path(&u.username); 106 | let username = Arc::new(u.username); 107 | Arc::new(forge_core::User { 108 | username, 109 | html_link: profile_url.to_string(), 110 | profile_photo: Some(u.avatar_url), 111 | url: g.url.clone(), 112 | }) 113 | } 114 | 115 | let mut sleep_fut: Option> = None; 116 | 117 | for repo in res.data.drain(0..) { 118 | let user = if !users.contains_key(&repo.owner.username) { 119 | let u = to_user(repo.owner, self); 120 | let username = u.username.clone(); 121 | users.insert(username.clone().clone(), u.clone()); 122 | u 123 | } else { 124 | users.get(&repo.owner.username).unwrap().clone() 125 | }; 126 | 127 | let mut url = instance_url.clone(); 128 | url.set_path(&format!( 129 | "/api/v1/repos/{}/{}/topics", 130 | &user.username, repo.name 131 | )); 132 | 133 | if let Some(sleep_fut) = sleep_fut { 134 | sleep_fut.await.unwrap(); 135 | } 136 | 137 | let mut topics: schema::Topics = self 138 | .client 139 | .get(url) 140 | .send() 141 | .await 142 | .unwrap() 143 | .json() 144 | .await 145 | .unwrap(); 146 | sleep_fut = Some(tokio::spawn(tokio::time::sleep(Duration::new( 147 | rate_limit, 0, 148 | )))); 149 | 150 | let mut rtopics = Vec::with_capacity(topics.topics.len()); 151 | for t in topics.topics.drain(0..) { 152 | let t = Arc::new(t); 153 | if !tags.contains(&t) { 154 | tags.insert(t.clone()); 155 | } 156 | rtopics.push(t); 157 | } 158 | 159 | let frepo = Repository { 160 | url: self.url.clone(), 161 | website: empty_is_none(&repo.website), 162 | name: repo.name, 163 | owner: user, 164 | html_link: repo.html_url, 165 | tags: Some(rtopics), 166 | description: Some(repo.description), 167 | }; 168 | 169 | repos.push(frepo); 170 | } 171 | CrawlResp { repos, tags, users } 172 | } 173 | } 174 | 175 | #[cfg(test)] 176 | mod tests { 177 | use super::*; 178 | use url::Url; 179 | 180 | pub const GITEA_HOST: &str = "http://localhost:8080"; 181 | pub const NET_REPOSITORIES: u64 = 100; 182 | pub const PER_CRAWL: u64 = 10; 183 | 184 | #[actix_rt::test] 185 | async fn gitea_works() { 186 | let ctx = Gitea::new(Url::parse(GITEA_HOST).unwrap(), Client::new()); 187 | assert!(ctx.is_forge().await); 188 | let steps = NET_REPOSITORIES / PER_CRAWL; 189 | 190 | for i in 0..steps { 191 | let res = ctx.crawl(PER_CRAWL, i, 0).await; 192 | assert_eq!(res.repos.len() as u64, PER_CRAWL); 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /forge/gitea/src/schema.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2usize22 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::collections::HashMap; 19 | 20 | use serde::{Deserialize, Serialize}; 21 | 22 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 23 | pub struct SearchResults { 24 | pub ok: bool, 25 | pub data: Vec, 26 | } 27 | 28 | #[derive(Debug, Clone, PartialEq, Hash, Eq, Serialize, Deserialize)] 29 | pub struct User { 30 | pub id: usize, 31 | pub login: String, 32 | pub full_name: String, 33 | pub email: String, 34 | pub avatar_url: String, 35 | pub language: String, 36 | pub is_admin: bool, 37 | pub last_login: String, 38 | pub created: String, 39 | pub restricted: bool, 40 | pub active: bool, 41 | pub prohibit_login: bool, 42 | pub location: String, 43 | pub website: String, 44 | pub description: String, 45 | pub visibility: String, 46 | pub followers_count: usize, 47 | pub following_count: usize, 48 | pub starred_repos_count: usize, 49 | pub username: String, 50 | } 51 | 52 | #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] 53 | pub struct Repository { 54 | pub name: String, 55 | pub full_name: String, 56 | pub description: String, 57 | pub empty: bool, 58 | pub private: bool, 59 | pub fork: bool, 60 | pub template: bool, 61 | pub parent: Option>, 62 | pub mirror: bool, 63 | pub size: usize, 64 | pub html_url: String, 65 | pub ssh_url: String, 66 | pub clone_url: String, 67 | pub original_url: String, 68 | pub owner: User, 69 | pub website: String, 70 | pub stars_count: usize, 71 | pub forks_count: usize, 72 | pub watchers_count: usize, 73 | pub open_issues_count: usize, 74 | pub open_pr_counter: usize, 75 | pub release_counter: usize, 76 | pub default_branch: String, 77 | pub archived: bool, 78 | pub created_at: String, 79 | pub updated_at: String, 80 | pub internal_tracker: InternalIssueTracker, 81 | pub has_issues: bool, 82 | pub has_wiki: bool, 83 | pub has_pull_requests: bool, 84 | pub has_projects: bool, 85 | pub ignore_whitespace_conflicts: bool, 86 | pub allow_merge_commits: bool, 87 | pub allow_rebase: bool, 88 | pub allow_rebase_explicit: bool, 89 | pub allow_squash_merge: bool, 90 | pub default_merge_style: String, 91 | pub avatar_url: String, 92 | pub internal: bool, 93 | pub mirror_interval: String, 94 | pub mirror_updated: String, 95 | pub repo_transfer: Option, 96 | } 97 | 98 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 99 | pub struct InternalIssueTracker { 100 | pub enable_time_tracker: bool, 101 | pub allow_only_contributors_to_track_time: bool, 102 | pub enable_issue_dependencies: bool, 103 | } 104 | 105 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 106 | pub struct RepoTransfer { 107 | pub doer: User, 108 | pub recipient: User, 109 | pub teams: Option, 110 | } 111 | 112 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Hash, Deserialize)] 113 | pub struct Organization { 114 | pub avatar_url: String, 115 | pub description: String, 116 | pub full_name: String, 117 | pub id: u64, 118 | pub location: String, 119 | pub repo_admin_change_team_access: bool, 120 | pub username: String, 121 | pub visibility: String, 122 | pub website: String, 123 | } 124 | 125 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Hash, Deserialize)] 126 | #[serde(rename_all = "lowercase")] 127 | pub enum Permission { 128 | None, 129 | Read, 130 | Write, 131 | Admin, 132 | Owner, 133 | } 134 | 135 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 136 | pub struct Team { 137 | pub can_create_org_repo: bool, 138 | pub description: String, 139 | pub id: u64, 140 | pub includes_all_repositories: bool, 141 | pub name: String, 142 | pub organization: Organization, 143 | pub permission: Permission, 144 | pub units: Vec, 145 | pub units_map: HashMap, 146 | } 147 | 148 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 149 | pub struct Topics { 150 | pub topics: Vec, 151 | } 152 | 153 | #[cfg(test)] 154 | mod tests { 155 | use super::*; 156 | 157 | use std::fs; 158 | 159 | #[test] 160 | /// Tests if Gitea responses panic when deserialized with serde into structs defined in this 161 | /// module/file. Since Go doesn't have abilities to describe nullable values, I(@realaravinth) 162 | /// am forced to do this as I my knowledge about Gitea codebase is very limited. 163 | fn schema_doesnt_panic() { 164 | let files = ["./tests/schema/gitea/git.batsense.net.json"]; 165 | for file in files.iter() { 166 | let contents = fs::read_to_string(file).unwrap(); 167 | for line in contents.lines() { 168 | let _: SearchResults = serde_json::from_str(line).expect("Gitea schema paniced"); 169 | } 170 | } 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | readonly username=starchart 4 | USER_ID=${LOCAL_USER_ID} 5 | echo "[*] Local user ID: $USER_ID" 6 | echo "[*] Starting with UID : $USER_ID" 7 | export HOME=/home/$username 8 | #adduser --disabled-password --shell /bin/bash --home $HOME --uid $USER_ID user 9 | #--uid 10 | useradd --uid $USER_ID -b /home -m -s /bin/bash $username 11 | su - $username 12 | starchart 13 | -------------------------------------------------------------------------------- /scripts/gitea.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlunparse, urlparse 2 | from html.parser import HTMLParser 3 | from time import sleep 4 | import random 5 | 6 | from requests import Session 7 | from requests.auth import HTTPBasicAuth 8 | import requests 9 | 10 | GITEA_USER = "bot" 11 | GITEA_EMAIL = "bot@example.com" 12 | GITEA_PASSWORD = "foobarpassword" 13 | 14 | TOTAL_NUM_REPOS = 100 15 | REPOS = [] 16 | 17 | 18 | def check_online(): 19 | count = 0 20 | while True: 21 | try: 22 | res = requests.get( 23 | "http://localhost:8080/api/v1/nodeinfo", allow_redirects=False 24 | ) 25 | if any([res.status_code == 302, res.status_code == 200]): 26 | break 27 | except: 28 | sleep(2) 29 | print(f"Retrying {count} time") 30 | count += 1 31 | continue 32 | 33 | 34 | def install(): 35 | INSTALL_PAYLOAD = { 36 | "db_type": "sqlite3", 37 | "db_host": "localhost:3306", 38 | "db_user": "root", 39 | "db_passwd": "", 40 | "db_name": "gitea", 41 | "ssl_mode": "disable", 42 | "db_schema": "", 43 | "charset": "utf8", 44 | "db_path": "/data/gitea/gitea.db", 45 | "app_name": "Gitea:+Git+with+a+cup+of+tea", 46 | "repo_root_path": "/data/git/repositories", 47 | "lfs_root_path": "/data/git/lfs", 48 | "run_user": "git", 49 | "domain": "localhost", 50 | "ssh_port": "2221", 51 | "http_port": "3000", 52 | "app_url": "http://localhost:8080/", 53 | "log_root_path": "/data/gitea/log", 54 | "smtp_host": "", 55 | "smtp_from": "", 56 | "smtp_user": "", 57 | "smtp_passwd": "", 58 | "enable_federated_avatar": "on", 59 | "enable_open_id_sign_in": "on", 60 | "enable_open_id_sign_up": "on", 61 | "default_allow_create_organization": "on", 62 | "default_enable_timetracking": "on", 63 | "no_reply_address": "noreply.localhost", 64 | "password_algorithm": "pbkdf2", 65 | "admin_name": "", 66 | "admin_passwd": "", 67 | "admin_confirm_passwd": "", 68 | "admin_email": "", 69 | } 70 | requests.post(f"http://localhost:8080", data=INSTALL_PAYLOAD) 71 | 72 | 73 | class ParseCSRFGiteaForm(HTMLParser): 74 | token: str = None 75 | 76 | def handle_starttag(self, tag: str, attrs: (str, str)): 77 | if self.token: 78 | return 79 | 80 | if tag != "input": 81 | return 82 | 83 | token = None 84 | for (index, (k, v)) in enumerate(attrs): 85 | if k == "value": 86 | token = v 87 | 88 | if all([k == "name", v == "_csrf"]): 89 | if token: 90 | self.token = token 91 | return 92 | for (inner_index, (nk, nv)) in enumerate(attrs, start=index): 93 | if nk == "value": 94 | self.token = nv 95 | return 96 | 97 | 98 | class HTMLClient: 99 | session: Session 100 | 101 | def __init__(self): 102 | self.session = Session() 103 | 104 | @staticmethod 105 | def __get_csrf_token(page: str) -> str: 106 | parser = ParseCSRFGiteaForm() 107 | parser.feed(page) 108 | csrf = parser.token 109 | return csrf 110 | 111 | def get_csrf_token(self, url: str) -> str: 112 | resp = self.session.get(url, allow_redirects=False) 113 | if resp.status_code != 200 and resp.status_code != 302: 114 | print(resp.status_code, resp.text) 115 | raise Exception(f"Can't get csrf token: {resp.status_code}") 116 | csrf = self.__get_csrf_token(resp.text) 117 | return csrf 118 | 119 | 120 | def register(client: HTMLClient): 121 | url = "http://localhost:8080/user/sign_up" 122 | csrf = client.get_csrf_token(url) 123 | payload = { 124 | "_csrf": csrf, 125 | "user_name": GITEA_USER, 126 | "password": GITEA_PASSWORD, 127 | "retype": GITEA_PASSWORD, 128 | "email": GITEA_EMAIL, 129 | } 130 | resp = client.session.post(url, data=payload, allow_redirects=False) 131 | 132 | 133 | def login(client: HTMLClient): 134 | url = "http://localhost:8080/user/login" 135 | csrf = client.get_csrf_token(url) 136 | payload = { 137 | "_csrf": csrf, 138 | "user_name": GITEA_USER, 139 | "password": GITEA_PASSWORD, 140 | "remember": "on", 141 | } 142 | resp = client.session.post(url, data=payload, allow_redirects=False) 143 | print(f"login {client.session.cookies}") 144 | if resp.status_code == 302: 145 | print("User logged in") 146 | return 147 | 148 | raise Exception(f"[ERROR] Authentication failed. status code {resp.status_code}") 149 | 150 | 151 | def create_repositories(client: HTMLClient): 152 | print("foo") 153 | 154 | def get_repository_payload(csrf: str, name: str): 155 | data = { 156 | "_csrf": csrf, 157 | "uid": "1", 158 | "repo_name": name, 159 | "description": f"this repository is named {name}", 160 | "repo_template": "", 161 | "issue_labels": "", 162 | "gitignores": "", 163 | "license": "", 164 | "readme": "Default", 165 | "default_branch": "master", 166 | "trust_model": "default", 167 | } 168 | return data 169 | 170 | url = "http://localhost:8080/repo/create" 171 | for repo in REPOS: 172 | csrf = client.get_csrf_token(url) 173 | resp = client.session.post(url, data=get_repository_payload(csrf, repo)) 174 | print(f"Created repository {repo}") 175 | if resp.status_code != 302 and resp.status_code != 200: 176 | raise Exception( 177 | f"Error while creating repository: {repo} {resp.status_code}" 178 | ) 179 | add_tag(repo, client) 180 | 181 | 182 | def add_tag(repo: str, client: HTMLClient): 183 | print("adding tags") 184 | tag = "testing" 185 | url = f"http://{GITEA_USER}:{GITEA_PASSWORD}@localhost:8080/api/v1/repos/{GITEA_USER}/{repo}/topics/{tag}" 186 | resp = requests.put(url) 187 | if resp.status_code != 204: 188 | print(f"Error while adding tags repository: {repo} {resp.status_code}") 189 | raise Exception( 190 | f"Error while adding tags repository: {repo} {resp.status_code}" 191 | ) 192 | 193 | 194 | if __name__ == "__main__": 195 | for i in range(TOTAL_NUM_REPOS): 196 | REPOS.append(f"repository_{i}") 197 | check_online() 198 | print("Instance online") 199 | install() 200 | print("Instance configured and installed") 201 | client = HTMLClient() 202 | count = 0 203 | while True: 204 | try: 205 | register(client) 206 | print("User registered") 207 | login(client) 208 | create_repositories(client) 209 | break 210 | except Exception as e: 211 | print(f"Error: {e}") 212 | print(f"Retrying {count} time") 213 | count += 1 214 | sleep(5) 215 | continue 216 | -------------------------------------------------------------------------------- /sqlx-data.json: -------------------------------------------------------------------------------- 1 | { 2 | "db": "SQLite" 3 | } -------------------------------------------------------------------------------- /src/api.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2023 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::web; 19 | use actix_web::{HttpResponse, Responder}; 20 | use actix_web_codegen_const_routes::get; 21 | 22 | pub use api_routes::*; 23 | 24 | use crate::pages::chart::home::{OptionalPage, Page}; 25 | use crate::WebFederate; 26 | use crate::{errors::*, WebDB}; 27 | use crate::{introduce, search}; 28 | 29 | const LIMIT: u32 = 50; 30 | 31 | #[get(path = "ROUTES.forges")] 32 | pub async fn forges(db: WebDB, q: web::Query) -> ServiceResult { 33 | let q = q.into_inner(); 34 | let q: Page = q.into(); 35 | let offset = q.page * LIMIT; 36 | let forges = db.get_all_forges(false, offset, LIMIT).await?; 37 | 38 | Ok(HttpResponse::Ok().json(forges)) 39 | } 40 | 41 | #[get(path = "ROUTES.get_latest")] 42 | pub async fn lastest(federate: WebFederate) -> ServiceResult { 43 | let latest = federate.latest_tar_json().await.unwrap(); 44 | Ok(HttpResponse::Ok().json(latest)) 45 | } 46 | 47 | pub fn services(cfg: &mut web::ServiceConfig) { 48 | cfg.service(lastest); 49 | cfg.service(forges); 50 | search::services(cfg); 51 | introduce::services(cfg); 52 | } 53 | 54 | #[cfg(test)] 55 | mod tests { 56 | use super::*; 57 | use crate::tests::*; 58 | use crate::*; 59 | 60 | use actix_web::http::StatusCode; 61 | use actix_web::test; 62 | use db_core::prelude::*; 63 | use url::Url; 64 | 65 | #[actix_rt::test] 66 | async fn list_forges_works() { 67 | const URL: &str = "https://list-forges-works-test.example.com"; 68 | const HTML_PROFILE_URL: &str = "https://list-forges-works-test.example.com/user1"; 69 | const USERNAME: &str = "user1"; 70 | 71 | const REPO_NAME: &str = "asdlkfjaldsfjaksdf"; 72 | const HTML_REPO_URL: &str = 73 | "https://list-forges-works-test.example.com/user1/asdlkfjaldsfjaksdf"; 74 | const TAGS: [&str; 3] = ["test", "starchart", "spider"]; 75 | 76 | let (db, ctx, federate, _tmpdir) = sqlx_sqlite::get_ctx().await; 77 | let app = get_app!(ctx, db, federate).await; 78 | 79 | let url = Url::parse(URL).unwrap(); 80 | 81 | let create_forge_msg = CreateForge { 82 | url: url.clone(), 83 | forge_type: ForgeImplementation::Gitea, 84 | starchart_url: None, 85 | }; 86 | 87 | let _ = db.delete_forge_instance(&create_forge_msg.url).await; 88 | db.create_forge_instance(&create_forge_msg).await.unwrap(); 89 | assert!( 90 | db.forge_exists(&create_forge_msg.url).await.unwrap(), 91 | "forge creation failed, forge existence check failure" 92 | ); 93 | 94 | // test starts 95 | let lisit_res_resp = get_request!(&app, ROUTES.forges); 96 | assert_eq!(lisit_res_resp.status(), StatusCode::OK); 97 | let forges_list: Vec = test::read_body_json(lisit_res_resp).await; 98 | assert!(!forges_list.is_empty()); 99 | assert!(forges_list 100 | .iter() 101 | .any(|f| f.url == create_forge_msg.url.to_string())); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/counter.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * counter - A proof of work based DoS protection system 3 | * Copyright © 2021 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * al 17 | */ 18 | use std::time::Duration; 19 | 20 | use actix::clock::sleep; 21 | use actix::dev::*; 22 | use serde::{Deserialize, Serialize}; 23 | 24 | #[derive(Clone, Serialize, Deserialize, Debug)] 25 | pub struct Count { 26 | pub search_threshold: u32, 27 | pub duration: u64, 28 | } 29 | 30 | impl Count { 31 | /// increments the search count by one 32 | #[inline] 33 | pub fn add_search(&mut self) { 34 | self.search_threshold += 1; 35 | } 36 | 37 | /// decrements the search count by specified count 38 | #[inline] 39 | pub fn decrement_search_by(&mut self, count: u32) { 40 | if self.search_threshold > 0 { 41 | if self.search_threshold >= count { 42 | self.search_threshold -= count; 43 | } else { 44 | self.search_threshold = 0; 45 | } 46 | } 47 | } 48 | 49 | /// get [Counter]'s current search_threshold 50 | #[inline] 51 | pub fn get_searches(&self) -> u32 { 52 | self.search_threshold 53 | } 54 | } 55 | 56 | /// This struct represents the counter state and is used 57 | /// to configure leaky-bucket lifetime and manage defense 58 | #[derive(Clone, Serialize, Deserialize, Debug)] 59 | pub struct Counter(Count); 60 | 61 | impl From for Counter { 62 | fn from(c: Count) -> Counter { 63 | Counter(c) 64 | } 65 | } 66 | impl Actor for Counter { 67 | type Context = Context; 68 | } 69 | 70 | /// Message to decrement the search count 71 | #[derive(Message)] 72 | #[rtype(result = "()")] 73 | struct DeleteSearch; 74 | 75 | impl Handler for Counter { 76 | type Result = (); 77 | fn handle(&mut self, _msg: DeleteSearch, _ctx: &mut Self::Context) -> Self::Result { 78 | self.0.decrement_search_by(1); 79 | } 80 | } 81 | 82 | /// Message to increment the search count 83 | /// returns difficulty factor and lifetime 84 | #[derive(Message)] 85 | #[rtype(result = "u32")] 86 | pub struct AddSearch; 87 | 88 | impl Handler for Counter { 89 | type Result = MessageResult; 90 | 91 | fn handle(&mut self, _: AddSearch, ctx: &mut Self::Context) -> Self::Result { 92 | self.0.add_search(); 93 | let addr = ctx.address(); 94 | 95 | let duration: Duration = Duration::new(self.0.duration, 0); 96 | let wait_for = async move { 97 | sleep(duration).await; 98 | //delay_for(duration).await; 99 | addr.send(DeleteSearch).await.unwrap(); 100 | } 101 | .into_actor(self); 102 | ctx.spawn(wait_for); 103 | 104 | MessageResult(self.0.get_searches()) 105 | } 106 | } 107 | 108 | /// Message to get the search count 109 | #[derive(Message)] 110 | #[rtype(result = "u32")] 111 | pub struct GetCurrentSearchCount; 112 | 113 | impl Handler for Counter { 114 | type Result = MessageResult; 115 | 116 | fn handle(&mut self, _: GetCurrentSearchCount, _ctx: &mut Self::Context) -> Self::Result { 117 | MessageResult(self.0.get_searches()) 118 | } 119 | } 120 | 121 | /// Message to stop [Counter] 122 | #[derive(Message)] 123 | #[rtype(result = "()")] 124 | pub struct Stop; 125 | 126 | impl Handler for Counter { 127 | type Result = (); 128 | 129 | fn handle(&mut self, _: Stop, ctx: &mut Self::Context) -> Self::Result { 130 | ctx.stop() 131 | } 132 | } 133 | 134 | #[cfg(test)] 135 | pub mod tests { 136 | use super::*; 137 | 138 | // constants for testing 139 | // (search count, level) 140 | pub const LEVEL_1: (u32, u32) = (50, 50); 141 | pub const LEVEL_2: (u32, u32) = (500, 500); 142 | pub const DURATION: u64 = 5; 143 | 144 | type MyActor = Addr; 145 | 146 | async fn race(addr: Addr, count: (u32, u32)) { 147 | for _ in 0..count.0 as usize - 1 { 148 | let _ = addr.send(AddSearch).await.unwrap(); 149 | } 150 | } 151 | 152 | pub fn get_counter() -> Counter { 153 | Counter(Count { 154 | duration: DURATION, 155 | search_threshold: 0, 156 | }) 157 | } 158 | 159 | #[test] 160 | fn counter_decrement_by_works() { 161 | let mut m = get_counter(); 162 | for _ in 0..100 { 163 | m.0.add_search(); 164 | } 165 | assert_eq!(m.0.get_searches(), 100); 166 | m.0.decrement_search_by(50); 167 | assert_eq!(m.0.get_searches(), 50); 168 | m.0.decrement_search_by(500); 169 | assert_eq!(m.0.get_searches(), 0); 170 | } 171 | 172 | #[actix_rt::test] 173 | async fn get_current_search_count_works() { 174 | let addr: MyActor = get_counter().start(); 175 | 176 | addr.send(AddSearch).await.unwrap(); 177 | addr.send(AddSearch).await.unwrap(); 178 | addr.send(AddSearch).await.unwrap(); 179 | addr.send(AddSearch).await.unwrap(); 180 | let count = addr.send(GetCurrentSearchCount).await.unwrap(); 181 | 182 | assert_eq!(count, 4); 183 | } 184 | 185 | #[actix_rt::test] 186 | async fn counter_defense_loosenup_works() { 187 | let addr: MyActor = get_counter().start(); 188 | 189 | race(addr.clone(), LEVEL_2).await; 190 | addr.send(AddSearch).await.unwrap(); 191 | assert_eq!(addr.send(GetCurrentSearchCount).await.unwrap(), LEVEL_2.1); 192 | 193 | let duration = Duration::new(DURATION + 1, 0); 194 | sleep(duration).await; 195 | //delay_for(duration).await; 196 | 197 | addr.send(AddSearch).await.unwrap(); 198 | let count = addr.send(GetCurrentSearchCount).await.unwrap(); 199 | assert_eq!(count, 1); 200 | } 201 | 202 | #[actix_rt::test] 203 | #[should_panic] 204 | async fn stop_works() { 205 | let addr: MyActor = get_counter().start(); 206 | addr.send(Stop).await.unwrap(); 207 | addr.send(AddSearch).await.unwrap(); 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/ctx.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::sync::Arc; 19 | use std::time::Duration; 20 | 21 | use actix::dev::*; 22 | use reqwest::{Client, ClientBuilder}; 23 | 24 | use crate::master::Master; 25 | use crate::settings::Settings; 26 | use crate::{PKG_NAME, VERSION}; 27 | 28 | /// in seconds 29 | const CLIENT_TIMEOUT: u64 = 60; 30 | 31 | #[derive(Clone)] 32 | pub struct Ctx { 33 | pub client: Client, 34 | pub settings: Settings, 35 | pub master: Addr, 36 | } 37 | 38 | impl Ctx { 39 | pub async fn new(settings: Settings) -> Arc { 40 | let host = settings.introducer.public_url.host_str().unwrap(); 41 | let host = if let Some(port) = settings.introducer.public_url.port() { 42 | format!("{host}:{port}") 43 | } else { 44 | host.to_owned() 45 | }; 46 | let ua = format!("{VERSION}---{PKG_NAME}---{host}"); 47 | let timeout = Duration::new(CLIENT_TIMEOUT, 0); 48 | let client = ClientBuilder::new() 49 | .user_agent(&*ua) 50 | .use_rustls_tls() 51 | .timeout(timeout) 52 | .connect_timeout(timeout) 53 | .tcp_keepalive(timeout) 54 | .build() 55 | .unwrap(); 56 | 57 | let master = Master::new(45).start(); 58 | 59 | Arc::new(Self { 60 | client, 61 | settings, 62 | master, 63 | }) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/db.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use crate::settings::Settings; 18 | use db_core::prelude::*; 19 | 20 | pub type BoxDB = Box; 21 | 22 | pub mod sqlite { 23 | use super::*; 24 | use db_sqlx_sqlite::{ConnectionOptions, Fresh}; 25 | use sqlx::sqlite::SqlitePoolOptions; 26 | 27 | pub async fn get_data(settings: Option) -> BoxDB { 28 | let settings = settings.unwrap_or_else(|| Settings::new().unwrap()); 29 | 30 | let pool = settings.database.pool; 31 | let pool_options = SqlitePoolOptions::new().max_connections(pool); 32 | let connection_options = ConnectionOptions::Fresh(Fresh { 33 | pool_options, 34 | url: settings.database.url, 35 | }); 36 | 37 | let db = connection_options.connect().await.unwrap(); 38 | db.migrate().await.unwrap(); 39 | Box::new(db) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/dns/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use serde::{Deserialize, Serialize}; 19 | 20 | #[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize)] 21 | pub struct Configuration { 22 | pub spidering: bool, 23 | pub rate: Option, 24 | } 25 | 26 | impl Configuration { 27 | pub fn parse(s: &str) -> Self { 28 | fn parse_inner(config: &mut Configuration, s: &str) { 29 | let mut inner = s.split('='); 30 | let k = inner.next().unwrap().trim(); 31 | let v = inner.next().unwrap().trim(); 32 | println!("split inner: {:?}: {:?}", k, v); 33 | 34 | if k == "spidering" { 35 | if v == "false" { 36 | config.spidering = false; 37 | } else if v == "true" { 38 | config.spidering = true; 39 | } else { 40 | panic!("Value {k} is not bool, can't set for spidering"); 41 | } 42 | } else if k == "rate" { 43 | let x: u64 = v.parse().unwrap(); 44 | config.rate = Some(x); 45 | } else { 46 | panic!("Key {k} and Value {v} is implemented or supported"); 47 | } 48 | } 49 | let mut config = Self::default(); 50 | if s.contains(',') { 51 | for spilt in s.split(',') { 52 | println!("split: {:?}", spilt); 53 | parse_inner(&mut config, spilt); 54 | } 55 | } else { 56 | parse_inner(&mut config, s); 57 | } 58 | config 59 | } 60 | } 61 | #[cfg(test)] 62 | mod tests { 63 | use super::*; 64 | 65 | #[test] 66 | fn dns_txt_parser_works() { 67 | const REQ: &str = "spidering=false,rate=500"; 68 | const RES: Configuration = Configuration { 69 | spidering: false, 70 | rate: Some(500), 71 | }; 72 | 73 | const REQ_2: &str = "spidering=true"; 74 | const RES_2: Configuration = Configuration { 75 | spidering: true, 76 | rate: None, 77 | }; 78 | 79 | assert_eq!(Configuration::parse(REQ), RES); 80 | assert_eq!(Configuration::parse(REQ_2), RES_2); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | use std::convert::From; 20 | 21 | use actix_web::{ 22 | error::ResponseError, 23 | http::{header, StatusCode}, 24 | HttpResponse, HttpResponseBuilder, 25 | }; 26 | use db_core::errors::DBError; 27 | use derive_more::{Display, Error}; 28 | use serde::{Deserialize, Serialize}; 29 | use url::ParseError; 30 | use validator::ValidationErrors; 31 | 32 | #[derive(Debug, Display, Error)] 33 | pub struct DBErrorWrapper(DBError); 34 | 35 | impl std::cmp::PartialEq for DBErrorWrapper { 36 | fn eq(&self, other: &Self) -> bool { 37 | format!("{}", self.0) == format!("{}", other.0) 38 | } 39 | } 40 | 41 | #[derive(Debug, Display, PartialEq, Error)] 42 | #[cfg(not(tarpaulin_include))] 43 | pub enum ServiceError { 44 | #[display(fmt = "internal server error")] 45 | InternalServerError, 46 | 47 | #[display( 48 | fmt = "This server is is closed for registration. Contact admin if this is unexpecter" 49 | )] 50 | ClosedForRegistration, 51 | 52 | #[display(fmt = "The value you entered for email is not an email")] //405j 53 | NotAnEmail, 54 | #[display(fmt = "The value you entered for URL is not a URL")] //405j 55 | NotAUrl, 56 | 57 | #[display(fmt = "{}", _0)] 58 | DBError(DBErrorWrapper), 59 | 60 | /// DNS challenge value is already taken 61 | #[display(fmt = "DNS challenge is already taken")] 62 | DuplicateChallengeText, 63 | 64 | /// DNS challenge hostname is already taken 65 | #[display(fmt = "DNS challenge hostname is already taken")] 66 | DuplicateChallengeHostname, 67 | 68 | /// Hostname is already taken 69 | #[display(fmt = "Hostname is already taken")] 70 | DuplicateHostname, 71 | 72 | /// Forge Type is already taken 73 | #[display(fmt = "Forge Type is already taken")] 74 | DuplicateForgeType, 75 | 76 | /// HTML link Type is already taken 77 | #[display(fmt = "User HTML link is already taken")] 78 | DuplicateUserLink, 79 | 80 | /// Topic is already taken 81 | #[display(fmt = "Topic is already taken")] 82 | DuplicateTopic, 83 | 84 | /// Repository link is already taken 85 | #[display(fmt = "Repository link is already taken")] 86 | DuplicateRepositoryLink, 87 | } 88 | 89 | #[derive(Serialize, Deserialize)] 90 | #[cfg(not(tarpaulin_include))] 91 | pub struct ErrorToResponse { 92 | pub error: String, 93 | } 94 | 95 | #[cfg(not(tarpaulin_include))] 96 | impl ResponseError for ServiceError { 97 | #[cfg(not(tarpaulin_include))] 98 | fn error_response(&self) -> HttpResponse { 99 | HttpResponseBuilder::new(self.status_code()) 100 | .append_header((header::CONTENT_TYPE, "application/json; charset=UTF-8")) 101 | .body( 102 | serde_json::to_string(&ErrorToResponse { 103 | error: self.to_string(), 104 | }) 105 | .unwrap(), 106 | ) 107 | } 108 | 109 | #[cfg(not(tarpaulin_include))] 110 | fn status_code(&self) -> StatusCode { 111 | match self { 112 | ServiceError::ClosedForRegistration => StatusCode::FORBIDDEN, 113 | ServiceError::InternalServerError => StatusCode::INTERNAL_SERVER_ERROR, 114 | ServiceError::NotAnEmail => StatusCode::BAD_REQUEST, 115 | ServiceError::NotAUrl => StatusCode::BAD_REQUEST, 116 | ServiceError::DBError(_) => StatusCode::INTERNAL_SERVER_ERROR, 117 | ServiceError::DuplicateChallengeHostname 118 | | ServiceError::DuplicateHostname 119 | | ServiceError::DuplicateUserLink 120 | | ServiceError::DuplicateTopic 121 | | ServiceError::DuplicateRepositoryLink => StatusCode::BAD_REQUEST, 122 | 123 | ServiceError::DuplicateChallengeText | ServiceError::DuplicateForgeType => { 124 | StatusCode::INTERNAL_SERVER_ERROR 125 | } 126 | } 127 | } 128 | } 129 | 130 | impl From for ServiceError { 131 | #[cfg(not(tarpaulin_include))] 132 | fn from(e: DBError) -> ServiceError { 133 | println!("from conversin: {}", e); 134 | ServiceError::DBError(DBErrorWrapper(e)) 135 | // match e { 136 | // // TODO: resolve all errors to ServiceError::* 137 | // _ => ServiceError::DBError(DBErrorWrapper(e)), 138 | // } 139 | } 140 | } 141 | 142 | impl From for ServiceError { 143 | #[cfg(not(tarpaulin_include))] 144 | fn from(_: ValidationErrors) -> ServiceError { 145 | ServiceError::NotAnEmail 146 | } 147 | } 148 | 149 | impl From for ServiceError { 150 | #[cfg(not(tarpaulin_include))] 151 | fn from(_: ParseError) -> ServiceError { 152 | ServiceError::NotAUrl 153 | } 154 | } 155 | 156 | #[cfg(not(tarpaulin_include))] 157 | impl From for ServiceError { 158 | fn from(e: actix::MailboxError) -> Self { 159 | log::debug!("Actor mailbox error: {:?}", e); 160 | Self::InternalServerError 161 | } 162 | } 163 | 164 | #[cfg(not(tarpaulin_include))] 165 | pub type ServiceResult = std::result::Result; 166 | -------------------------------------------------------------------------------- /src/federate.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::sync::Arc; 18 | 19 | use federate_core::Federate; 20 | use publiccodeyml::{errors::FederateErorr, PccFederate}; 21 | 22 | use crate::settings::Settings; 23 | 24 | pub type ArcFederate = Arc>; 25 | 26 | pub async fn get_federate(settings: Option) -> ArcFederate { 27 | let settings = settings.unwrap_or_else(|| Settings::new().unwrap()); 28 | Arc::new(PccFederate::new(settings.repository.root).await.unwrap()) 29 | } 30 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::sync::Arc; 19 | 20 | use actix_files::Files; 21 | use actix_web::{middleware, web::Data, App, HttpServer}; 22 | use lazy_static::lazy_static; 23 | use tokio::sync::oneshot; 24 | 25 | pub mod api; 26 | pub mod counter; 27 | pub mod ctx; 28 | pub mod db; 29 | pub mod dns; 30 | pub mod errors; 31 | pub mod federate; 32 | pub mod introduce; 33 | pub mod master; 34 | pub mod pages; 35 | pub mod routes; 36 | pub mod search; 37 | pub mod settings; 38 | pub mod spider; 39 | pub mod static_assets; 40 | 41 | #[cfg(test)] 42 | mod tests; 43 | pub mod utils; 44 | pub mod verify; 45 | 46 | use crate::federate::{get_federate, ArcFederate}; 47 | use ctx::Ctx; 48 | use db::{sqlite, BoxDB}; 49 | use settings::Settings; 50 | use static_assets::FileMap; 51 | 52 | pub use crate::pages::routes::PAGES; 53 | 54 | pub const CACHE_AGE: u32 = 60 * 60 * 24 * 30; // one month, I think? 55 | pub const VERSION: &str = env!("CARGO_PKG_VERSION"); 56 | pub const PKG_NAME: &str = env!("CARGO_PKG_NAME"); 57 | pub const GIT_COMMIT_HASH: &str = env!("GIT_HASH"); 58 | 59 | pub type ArcCtx = Arc; 60 | pub type WebCtx = Data; 61 | pub type WebDB = Data; 62 | pub type WebFederate = Data; 63 | 64 | lazy_static! { 65 | pub static ref FILES: FileMap = FileMap::new(); 66 | } 67 | 68 | #[actix_rt::main] 69 | async fn main() { 70 | let settings = Settings::new().unwrap(); 71 | pretty_env_logger::init(); 72 | lazy_static::initialize(&pages::TEMPLATES); 73 | 74 | let ctx = Ctx::new(settings.clone()).await; 75 | let db = WebDB::new(sqlite::get_data(Some(settings.clone())).await); 76 | let federate = WebFederate::new(get_federate(Some(settings.clone())).await); 77 | 78 | let (kill_crawler, rx) = oneshot::channel(); 79 | let crawler = spider::Crawler::new( 80 | rx, 81 | ctx.clone(), 82 | db.as_ref().clone(), 83 | federate.as_ref().clone(), 84 | ); 85 | 86 | let crawler_fut = tokio::spawn(spider::Crawler::start(crawler.clone())); 87 | let ctx = WebCtx::new(ctx); 88 | 89 | let c = ctx.clone(); 90 | let d = db.clone(); 91 | let (kill_introducer, introducer_fut) = 92 | Ctx::spawn_bootstrap(c, d.as_ref().clone()).await.unwrap(); 93 | 94 | let _c = ctx.clone(); 95 | let _d = db.clone(); 96 | let _f = federate.clone(); 97 | 98 | let socket_addr = settings.server.get_ip(); 99 | HttpServer::new(move || { 100 | App::new() 101 | .wrap(middleware::Logger::default()) 102 | .wrap(middleware::Compress::default()) 103 | .app_data(federate.clone()) 104 | .app_data(db.clone()) 105 | .app_data(ctx.clone()) 106 | .wrap( 107 | middleware::DefaultHeaders::new().add(("Permissions-Policy", "interest-cohort=()")), 108 | ) 109 | .configure(routes::services) 110 | .service(Files::new("/federate", &settings.repository.root).show_files_listing()) 111 | }) 112 | .bind(&socket_addr) 113 | .unwrap() 114 | .run() 115 | .await 116 | .unwrap(); 117 | 118 | // let s = tokio::spawn(server_fut); 119 | // f.import( 120 | // url::Url::parse("http://localhost:7000").unwrap(), 121 | // &c.client, 122 | // &d, 123 | // ) 124 | // .await 125 | // .unwrap(); 126 | kill_crawler.send(true).unwrap(); 127 | kill_introducer.send(true).unwrap(); 128 | crawler_fut.await.unwrap().await; 129 | introducer_fut.await; 130 | } 131 | -------------------------------------------------------------------------------- /src/pages/auth/add.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::http::{self, header::ContentType}; 19 | use actix_web::{HttpResponse, Responder}; 20 | use actix_web_codegen_const_routes::{get, post}; 21 | use serde::{Deserialize, Serialize}; 22 | use std::cell::RefCell; 23 | use tera::Context; 24 | use url::Url; 25 | 26 | use crate::pages::errors::*; 27 | use crate::settings::Settings; 28 | use crate::*; 29 | 30 | pub use crate::pages::*; 31 | 32 | pub const TITLE: &str = "Setup spidering"; 33 | pub const AUTH_ADD: TemplateFile = TemplateFile::new("auth_add", "pages/auth/add.html"); 34 | 35 | pub struct AddChallenge { 36 | ctx: RefCell, 37 | } 38 | 39 | impl CtxError for AddChallenge { 40 | fn with_error(&self, e: &ReadableError) -> String { 41 | self.ctx.borrow_mut().insert(ERROR_KEY, e); 42 | self.render() 43 | } 44 | } 45 | 46 | #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] 47 | pub struct AddChallengePayload { 48 | pub hostname: Url, 49 | } 50 | 51 | impl AddChallenge { 52 | fn new(settings: &Settings, payload: Option<&AddChallengePayload>) -> Self { 53 | let ctx = RefCell::new(ctx(settings)); 54 | ctx.borrow_mut().insert(TITLE_KEY, TITLE); 55 | if let Some(payload) = payload { 56 | ctx.borrow_mut().insert(PAYLOAD_KEY, payload); 57 | } 58 | Self { ctx } 59 | } 60 | 61 | pub fn render(&self) -> String { 62 | TEMPLATES.render(AUTH_ADD.name, &self.ctx.borrow()).unwrap() 63 | } 64 | 65 | pub fn page(s: &Settings) -> String { 66 | let p = Self::new(s, None); 67 | p.render() 68 | } 69 | } 70 | 71 | #[get(path = "PAGES.auth.add")] 72 | pub async fn get_add(ctx: WebCtx) -> impl Responder { 73 | let login = AddChallenge::page(&ctx.settings); 74 | let html = ContentType::html(); 75 | HttpResponse::Ok().content_type(html).body(login) 76 | } 77 | 78 | pub fn services(cfg: &mut web::ServiceConfig) { 79 | cfg.service(get_add); 80 | cfg.service(add_submit); 81 | } 82 | 83 | #[post(path = "PAGES.auth.add")] 84 | pub async fn add_submit( 85 | payload: web::Form, 86 | ) -> PageResult { 87 | let link = PAGES.auth.verify_get(payload.hostname.as_ref()); 88 | 89 | Ok(HttpResponse::Found() 90 | .insert_header((http::header::LOCATION, link)) 91 | .finish()) 92 | } 93 | 94 | #[cfg(test)] 95 | mod tests { 96 | use actix_web::http::StatusCode; 97 | use actix_web::test; 98 | use url::Url; 99 | 100 | use super::AddChallengePayload; 101 | use crate::errors::*; 102 | 103 | #[cfg(test)] 104 | mod isolated { 105 | use crate::errors::ServiceError; 106 | use crate::pages::auth::add::{AddChallenge, AddChallengePayload, ReadableError}; 107 | use crate::pages::errors::*; 108 | use crate::settings::Settings; 109 | 110 | #[test] 111 | fn add_page_works() { 112 | let settings = Settings::new().unwrap(); 113 | AddChallenge::page(&settings); 114 | let payload = AddChallengePayload { 115 | hostname: url::Url::parse("https://example.com").unwrap(), 116 | }; 117 | let page = AddChallenge::new(&settings, Some(&payload)); 118 | page.with_error(&ReadableError::new(&ServiceError::ClosedForRegistration)); 119 | page.render(); 120 | } 121 | } 122 | 123 | #[actix_rt::test] 124 | async fn add_routes_work() { 125 | use crate::tests::*; 126 | use crate::*; 127 | const BASE_DOMAIN: &str = "add_routes_work.example.org"; 128 | 129 | let (db, ctx, federate, _tmpdir) = sqlx_sqlite::get_ctx().await; 130 | let app = get_app!(ctx, db, federate).await; 131 | 132 | let payload = AddChallengePayload { 133 | hostname: Url::parse(&format!("https://{BASE_DOMAIN}")).unwrap(), 134 | }; 135 | 136 | println!("{}", payload.hostname); 137 | 138 | let resp = test::call_service( 139 | &app, 140 | post_request!(&payload, PAGES.auth.add, FORM).to_request(), 141 | ) 142 | .await; 143 | if resp.status() != StatusCode::FOUND { 144 | let resp_err: ErrorToResponse = test::read_body_json(resp).await; 145 | panic!("{}", resp_err.error); 146 | } 147 | assert_eq!(resp.status(), StatusCode::FOUND); 148 | 149 | // replay config 150 | let resp = test::call_service( 151 | &app, 152 | post_request!(&payload, PAGES.auth.add, FORM).to_request(), 153 | ) 154 | .await; 155 | 156 | assert_eq!(resp.status(), StatusCode::FOUND); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/pages/auth/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | pub mod add; 19 | pub mod verify; 20 | pub use add::AUTH_ADD; 21 | pub use verify::AUTH_CHALLENGE; 22 | 23 | pub use super::{ctx, TemplateFile, ERROR_KEY, PAGES, PAYLOAD_KEY, TITLE_KEY}; 24 | 25 | pub fn register_templates(t: &mut tera::Tera) { 26 | AUTH_ADD.register(t).expect(AUTH_ADD.name); 27 | AUTH_CHALLENGE.register(t).expect(AUTH_ADD.name); 28 | } 29 | 30 | pub fn services(cfg: &mut actix_web::web::ServiceConfig) { 31 | add::services(cfg); 32 | verify::services(cfg); 33 | } 34 | -------------------------------------------------------------------------------- /src/pages/auth/verify.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::http::{self, header::ContentType}; 19 | use actix_web::{HttpResponse, Responder}; 20 | use actix_web_codegen_const_routes::{get, post}; 21 | use serde::{Deserialize, Serialize}; 22 | use std::cell::RefCell; 23 | use tera::Context; 24 | use url::Url; 25 | 26 | use crate::pages::errors::*; 27 | use crate::settings::Settings; 28 | use crate::verify::{Challenge, TXTChallenge}; 29 | use crate::*; 30 | 31 | pub use crate::pages::*; 32 | 33 | pub const TITLE: &str = "Setup spidering"; 34 | pub const AUTH_CHALLENGE: TemplateFile = 35 | TemplateFile::new("auth_challenge", "pages/auth/challenge.html"); 36 | 37 | pub struct VerifyChallenge { 38 | ctx: RefCell, 39 | } 40 | 41 | impl CtxError for VerifyChallenge { 42 | fn with_error(&self, e: &ReadableError) -> String { 43 | self.ctx.borrow_mut().insert(ERROR_KEY, e); 44 | self.render() 45 | } 46 | } 47 | 48 | #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] 49 | pub struct VerifyChallengePayload { 50 | pub hostname: Url, 51 | } 52 | 53 | impl VerifyChallenge { 54 | fn new(settings: &Settings, payload: &Challenge) -> Self { 55 | let ctx = RefCell::new(ctx(settings)); 56 | ctx.borrow_mut().insert(TITLE_KEY, TITLE); 57 | ctx.borrow_mut().insert(PAYLOAD_KEY, payload); 58 | ctx.borrow_mut() 59 | .insert("form_url", &PAGES.auth.verify_get(&payload.key)); 60 | Self { ctx } 61 | } 62 | 63 | pub fn render(&self) -> String { 64 | TEMPLATES 65 | .render(AUTH_CHALLENGE.name, &self.ctx.borrow()) 66 | .unwrap() 67 | } 68 | 69 | pub fn page(s: &Settings, payload: &Challenge) -> String { 70 | let p = Self::new(s, payload); 71 | p.render() 72 | } 73 | } 74 | 75 | #[get(path = "PAGES.auth.verify")] 76 | pub async fn get_verify( 77 | ctx: WebCtx, 78 | query: web::Query, 79 | ) -> PageResult { 80 | let challenge = TXTChallenge::new(&ctx, &query.hostname); 81 | let value = Challenge { 82 | key: challenge.key, 83 | value: challenge.value, 84 | url: query.hostname.to_string(), 85 | }; 86 | 87 | let login = VerifyChallenge::page(&ctx.settings, &value); 88 | let html = ContentType::html(); 89 | Ok(HttpResponse::Ok().content_type(html).body(login)) 90 | } 91 | 92 | pub fn services(cfg: &mut web::ServiceConfig) { 93 | cfg.service(get_verify); 94 | cfg.service(submit_verify); 95 | } 96 | 97 | #[post(path = "PAGES.auth.verify")] 98 | pub async fn submit_verify( 99 | payload: web::Form, 100 | ctx: WebCtx, 101 | db: WebDB, 102 | federate: WebFederate, 103 | ) -> PageResult { 104 | let payload = payload.into_inner(); 105 | let challenge = TXTChallenge::new(&ctx, &payload.hostname); 106 | 107 | match challenge.verify_txt().await { 108 | Ok(true) => { 109 | let ctx = ctx.clone(); 110 | let federate = federate.clone(); 111 | let db = db.clone(); 112 | let fut = async move { 113 | ctx.crawl(&payload.hostname, &db, &federate).await; 114 | }; 115 | 116 | tokio::spawn(fut); 117 | Ok(HttpResponse::Found() 118 | .insert_header((http::header::LOCATION, PAGES.home)) 119 | .finish()) 120 | } 121 | _ => Ok(HttpResponse::Found() 122 | .insert_header(( 123 | http::header::LOCATION, 124 | PAGES.auth.verify_get(&challenge.key), 125 | )) 126 | .finish()), 127 | } 128 | } 129 | 130 | //#[cfg(test)] 131 | //mod tests { 132 | // use actix_web::http::StatusCode; 133 | // use actix_web::test; 134 | // use url::Url; 135 | // 136 | // use super::VerifyChallenge; 137 | // use super::VerifyChallengePayload; 138 | // use super::TXTChallenge; 139 | // use crate::errors::*; 140 | // use crate::pages::errors::*; 141 | // use crate::settings::Settings; 142 | // 143 | // use db_core::prelude::*; 144 | // 145 | // #[cfg(test)] 146 | // mod isolated { 147 | // use crate::errors::ServiceError; 148 | // use crate::pages::auth::add::{VerifyChallenge, VerifyChallengePayload, ReadableError}; 149 | // use crate::pages::errors::*; 150 | // use crate::settings::Settings; 151 | // 152 | // #[test] 153 | // fn add_page_works() { 154 | // let settings = Settings::new().unwrap(); 155 | // VerifyChallenge::page(&settings); 156 | // let payload = VerifyChallengePayload { 157 | // hostname: "https://example.com".into(), 158 | // }; 159 | // let page = VerifyChallenge::new(&settings, Some(&payload)); 160 | // page.with_error(&ReadableError::new(&ServiceError::ClosedForRegistration)); 161 | // page.render(); 162 | // } 163 | // } 164 | // 165 | // #[actix_rt::test] 166 | // async fn add_routes_work() { 167 | // use crate::tests::*; 168 | // use crate::*; 169 | // const BASE_DOMAIN: &str = "add_routes_work.example.org"; 170 | // 171 | // let (db, ctx, federate, _tmpdir) = sqlx_sqlite::get_ctx().await; 172 | // let app = get_app!(ctx, db, federate).await; 173 | // 174 | // let payload = VerifyChallengePayload { 175 | // hostname: format!("https://{BASE_DOMAIN}"), 176 | // }; 177 | // 178 | // println!("{}", payload.hostname); 179 | // 180 | // let hostname = get_hostname(&Url::parse(&payload.hostname).unwrap()); 181 | // let key = TXTChallenge::get_challenge_txt_key(&ctx, &hostname); 182 | // 183 | // db.delete_dns_challenge(&key).await.unwrap(); 184 | // assert!(!db.dns_challenge_exists(&key).await.unwrap()); 185 | // 186 | // let resp = test::call_service( 187 | // &app, 188 | // post_request!(&payload, PAGES.auth.add, FORM).to_request(), 189 | // ) 190 | // .await; 191 | // if resp.status() != StatusCode::FOUND { 192 | // let resp_err: ErrorToResponse = test::read_body_json(resp).await; 193 | // panic!("{}", resp_err.error); 194 | // } 195 | // assert_eq!(resp.status(), StatusCode::FOUND); 196 | // 197 | // assert!(db.dns_challenge_exists(&key).await.unwrap()); 198 | // 199 | // let challenge = db.get_dns_challenge_solution(&key).await.unwrap(); 200 | // 201 | // // replay config 202 | // let resp = test::call_service( 203 | // &app, 204 | // post_request!(&payload, PAGES.auth.add, FORM).to_request(), 205 | // ) 206 | // .await; 207 | // 208 | // assert_eq!(resp.status(), StatusCode::FOUND); 209 | // 210 | // assert!(db.dns_challenge_exists(&key).await.unwrap()); 211 | // assert_eq!( 212 | // challenge, 213 | // db.get_dns_challenge_solution(&key).await.unwrap() 214 | // ); 215 | // } 216 | //} 217 | -------------------------------------------------------------------------------- /src/pages/chart/home.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::http::header::ContentType; 19 | use actix_web::{HttpResponse, Responder}; 20 | use actix_web_codegen_const_routes::get; 21 | use serde::{Deserialize, Serialize}; 22 | use std::cell::RefCell; 23 | use tera::Context; 24 | 25 | use db_core::prelude::*; 26 | 27 | use crate::errors::ServiceResult; 28 | use crate::pages::errors::*; 29 | use crate::settings::Settings; 30 | use crate::*; 31 | 32 | pub use crate::pages::*; 33 | 34 | pub const TITLE: &str = "Explore"; 35 | pub const EXPLORE: TemplateFile = TemplateFile::new("explore_page", "pages/chart/index.html"); 36 | pub const REPO_INFO: TemplateFile = 37 | TemplateFile::new("repo_info", "pages/chart/components/repo_info.html"); 38 | 39 | pub const SEARCH_BAR: TemplateFile = TemplateFile::new("search_bar", "components/nav/search.html"); 40 | 41 | pub struct ExplorePage { 42 | ctx: RefCell, 43 | } 44 | 45 | impl CtxError for ExplorePage { 46 | fn with_error(&self, e: &ReadableError) -> String { 47 | self.ctx.borrow_mut().insert(ERROR_KEY, e); 48 | self.render() 49 | } 50 | } 51 | 52 | #[derive(Clone, Debug, PartialEq, Eq, Default, Deserialize, Serialize)] 53 | pub struct ExplorePagePayload { 54 | pub repos: Vec, 55 | pub next_page: String, 56 | pub prev_page: String, 57 | } 58 | 59 | impl ExplorePage { 60 | fn new(settings: &Settings, payload: &ExplorePagePayload) -> Self { 61 | let ctx = RefCell::new(ctx(settings)); 62 | ctx.borrow_mut().insert(TITLE_KEY, TITLE); 63 | ctx.borrow_mut().insert(PAYLOAD_KEY, payload); 64 | Self { ctx } 65 | } 66 | 67 | pub fn render(&self) -> String { 68 | TEMPLATES.render(EXPLORE.name, &self.ctx.borrow()).unwrap() 69 | } 70 | 71 | pub fn page(s: &Settings, payload: &ExplorePagePayload) -> String { 72 | let p = Self::new(s, payload); 73 | p.render() 74 | } 75 | } 76 | 77 | pub fn services(cfg: &mut web::ServiceConfig) { 78 | cfg.service(explore); 79 | } 80 | 81 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 82 | pub struct Page { 83 | pub page: u32, 84 | } 85 | 86 | impl Page { 87 | pub fn next(&self) -> u32 { 88 | self.page + 2 89 | } 90 | 91 | pub fn prev(&self) -> u32 { 92 | if self.page == 0 { 93 | 1 94 | } else { 95 | self.page 96 | } 97 | } 98 | } 99 | 100 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 101 | pub struct OptionalPage { 102 | pub page: Option, 103 | } 104 | 105 | impl From for Page { 106 | fn from(o: OptionalPage) -> Self { 107 | match o.page { 108 | Some(page) => Self { page: page - 1 }, 109 | None => Page { page: 0 }, 110 | } 111 | } 112 | } 113 | 114 | #[get(path = "PAGES.explore")] 115 | pub async fn explore( 116 | q: web::Query, 117 | ctx: WebCtx, 118 | db: WebDB, 119 | ) -> PageResult { 120 | let q = q.into_inner(); 121 | async fn _explore( 122 | _ctx: &ArcCtx, 123 | db: &BoxDB, 124 | p: &Page, 125 | ) -> ServiceResult> { 126 | const LIMIT: u32 = 10; 127 | let offset = p.page * LIMIT; 128 | let responses = db.get_all_repositories(offset, LIMIT).await?; 129 | Ok(responses) 130 | } 131 | let q: Page = q.into(); 132 | 133 | let repos = _explore(&ctx, &db, &q).await.map_err(|e| { 134 | let x = ExplorePagePayload::default(); 135 | PageError::new(ExplorePage::new(&ctx.settings, &x), e) 136 | })?; 137 | 138 | let payload = ExplorePagePayload { 139 | repos, 140 | next_page: PAGES.explore_next(q.next()), 141 | prev_page: PAGES.explore_next(q.prev()), 142 | }; 143 | let page = ExplorePage::page(&ctx.settings, &payload); 144 | 145 | let html = ContentType::html(); 146 | Ok(HttpResponse::Ok().content_type(html).body(page)) 147 | } 148 | 149 | #[cfg(test)] 150 | mod tests { 151 | 152 | #[test] 153 | fn page_counter_increases() { 154 | use super::*; 155 | 156 | let mut page = Page { page: 0 }; 157 | 158 | assert_eq!(page.next(), 2); 159 | assert_eq!(page.prev(), 1); 160 | 161 | page.page = 1; 162 | assert_eq!(page.next(), 3); 163 | assert_eq!(page.prev(), 1); 164 | 165 | let op = OptionalPage { page: None }; 166 | let p: Page = op.into(); 167 | assert_eq!(p.page, 0); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /src/pages/chart/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | pub mod home; 20 | pub mod search; 21 | pub use home::EXPLORE; 22 | pub use home::REPO_INFO; 23 | pub use home::SEARCH_BAR; 24 | pub use search::SEARCH_RESULTS; 25 | 26 | pub use super::{ctx, TemplateFile, ERROR_KEY, PAGES, PAYLOAD_KEY, TITLE_KEY}; 27 | 28 | pub fn register_templates(t: &mut tera::Tera) { 29 | EXPLORE.register(t).expect(EXPLORE.name); 30 | REPO_INFO.register(t).expect(REPO_INFO.name); 31 | SEARCH_BAR.register(t).expect(SEARCH_BAR.name); 32 | SEARCH_RESULTS.register(t).expect(SEARCH_RESULTS.name); 33 | } 34 | 35 | pub fn services(cfg: &mut actix_web::web::ServiceConfig) { 36 | home::services(cfg); 37 | search::services(cfg); 38 | } 39 | -------------------------------------------------------------------------------- /src/pages/chart/search.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::http::header::ContentType; 19 | use actix_web::{HttpResponse, Responder}; 20 | use actix_web_codegen_const_routes::post; 21 | use serde::{Deserialize, Serialize}; 22 | use std::cell::RefCell; 23 | use tera::Context; 24 | 25 | use db_core::prelude::*; 26 | 27 | use crate::errors::ServiceResult; 28 | use crate::pages::errors::*; 29 | use crate::settings::Settings; 30 | use crate::*; 31 | 32 | pub use crate::pages::*; 33 | 34 | pub const TITLE: &str = "Search"; 35 | pub const SEARCH_QUERY_KEY: &str = "search_query"; 36 | pub const SEARCH_RESULTS: TemplateFile = 37 | TemplateFile::new("search_results", "pages/chart/search.html"); 38 | 39 | pub struct SearchPage { 40 | ctx: RefCell, 41 | } 42 | 43 | impl CtxError for SearchPage { 44 | fn with_error(&self, e: &ReadableError) -> String { 45 | self.ctx.borrow_mut().insert(ERROR_KEY, e); 46 | self.render() 47 | } 48 | } 49 | 50 | #[derive(Clone, Debug, PartialEq, Eq, Default, Deserialize, Serialize)] 51 | pub struct SearchPagePayload { 52 | pub repos: Vec, 53 | } 54 | 55 | impl SearchPage { 56 | fn new(settings: &Settings, payload: &SearchPagePayload, search_query: Option<&str>) -> Self { 57 | let ctx = RefCell::new(ctx(settings)); 58 | ctx.borrow_mut().insert(TITLE_KEY, TITLE); 59 | ctx.borrow_mut().insert(PAYLOAD_KEY, payload); 60 | if let Some(search_query) = search_query { 61 | ctx.borrow_mut().insert(SEARCH_QUERY_KEY, search_query); 62 | } 63 | Self { ctx } 64 | } 65 | 66 | pub fn render(&self) -> String { 67 | TEMPLATES 68 | .render(SEARCH_RESULTS.name, &self.ctx.borrow()) 69 | .unwrap() 70 | } 71 | 72 | pub fn page(s: &Settings, payload: &SearchPagePayload, search_query: Option<&str>) -> String { 73 | let p = Self::new(s, payload, search_query); 74 | p.render() 75 | } 76 | } 77 | 78 | pub fn services(cfg: &mut web::ServiceConfig) { 79 | cfg.service(search); 80 | } 81 | 82 | #[post(path = "PAGES.search")] 83 | pub async fn search( 84 | payload: web::Form, 85 | ctx: WebCtx, 86 | db: WebDB, 87 | ) -> PageResult { 88 | async fn _search( 89 | ctx: &ArcCtx, 90 | db: &BoxDB, 91 | query: String, 92 | ) -> ServiceResult> { 93 | let responses = ctx.search_repository(db, query).await?; 94 | 95 | Ok(responses) 96 | } 97 | 98 | let query = payload.into_inner().query; 99 | let repos = _search(&ctx, &db, query.clone()).await.map_err(|e| { 100 | let x = SearchPagePayload::default(); 101 | PageError::new(SearchPage::new(&ctx.settings, &x, Some(&query)), e) 102 | })?; 103 | 104 | let payload = SearchPagePayload { repos }; 105 | let page = SearchPage::page(&ctx.settings, &payload, Some(&query)); 106 | 107 | let html = ContentType::html(); 108 | Ok(HttpResponse::Ok().content_type(html).body(page)) 109 | } 110 | -------------------------------------------------------------------------------- /src/pages/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use std::fmt; 19 | 20 | use actix_web::{ 21 | error::ResponseError, 22 | http::{header::ContentType, StatusCode}, 23 | HttpResponse, HttpResponseBuilder, 24 | }; 25 | use derive_more::Display; 26 | use derive_more::Error; 27 | use serde::*; 28 | 29 | use super::TemplateFile; 30 | use crate::errors::ServiceError; 31 | 32 | pub const ERROR_KEY: &str = "error"; 33 | 34 | pub const ERROR_TEMPLATE: TemplateFile = TemplateFile::new("error_comp", "components/error.html"); 35 | pub fn register_templates(t: &mut tera::Tera) { 36 | ERROR_TEMPLATE.register(t).expect(ERROR_TEMPLATE.name); 37 | } 38 | 39 | /// Render template with error context 40 | pub trait CtxError { 41 | fn with_error(&self, e: &ReadableError) -> String; 42 | } 43 | 44 | #[derive(Serialize, Debug, Display, Clone)] 45 | #[display(fmt = "title: {} reason: {}", title, reason)] 46 | pub struct ReadableError { 47 | pub reason: String, 48 | pub title: String, 49 | } 50 | 51 | impl ReadableError { 52 | pub fn new(e: &ServiceError) -> Self { 53 | let reason = format!("{}", e); 54 | let title = format!("{}", e.status_code()); 55 | 56 | Self { reason, title } 57 | } 58 | } 59 | 60 | #[derive(Error, Display)] 61 | #[display(fmt = "{}", readable)] 62 | pub struct PageError { 63 | #[error(not(source))] 64 | template: T, 65 | readable: ReadableError, 66 | #[error(not(source))] 67 | error: ServiceError, 68 | } 69 | 70 | impl fmt::Debug for PageError { 71 | #[cfg(not(tarpaulin_include))] 72 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 73 | f.debug_struct("PageError") 74 | .field("readable", &self.readable) 75 | .finish() 76 | } 77 | } 78 | 79 | impl PageError { 80 | /// create new instance of [PageError] from a template and an error 81 | pub fn new(template: T, error: ServiceError) -> Self { 82 | let readable = ReadableError::new(&error); 83 | Self { 84 | error, 85 | template, 86 | readable, 87 | } 88 | } 89 | } 90 | 91 | #[cfg(not(tarpaulin_include))] 92 | impl ResponseError for PageError { 93 | fn error_response(&self) -> HttpResponse { 94 | HttpResponseBuilder::new(self.status_code()) 95 | .content_type(ContentType::html()) 96 | .body(self.template.with_error(&self.readable)) 97 | } 98 | 99 | fn status_code(&self) -> StatusCode { 100 | self.error.status_code() 101 | } 102 | } 103 | 104 | /// Generic result data structure 105 | #[cfg(not(tarpaulin_include))] 106 | pub type PageResult = std::result::Result>; 107 | -------------------------------------------------------------------------------- /src/pages/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use actix_web::*; 19 | use lazy_static::lazy_static; 20 | use rust_embed::RustEmbed; 21 | use serde::*; 22 | use tera::*; 23 | 24 | use crate::settings::Settings; 25 | use crate::static_assets::ASSETS; 26 | use crate::{GIT_COMMIT_HASH, VERSION}; 27 | 28 | pub mod auth; 29 | pub mod chart; 30 | mod errors; 31 | pub mod routes; 32 | 33 | pub use errors::ERROR_KEY; 34 | pub use routes::PAGES; 35 | 36 | pub const TITLE_KEY: &str = "title"; 37 | 38 | pub struct TemplateFile { 39 | pub name: &'static str, 40 | pub path: &'static str, 41 | } 42 | 43 | impl TemplateFile { 44 | pub const fn new(name: &'static str, path: &'static str) -> Self { 45 | Self { name, path } 46 | } 47 | 48 | pub fn register(&self, t: &mut Tera) -> std::result::Result<(), tera::Error> { 49 | t.add_raw_template(self.name, &Templates::get_template(self).expect(self.name)) 50 | } 51 | 52 | #[cfg(test)] 53 | #[allow(dead_code)] 54 | pub fn register_from_file(&self, t: &mut Tera) -> std::result::Result<(), tera::Error> { 55 | use std::path::Path; 56 | t.add_template_file(Path::new("templates/").join(self.path), Some(self.name)) 57 | } 58 | } 59 | 60 | pub const PAYLOAD_KEY: &str = "payload"; 61 | 62 | pub const BASE: TemplateFile = TemplateFile::new("base", "components/base.html"); 63 | pub const FOOTER: TemplateFile = TemplateFile::new("footer", "components/footer.html"); 64 | pub const PUB_NAV: TemplateFile = TemplateFile::new("pub_nav", "components/nav/pub.html"); 65 | 66 | lazy_static! { 67 | pub static ref TEMPLATES: Tera = { 68 | let mut tera = Tera::default(); 69 | for t in [BASE, FOOTER, PUB_NAV ].iter() { 70 | t.register(&mut tera).unwrap(); 71 | } 72 | errors::register_templates(&mut tera); 73 | auth::register_templates(&mut tera); 74 | chart::register_templates(&mut tera); 75 | tera.autoescape_on(vec![".html", ".sql"]); 76 | //auth::register_templates(&mut tera); 77 | //gists::register_templates(&mut tera); 78 | tera 79 | }; 80 | } 81 | 82 | #[derive(RustEmbed)] 83 | #[folder = "templates/"] 84 | pub struct Templates; 85 | 86 | impl Templates { 87 | pub fn get_template(t: &TemplateFile) -> Option { 88 | match Self::get(t.path) { 89 | Some(file) => Some(String::from_utf8_lossy(&file.data).into_owned()), 90 | None => None, 91 | } 92 | } 93 | } 94 | 95 | pub fn ctx(s: &Settings) -> Context { 96 | let mut ctx = Context::new(); 97 | let footer = Footer::new(s); 98 | ctx.insert("footer", &footer); 99 | ctx.insert("page", &PAGES); 100 | ctx.insert("assets", &*ASSETS); 101 | ctx 102 | } 103 | 104 | #[derive(Serialize)] 105 | pub struct Footer<'a> { 106 | version: &'a str, 107 | admin_email: &'a str, 108 | source_code: &'a str, 109 | git_hash: &'a str, 110 | settings: &'a Settings, 111 | } 112 | 113 | impl<'a> Footer<'a> { 114 | pub fn new(settings: &'a Settings) -> Self { 115 | Self { 116 | version: VERSION, 117 | source_code: &settings.source_code, 118 | admin_email: &settings.admin_email, 119 | git_hash: &GIT_COMMIT_HASH[..8], 120 | settings, 121 | } 122 | } 123 | } 124 | 125 | pub fn services(cfg: &mut web::ServiceConfig) { 126 | auth::services(cfg); 127 | chart::services(cfg); 128 | } 129 | 130 | #[cfg(test)] 131 | mod tests { 132 | 133 | #[test] 134 | fn templates_work_basic() { 135 | use super::*; 136 | use tera::Tera; 137 | 138 | let mut tera = Tera::default(); 139 | let mut tera2 = Tera::default(); 140 | for t in [ 141 | BASE, 142 | FOOTER, 143 | PUB_NAV, 144 | auth::AUTH_CHALLENGE, 145 | auth::AUTH_ADD, 146 | chart::EXPLORE, 147 | // auth::AUTH_BASE, 148 | // auth::login::LOGIN, 149 | // auth::register::REGISTER, 150 | // errors::ERROR_TEMPLATE, 151 | // gists::GIST_BASE, 152 | // gists::GIST_EXPLORE, 153 | // gists::new::NEW_GIST, 154 | ] 155 | .iter() 156 | { 157 | t.register_from_file(&mut tera2).unwrap(); 158 | t.register(&mut tera).unwrap(); 159 | } 160 | } 161 | } 162 | 163 | //#[cfg(test)] 164 | //mod http_page_tests { 165 | // use actix_web::http::StatusCode; 166 | // use actix_web::test; 167 | // 168 | // use crate::ctx::Ctx; 169 | // use crate::db::BoxDB; 170 | // use crate::tests::*; 171 | // use crate::*; 172 | // 173 | // use super::PAGES; 174 | // 175 | // #[actix_rt::test] 176 | // async fn sqlite_templates_work() { 177 | // let (db, data, _federate, _tmp_dir) = sqlx_sqlite::get_ctx().await; 178 | // templates_work(data, db).await; 179 | // } 180 | // 181 | // async fn templates_work(data: ArcCtx, db: BoxDB) { 182 | // let app = get_app!(data, db).await; 183 | // 184 | // for file in [PAGES.auth.login, PAGES.auth.register].iter() { 185 | // let resp = get_request!(&app, file); 186 | // assert_eq!(resp.status(), StatusCode::OK); 187 | // } 188 | // } 189 | //} 190 | -------------------------------------------------------------------------------- /src/pages/routes.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright (C) 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use serde::Serialize; 19 | 20 | /// constant [Pages](Pages) instance 21 | pub const PAGES: Pages = Pages::new(); 22 | 23 | #[derive(Serialize)] 24 | /// Top-level routes data structure for V1 AP1 25 | pub struct Pages { 26 | /// home page 27 | pub home: &'static str, 28 | pub explore: &'static str, 29 | pub search: &'static str, 30 | /// auth routes 31 | pub auth: Auth, 32 | } 33 | 34 | impl Pages { 35 | /// create new instance of Routes 36 | const fn new() -> Pages { 37 | let explore = "/"; 38 | let home = explore; 39 | let search = "/search"; 40 | let auth = Auth::new(); 41 | Pages { 42 | home, 43 | auth, 44 | explore, 45 | search, 46 | } 47 | } 48 | 49 | pub fn explore_next(&self, page: u32) -> String { 50 | format!("{}?page={page}", self.explore) 51 | } 52 | } 53 | 54 | #[derive(Serialize)] 55 | /// Authentication routes 56 | pub struct Auth { 57 | /// logout route 58 | pub logout: &'static str, 59 | /// login route 60 | pub add: &'static str, 61 | 62 | /// verify route 63 | pub verify: &'static str, 64 | } 65 | 66 | impl Auth { 67 | /// create new instance of Authentication route 68 | pub const fn new() -> Auth { 69 | let add = "/add"; 70 | let logout = "/logout"; 71 | let verify = "/verify"; 72 | Auth { 73 | add, 74 | logout, 75 | verify, 76 | } 77 | } 78 | 79 | pub fn verify_get(&self, hostname: &str) -> String { 80 | format!("{}?hostname={hostname}", self.verify) 81 | } 82 | } 83 | 84 | //#[cfg(test)] 85 | //mod tests { 86 | // use super::*; 87 | // #[test] 88 | // fn gist_route_substitution_works() { 89 | // const NAME: &str = "bob"; 90 | // const GIST: &str = "foo"; 91 | // const FILE: &str = "README.md"; 92 | // let get_profile = format!("/~{NAME}"); 93 | // let view_gist = format!("/~{NAME}/{GIST}"); 94 | // let post_comment = format!("/~{NAME}/{GIST}/comment"); 95 | // let get_file = format!("/~{NAME}/{GIST}/contents/{FILE}"); 96 | // 97 | // let profile_component = GistProfilePathComponent { username: NAME }; 98 | // 99 | // assert_eq!(get_profile, PAGES.gist.get_profile_route(profile_component)); 100 | // 101 | // let profile_component = PostCommentPath { 102 | // username: NAME.into(), 103 | // gist: GIST.into(), 104 | // }; 105 | // 106 | // assert_eq!(view_gist, PAGES.gist.get_gist_route(&profile_component)); 107 | // 108 | // let post_comment_path = PostCommentPath { 109 | // gist: GIST.into(), 110 | // username: NAME.into(), 111 | // }; 112 | // 113 | // assert_eq!( 114 | // post_comment, 115 | // PAGES.gist.get_post_comment_route(&post_comment_path) 116 | // ); 117 | // 118 | // let file_component = GetFilePath { 119 | // username: NAME.into(), 120 | // gist: GIST.into(), 121 | // file: FILE.into(), 122 | // }; 123 | // assert_eq!(get_file, PAGES.gist.get_file_route(&file_component)); 124 | // } 125 | //} 126 | -------------------------------------------------------------------------------- /src/routes.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | pub fn services(cfg: &mut actix_web::web::ServiceConfig) { 20 | crate::api::services(cfg); 21 | crate::pages::services(cfg); 22 | crate::static_assets::services(cfg); 23 | } 24 | -------------------------------------------------------------------------------- /src/search.rs: -------------------------------------------------------------------------------- 1 | use crate::counter::AddSearch; 2 | use crate::master::{AddCounter, GetSite}; 3 | /* 4 | * ForgeFlux StarChart - A federated software forge spider 5 | * Copyright (C) 2022 Aravinth Manivannan 6 | * 7 | * This program is free software: you can redistribute it and/or modify 8 | * it under the terms of the GNU Affero General Public License as 9 | * published by the Free Software Foundation, either version 3 of the 10 | * License, or (at your option) any later version. 11 | * 12 | * This program is distributed in the hope that it will be useful, 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | * GNU Affero General Public License for more details. 16 | * 17 | * You should have received a copy of the GNU Affero General Public License 18 | * along with this program. If not, see . 19 | */ 20 | use crate::{counter, errors::*, WebCtx}; 21 | use actix_web::web; 22 | use actix_web::{HttpResponse, Responder}; 23 | use actix_web_codegen_const_routes::post; 24 | use db_core::prelude::*; 25 | use url::Url; 26 | 27 | use crate::Ctx; 28 | use crate::WebDB; 29 | 30 | pub use crate::api::{SearchRepositoryReq, ROUTES}; 31 | 32 | impl Ctx { 33 | async fn client_federated_search( 34 | &self, 35 | mut starchart_url: Url, 36 | payload: &SearchRepositoryReq, 37 | ) -> ServiceResult> { 38 | starchart_url.set_path(ROUTES.search.repository); 39 | Ok(self 40 | .client 41 | .post(starchart_url) 42 | .json(&payload) 43 | .send() 44 | .await 45 | .unwrap() 46 | .json() 47 | .await 48 | .unwrap()) 49 | } 50 | 51 | pub async fn search_repository( 52 | &self, 53 | db: &Box, 54 | query: String, 55 | ) -> ServiceResult> { 56 | let query = if query.contains('*') { 57 | query 58 | } else { 59 | format!("*{}*", query) 60 | }; 61 | let federated_search_payload = SearchRepositoryReq { 62 | query: query.clone(), 63 | }; 64 | let local_resp = db.search_repository(&query).await?; 65 | let mut federated_resp = Vec::default(); 66 | 67 | for starchart in db.search_mini_index(&query).await?.iter() { 68 | if db.is_starchart_imported(&Url::parse(starchart)?).await? { 69 | log::debug!("{starchart} is imported"); 70 | continue; 71 | } 72 | let addr = if let Some(addr) = self.master.send(GetSite(starchart.clone())).await? { 73 | addr 74 | } else { 75 | self.master 76 | .send(AddCounter { 77 | id: starchart.clone(), 78 | counter: counter::Count { 79 | duration: 54, 80 | search_threshold: 0, 81 | } 82 | .into(), 83 | }) 84 | .await?; 85 | self.master.send(GetSite(starchart.clone())).await?.unwrap() 86 | }; 87 | 88 | let count = addr.send(AddSearch).await?; 89 | if count > 50 { 90 | todo!("Clone index"); 91 | } else { 92 | let resp = self 93 | .client_federated_search(Url::parse(starchart)?, &federated_search_payload) 94 | .await?; 95 | federated_resp.extend(resp); 96 | } 97 | } 98 | 99 | federated_resp.extend(local_resp); 100 | Ok(federated_resp) 101 | } 102 | } 103 | 104 | #[post(path = "ROUTES.search.repository")] 105 | pub async fn search_repository( 106 | payload: web::Json, 107 | ctx: WebCtx, 108 | db: WebDB, 109 | ) -> ServiceResult { 110 | let resp = ctx 111 | .search_repository(&db, payload.into_inner().query) 112 | .await?; 113 | Ok(HttpResponse::Ok().json(resp)) 114 | } 115 | 116 | pub fn services(cfg: &mut web::ServiceConfig) { 117 | cfg.service(search_repository); 118 | } 119 | 120 | #[cfg(test)] 121 | mod tests { 122 | use actix_web::test; 123 | use url::Url; 124 | 125 | use super::*; 126 | use actix_web::http::StatusCode; 127 | 128 | use crate::tests::*; 129 | use crate::*; 130 | 131 | #[actix_rt::test] 132 | async fn search_works() { 133 | const URL: &str = "https://search-works-test.example.com"; 134 | const HTML_PROFILE_URL: &str = "https://search-works-test.example.com/user1"; 135 | const USERNAME: &str = "user1"; 136 | 137 | const REPO_NAME: &str = "searchsasdf2"; 138 | const HTML_REPO_URL: &str = "https://search-works-test.example.com/user1/searchsasdf2"; 139 | const TAGS: [&str; 3] = ["test", "starchart", "spider"]; 140 | 141 | let (db, ctx, federate, _tmpdir) = sqlx_sqlite::get_ctx().await; 142 | let app = get_app!(ctx, db, federate).await; 143 | 144 | let url = Url::parse(URL).unwrap(); 145 | 146 | let create_forge_msg = CreateForge { 147 | url: url.clone(), 148 | forge_type: ForgeImplementation::Gitea, 149 | starchart_url: None, 150 | }; 151 | 152 | let add_user_msg = AddUser { 153 | url: url.clone(), 154 | html_link: HTML_PROFILE_URL, 155 | profile_photo: None, 156 | username: USERNAME, 157 | import: false, 158 | }; 159 | 160 | let add_repo_msg = AddRepository { 161 | html_link: HTML_REPO_URL, 162 | name: REPO_NAME, 163 | tags: Some(TAGS.into()), 164 | owner: USERNAME, 165 | website: None, 166 | description: None, 167 | url, 168 | import: false, 169 | }; 170 | 171 | let _ = db.delete_forge_instance(&create_forge_msg.url).await; 172 | db.create_forge_instance(&create_forge_msg).await.unwrap(); 173 | assert!( 174 | db.forge_exists(&create_forge_msg.url).await.unwrap(), 175 | "forge creation failed, forge existence check failure" 176 | ); 177 | 178 | // add user 179 | db.add_user(&add_user_msg).await.unwrap(); 180 | // add repository 181 | db.create_repository(&add_repo_msg).await.unwrap(); 182 | // verify repo exists 183 | assert!(db 184 | .repository_exists(add_repo_msg.name, add_repo_msg.owner, &add_repo_msg.url) 185 | .await 186 | .unwrap()); 187 | 188 | // test starts 189 | 190 | let payload = SearchRepositoryReq { 191 | query: REPO_NAME[0..REPO_NAME.len() - 4].to_string(), 192 | }; 193 | let search_res_resp = test::call_service( 194 | &app, 195 | post_request!(&payload, ROUTES.search.repository).to_request(), 196 | ) 197 | .await; 198 | assert_eq!(search_res_resp.status(), StatusCode::OK); 199 | let search_res: Vec = test::read_body_json(search_res_resp).await; 200 | println!("{:?}", search_res); 201 | assert!(!search_res.is_empty()); 202 | assert_eq!(search_res.first().as_ref().unwrap().name, REPO_NAME); 203 | 204 | let mini_index_resp = get_request!(&app, ROUTES.introducer.get_mini_index); 205 | assert_eq!(mini_index_resp.status(), StatusCode::OK); 206 | let mini_index: api_routes::MiniIndex = test::read_body_json(mini_index_resp).await; 207 | assert!(!mini_index.mini_index.is_empty()); 208 | assert!(mini_index.mini_index.contains(USERNAME)); 209 | 210 | // test ends 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/static_assets/filemap.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use cache_buster::Files; 18 | 19 | pub struct FileMap { 20 | pub files: Files, 21 | } 22 | 23 | impl FileMap { 24 | #[allow(clippy::new_without_default)] 25 | pub fn new() -> Self { 26 | let map = include_str!("../cache_buster_data.json"); 27 | let files = Files::new(map); 28 | Self { files } 29 | } 30 | pub fn get(&self, path: impl AsRef) -> Option<&str> { 31 | let file_path = self.files.get_full_path(path); 32 | file_path.map(|file_path| &file_path[1..]) 33 | } 34 | } 35 | 36 | #[cfg(test)] 37 | mod tests { 38 | 39 | #[test] 40 | fn filemap_works() { 41 | let files = super::FileMap::new(); 42 | let css = files.get("./static/cache/css/main.css").unwrap(); 43 | println!("{}", css); 44 | assert!(css.contains("/assets/css/main")); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/static_assets/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use actix_web::*; 18 | 19 | pub mod filemap; 20 | pub mod static_files; 21 | 22 | pub use filemap::FileMap; 23 | pub use routes::{Assets, ASSETS}; 24 | 25 | pub fn services(cfg: &mut web::ServiceConfig) { 26 | cfg.service(static_files::static_files); 27 | } 28 | 29 | pub mod routes { 30 | use lazy_static::lazy_static; 31 | use serde::*; 32 | 33 | use super::*; 34 | 35 | lazy_static! { 36 | pub static ref ASSETS: Assets = Assets::default(); 37 | } 38 | 39 | #[derive(Serialize)] 40 | /// Top-level routes data structure for V1 AP1 41 | pub struct Assets { 42 | /// Authentication routes 43 | pub css: &'static str, 44 | } 45 | 46 | impl Default for Assets { 47 | /// create new instance of Routes 48 | fn default() -> Assets { 49 | Assets { 50 | css: &static_files::assets::CSS, 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/static_assets/static_files.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2022 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::borrow::Cow; 18 | 19 | use actix_web::body::BoxBody; 20 | use actix_web::{get, http::header, web, HttpResponse, Responder}; 21 | use mime_guess::from_path; 22 | use rust_embed::RustEmbed; 23 | 24 | use crate::CACHE_AGE; 25 | 26 | pub mod assets { 27 | use crate::FILES; 28 | use lazy_static::lazy_static; 29 | 30 | lazy_static! { 31 | pub static ref CSS: &'static str = FILES.get("./static/cache/css/main.css").unwrap(); 32 | } 33 | } 34 | 35 | #[derive(RustEmbed)] 36 | #[folder = "assets/"] 37 | struct Asset; 38 | 39 | fn handle_assets(path: &str) -> HttpResponse { 40 | match Asset::get(path) { 41 | Some(content) => { 42 | let body: BoxBody = match content.data { 43 | Cow::Borrowed(bytes) => BoxBody::new(bytes), 44 | Cow::Owned(bytes) => BoxBody::new(bytes), 45 | }; 46 | 47 | HttpResponse::Ok() 48 | .insert_header(header::CacheControl(vec![ 49 | header::CacheDirective::Public, 50 | header::CacheDirective::Extension("immutable".into(), None), 51 | header::CacheDirective::MaxAge(CACHE_AGE), 52 | ])) 53 | .content_type(from_path(path).first_or_octet_stream().as_ref()) 54 | .body(body) 55 | } 56 | None => HttpResponse::NotFound().body("404 Not Found"), 57 | } 58 | } 59 | 60 | #[get("/assets/{_:.*}")] 61 | pub async fn static_files(path: web::Path) -> impl Responder { 62 | handle_assets(&path) 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use actix_web::http::StatusCode; 68 | use actix_web::test; 69 | 70 | use crate::tests::*; 71 | use crate::*; 72 | 73 | use super::assets::CSS; 74 | 75 | #[actix_rt::test] 76 | async fn static_assets_work() { 77 | let (db, ctx, federate, _tmpdir) = sqlx_sqlite::get_ctx().await; 78 | let app = get_app!(ctx, db, federate).await; 79 | 80 | let file = *CSS; 81 | let resp = get_request!(&app, file); 82 | assert_eq!(resp.status(), StatusCode::OK); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/utils.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | pub(crate) fn get_random(len: usize) -> String { 20 | use rand::{distributions::Alphanumeric, rngs::ThreadRng, thread_rng, Rng}; 21 | use std::iter; 22 | 23 | let mut rng: ThreadRng = thread_rng(); 24 | 25 | iter::repeat(()) 26 | .map(|()| rng.sample(Alphanumeric)) 27 | .map(char::from) 28 | .take(len) 29 | .collect::() 30 | } 31 | -------------------------------------------------------------------------------- /src/verify.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * ForgeFlux StarChart - A federated software forge spider 3 | * Copyright © 2022 Aravinth Manivannan 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU Affero General Public License as 7 | * published by the Free Software Foundation, either version 3 of the 8 | * License, or (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU Affero General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU Affero General Public License 16 | * along with this program. If not, see . 17 | */ 18 | use serde::{Deserialize, Serialize}; 19 | use trust_dns_resolver::{ 20 | config::{ResolverConfig, ResolverOpts}, 21 | AsyncResolver, 22 | }; 23 | use url::Url; 24 | 25 | use crate::ArcCtx; 26 | 27 | #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] 28 | /// represents a DNS challenge 29 | pub struct Challenge { 30 | /// url of the forge instance 31 | pub url: String, 32 | /// key of TXT record 33 | pub key: String, 34 | /// value of TXT record 35 | pub value: String, 36 | } 37 | 38 | #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] 39 | pub struct TXTChallenge { 40 | pub key: String, 41 | pub value: String, 42 | } 43 | 44 | impl TXTChallenge { 45 | pub fn get_challenge_txt_key_prefix(ctx: &ArcCtx) -> String { 46 | // starchart-{{ starchart instance's hostname}}.{{ forge instance's hostname }} 47 | format!("starchart-{}", &ctx.settings.server.domain) 48 | } 49 | 50 | pub fn get_challenge_txt_key(ctx: &ArcCtx, hostname: &Url) -> String { 51 | format!( 52 | "{}.{}", 53 | Self::get_challenge_txt_key_prefix(ctx), 54 | hostname.host_str().unwrap() 55 | ) 56 | } 57 | 58 | pub fn new(ctx: &ArcCtx, hostname: &Url) -> Self { 59 | let key = Self::get_challenge_txt_key(ctx, hostname); 60 | let value = ctx.settings.server.domain.clone(); 61 | Self { key, value } 62 | } 63 | 64 | pub async fn verify_txt(&self) -> Result> { 65 | let conf = ResolverConfig::cloudflare_tls(); 66 | let opts = ResolverOpts::default(); 67 | let resolver = AsyncResolver::tokio(conf, opts)?; 68 | let res = resolver.txt_lookup(&self.key).await?; 69 | Ok(res.iter().any(|r| r.to_string() == self.value)) 70 | } 71 | } 72 | 73 | #[cfg(test)] 74 | pub mod tests { 75 | use super::*; 76 | use crate::tests::sqlx_sqlite; 77 | pub const BASE_DOMAIN: &str = "https://forge.forgeflux.org"; 78 | pub const VALUE: &str = "ifthisvalueisretrievedbyforgefluxstarchartthenthetestshouldpass"; 79 | 80 | #[actix_rt::test] 81 | async fn verify_txt_works() { 82 | // please note that this DNS record is in prod 83 | 84 | let (_db, ctx, _federate, _tmp_dir) = sqlx_sqlite::get_ctx().await; 85 | 86 | let base_hostname = Url::parse(BASE_DOMAIN).unwrap(); 87 | 88 | let key = TXTChallenge::get_challenge_txt_key(&ctx, &base_hostname); 89 | let mut txt_challenge = TXTChallenge { 90 | value: VALUE.to_string(), 91 | key: key.clone(), 92 | }; 93 | assert_eq!( 94 | TXTChallenge::get_challenge_txt_key(&ctx, &base_hostname), 95 | key, 96 | ); 97 | 98 | assert!( 99 | txt_challenge.verify_txt().await.unwrap(), 100 | "TXT Challenge verification test" 101 | ); 102 | txt_challenge.value = key; 103 | assert!(!txt_challenge.verify_txt().await.unwrap()); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /static/cache/css/main.css: -------------------------------------------------------------------------------- 1 | * { 2 | margin: 0; 3 | padding: 0; 4 | } 5 | 6 | html { 7 | font-family: Georgia, "Times New Roman", Times, serif; 8 | color: #333; 9 | } 10 | 11 | h1 { 12 | letter-spacing: 0.3rem; 13 | font-size: 3rem; 14 | font-weight: 500; 15 | font-family: Arial, Helvetica, sans-serif; 16 | } 17 | 18 | h2 { 19 | font-size: 3rem; 20 | font-weight: 500; 21 | font-family: Arial, Helvetica, sans-serif; 22 | } 23 | 24 | body { 25 | width: 100%; 26 | min-height: 100vh; 27 | display: flex; 28 | flex-direction: column; 29 | justify-content: space-between; 30 | } 31 | 32 | .nav__container { 33 | margin-top: 5px; 34 | display: flex; 35 | flex-direction: row; 36 | 37 | box-sizing: border-box; 38 | width: 100%; 39 | align-items: center; 40 | height: 20px; 41 | } 42 | 43 | .nav__home-btn { 44 | font-family: monospace, monospace; 45 | font-weight: 500; 46 | margin: auto; 47 | margin-left: 5px; 48 | font-size: 1rem; 49 | letter-spacing: 0.1rem; 50 | } 51 | 52 | a:hover { 53 | color: rgb(0, 86, 179); 54 | text-decoration: underline; 55 | } 56 | 57 | .nav__hamburger-menu { 58 | display: none; 59 | } 60 | 61 | .nav__spacer { 62 | flex: 3; 63 | margin: auto; 64 | } 65 | 66 | .nav__logo-container { 67 | display: inline-flex; 68 | text-decoration: none; 69 | } 70 | 71 | .nav__toggle { 72 | display: none; 73 | } 74 | 75 | .nav__logo { 76 | display: inline-flex; 77 | margin: auto; 78 | padding: 5px; 79 | width: 40px; 80 | } 81 | 82 | .nav__link-group { 83 | list-style: none; 84 | display: flex; 85 | flex-direction: row; 86 | align-items: center; 87 | align-self: center; 88 | margin: auto; 89 | text-align: center; 90 | } 91 | 92 | .nav__link-container { 93 | display: flex; 94 | padding: 0 10px; 95 | height: 100%; 96 | } 97 | 98 | .nav__link { 99 | text-decoration: none; 100 | margin: 0 5px; 101 | } 102 | 103 | a { 104 | text-decoration: none; 105 | } 106 | 107 | a, 108 | a:visited { 109 | color: rgb(0, 86, 179); 110 | } 111 | 112 | main { 113 | flex: 4; 114 | width: 100%; 115 | margin: auto; 116 | display: flex; 117 | align-items: center; 118 | flex-direction: column; 119 | } 120 | 121 | .main { 122 | min-height: 80vh; 123 | width: 65%; 124 | align-items: center; 125 | display: flex; 126 | flex-direction: column; 127 | margin: auto; 128 | justify-content: center; 129 | } 130 | 131 | form { 132 | margin-top: 20px; 133 | width: 80%; 134 | } 135 | 136 | .search__bar { 137 | height: 1.6rem; 138 | width: 70%; 139 | padding: 0; 140 | margin: 0 auto; 141 | display: flex; 142 | } 143 | 144 | #search { 145 | width: 100%; 146 | margin: auto; 147 | height: 1.3rem; 148 | } 149 | 150 | #search:placeholder-shown { 151 | margin: auto; 152 | padding-left: 5px; 153 | display: inline-block; 154 | font-size: 0.8rem; 155 | } 156 | 157 | .search__button { 158 | position: relative; 159 | left: -30px; 160 | background: none; 161 | border: none; 162 | align-items: center; 163 | display: flex; 164 | } 165 | 166 | .search__icon { 167 | margin: auto; 168 | filter: invert(47%) sepia(1%) saturate(0%) hue-rotate(278deg) brightness(92%) 169 | contrast(88%); 170 | height: 55%; 171 | } 172 | 173 | .search__icon:hover { 174 | cursor: pointer; 175 | } 176 | 177 | footer { 178 | display: block; 179 | font-size: 0.7rem; 180 | margin-bottom: 5px; 181 | } 182 | 183 | .footer__container { 184 | width: 90%; 185 | justify-content: space-between; 186 | margin: auto; 187 | display: flex; 188 | flex-direction: row; 189 | } 190 | 191 | .footer__column { 192 | list-style: none; 193 | display: flex; 194 | margin: auto 50px; 195 | } 196 | 197 | .footer__link-container { 198 | margin: 5px; 199 | } 200 | .license__conatiner { 201 | display: flex; 202 | } 203 | 204 | .footer__link { 205 | text-decoration: none; 206 | padding: 0 10px; 207 | } 208 | 209 | .footer__column-divider, 210 | .footer__column-divider--mobile-visible { 211 | font-weight: 500; 212 | opacity: 0.7; 213 | margin: 0 5px; 214 | } 215 | 216 | .footer__icon { 217 | margin: auto 5px; 218 | height: 20px; 219 | } 220 | 221 | /* Inline #1 | http://localhost:7000/add */ 222 | 223 | #hostname { 224 | width: 100%; 225 | display: block; 226 | } 227 | 228 | form > input { 229 | display: block; 230 | width: 100%; 231 | margin: 10px 0px; 232 | height: 30px; 233 | } 234 | 235 | form > label { 236 | /* display: none; */ 237 | display: block; 238 | width: 100%; 239 | } 240 | 241 | button { 242 | display: block; 243 | width: 90px; 244 | height: 30px; 245 | align-self: left; 246 | border-radius: none; 247 | background: rgb(0, 86, 179) none repeat scroll 0% 0%; 248 | color: #fff; 249 | border: none; 250 | align-self: left; 251 | border-radius: none; 252 | background: rgb(0, 86, 179); 253 | color: #fff; 254 | border-radius: none; 255 | border: none; 256 | } 257 | 258 | /* Inline #1 | http://localhost:7000/ */ 259 | 260 | .repository__container { 261 | width: 100%; 262 | margin: 20px auto; 263 | display: flex; 264 | flex-direction: column; 265 | justify-content: center; 266 | border-bottom: 1px grey dotted; 267 | } 268 | 269 | .repository__tags { 270 | } 271 | 272 | .repository_tags > a { 273 | background: lightgreen; 274 | } 275 | 276 | .repository__tags > a { 277 | background: lightgreen; 278 | margin: 15px 2px; 279 | padding: 1px; 280 | border-radius: 5px; 281 | } 282 | 283 | .search__bar { 284 | height: 1.6rem; 285 | width: 70%; 286 | padding: 0; 287 | margin: 0 auto; 288 | display: flex; 289 | } 290 | 291 | #search { 292 | width: 100%; 293 | margin: auto; 294 | height: 1.3rem; 295 | } 296 | 297 | #search:placeholder-shown { 298 | margin: auto; 299 | padding-left: 5px; 300 | display: inline-block; 301 | font-size: 0.8rem; 302 | } 303 | 304 | .search__button { 305 | position: relative; 306 | left: -30px; 307 | background: none; 308 | border: none; 309 | align-items: center; 310 | display: flex; 311 | } 312 | 313 | .search__icon { 314 | margin: auto; 315 | filter: invert(47%) sepia(1%) saturate(0%) hue-rotate(278deg) brightness(92%) 316 | contrast(88%); 317 | height: 55%; 318 | } 319 | 320 | .search__icon:hover { 321 | cursor: pointer; 322 | } 323 | -------------------------------------------------------------------------------- /templates/components/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {% block title %} {% endblock %} | Starchart 8 | 9 | 10 |
{% block nav %} {% endblock %}
11 |
{% block main %} {% endblock %}
12 | {% include "footer" %} 13 | 14 | 15 | -------------------------------------------------------------------------------- /templates/components/error.html: -------------------------------------------------------------------------------- 1 | {% if error %} 2 |
3 |

ERROR: {{ error.title }}

4 |

{{ error.reason }}

5 |
6 | {% endif %} 7 | -------------------------------------------------------------------------------- /templates/components/footer.html: -------------------------------------------------------------------------------- 1 | 47 | -------------------------------------------------------------------------------- /templates/components/nav/base.html: -------------------------------------------------------------------------------- 1 | 19 | -------------------------------------------------------------------------------- /templates/components/nav/pub.html: -------------------------------------------------------------------------------- 1 | 23 | -------------------------------------------------------------------------------- /templates/components/nav/search.html: -------------------------------------------------------------------------------- 1 |
2 | 14 | 17 |
18 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/forgeflux-org/starchart/64daf6eb2f06912f099f8ad9e2cdef258a65689e/templates/index.html -------------------------------------------------------------------------------- /templates/pages/auth/add.html: -------------------------------------------------------------------------------- 1 | {% extends 'base' %} 2 | {% block title %} {{ title }} {% endblock %} 3 | {% block nav %} {% include "pub_nav" %} {% endblock %} 4 | 5 | {% block main %} 6 |
7 |

Add forge instance for spidering

8 |

Please not that only forge administratior or parties with access the forge's DNS server can register for spidering

9 |
10 | {% include "error_comp" %} 11 | 13 | 14 | 15 | 16 |
17 |
18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /templates/pages/auth/challenge.html: -------------------------------------------------------------------------------- 1 | {% extends 'base' %} 2 | {% block title %} {{ title }} {% endblock %} 3 | {% block nav %} {% include "pub_nav" %} {% endblock %} 4 | 5 | {% block main %} 6 |
7 |

Configure DNS To Prove ownership

8 |
    9 |
  1. 10 |

    Create the following record on DNS
    11 | TXT {{ payload.key }} {{ payload.value }} 12 |

    13 |
    2. 14 |
  2. 15 |

    Click verify to verify challenge 16 |

    17 | 18 | 19 |
    20 |

    21 |
    3. 22 |
23 |
24 | {% endblock %} 25 | -------------------------------------------------------------------------------- /templates/pages/chart/components/repo_info.html: -------------------------------------------------------------------------------- 1 | {% set name = repository.html_url %} 2 |
3 | 4 |

{{ repository.html_url }}

5 | 6 | {% if repository.description %} 7 |

{{ repository.description }}

8 | {% endif %} 9 | 10 |
11 | {% if repository.tags %} 12 | {% for tag in repository.tags %} 13 | {{ tag }} 14 | {% endfor %} 15 | {% endif %} 16 |
17 | 18 | 21 | {% if repository.website %} 22 | Homeage 23 | {% endif %} 24 |
25 | -------------------------------------------------------------------------------- /templates/pages/chart/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base' %} 2 | {% block title %} {{ title }} {% endblock %} 3 | {% block nav %} {% include "pub_nav" %} {% endblock %} 4 | 5 | {% block main %} 6 |
7 | {% for repository in payload.repos %} 8 | {% include "repo_info" %} 9 | {% endfor %} 10 | 11 | 12 |
13 | Back 14 | Next 15 |
16 | 17 |
18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /templates/pages/chart/search.html: -------------------------------------------------------------------------------- 1 | {% extends 'base' %} 2 | {% block title %} {{ title }} {% endblock %} 3 | {% block nav %} {% include "pub_nav" %} {% endblock %} 4 | 5 | {% block main %} 6 |
7 | {% for repository in payload.repos %} 8 | {% include "repo_info" %} 9 | {% endfor %} 10 |
11 | {% endblock %} 12 | -------------------------------------------------------------------------------- /utils/cache-bust/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | src/cache_buster_data.json 3 | src/cache_buster_data.json 4 | -------------------------------------------------------------------------------- /utils/cache-bust/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cache-bust" 3 | version = "0.1.0" 4 | edition = "2021" 5 | description = "ForgeFlux StarChart - Federated forge spider" 6 | homepage = "https://mcaptcha.org" 7 | license = "AGPLv3 or later version" 8 | authors = ["realaravinth "] 9 | repository = "https://github.com/forgeflux-org/starchart" 10 | documentation = "https://forgeflux.org/" 11 | default-run = "cache-bust" 12 | 13 | [[bin]] 14 | name = "cache-bust" 15 | path = "./src/main.rs" 16 | 17 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 18 | 19 | [workspace] 20 | 21 | [dependencies] 22 | cache-buster = { version = "0.2.0", git = "https://github.com/realaravinth/cache-buster" } 23 | serde = { version = "1", features = ["derive"] } 24 | serde_json = "1" 25 | -------------------------------------------------------------------------------- /utils/cache-bust/src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Aravinth Manivannan 3 | * 4 | * This program is free software: you can redistribute it and/or modify 5 | * it under the terms of the GNU Affero General Public License as 6 | * published by the Free Software Foundation, either version 3 of the 7 | * License, or (at your option) any later version. 8 | * 9 | * This program is distributed in the hope that it will be useful, 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | * GNU Affero General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU Affero General Public License 15 | * along with this program. If not, see . 16 | */ 17 | use std::fs; 18 | use std::path::Path; 19 | use std::collections::HashMap; 20 | 21 | use cache_buster::{BusterBuilder, CACHE_BUSTER_DATA_FILE, NoHashCategory}; 22 | use serde::{Serialize, Deserialize}; 23 | 24 | #[derive(Deserialize, Serialize)] 25 | struct FileMap { 26 | map: HashMap, 27 | base_dir: String, 28 | } 29 | 30 | fn main() { 31 | cache_bust(); 32 | process_file_map(); 33 | } 34 | 35 | fn cache_bust() { 36 | // until APPLICATION_WASM gets added to mime crate 37 | // PR: https://github.com/hyperium/mime/pull/138 38 | // let types = vec![ 39 | // mime::IMAGE_PNG, 40 | // mime::IMAGE_SVG, 41 | // mime::IMAGE_JPEG, 42 | // mime::IMAGE_GIF, 43 | // mime::APPLICATION_JAVASCRIPT, 44 | // mime::TEXT_CSS, 45 | // ]; 46 | 47 | println!("[*] Cache busting"); 48 | let no_hash = vec![NoHashCategory::FileExtentions(vec!["wasm"])]; 49 | 50 | let config = BusterBuilder::default() 51 | .source("../../static/cache/") 52 | .result("./../../assets") 53 | .no_hash(no_hash) 54 | .follow_links(true) 55 | .build() 56 | .unwrap(); 57 | 58 | config.process().unwrap(); 59 | } 60 | 61 | fn process_file_map() { 62 | let contents = fs::read_to_string(CACHE_BUSTER_DATA_FILE).unwrap(); 63 | let files: FileMap = serde_json::from_str(&contents).unwrap(); 64 | let mut map = HashMap::with_capacity(files.map.len()); 65 | for (k, v) in files.map.iter() { 66 | map.insert(k.strip_prefix("../.").unwrap().to_owned(), 67 | v.strip_prefix("./../.").unwrap().to_owned() 68 | ); 69 | } 70 | 71 | let new_filemap = FileMap{ 72 | map, 73 | base_dir: files.base_dir.strip_prefix("./../.").unwrap().to_owned(), 74 | }; 75 | 76 | let dest = Path::new("../../").join(CACHE_BUSTER_DATA_FILE); 77 | fs::write(&dest, serde_json::to_string(&new_filemap).unwrap()).unwrap(); 78 | } 79 | --------------------------------------------------------------------------------