├── xtask ├── .gitignore └── Cargo.toml ├── rfd-cli ├── .gitignore ├── src │ ├── generated │ │ └── mod.rs │ ├── cmd │ │ ├── mod.rs │ │ ├── auth │ │ │ ├── mod.rs │ │ │ ├── link.rs │ │ │ ├── oauth.rs │ │ │ └── login.rs │ │ ├── shortcut │ │ │ ├── mod.rs │ │ │ ├── mapper.rs │ │ │ └── access.rs │ │ └── config │ │ │ └── mod.rs │ ├── store │ │ └── mod.rs │ ├── err.rs │ └── printer │ │ └── json.rs └── Cargo.toml ├── rfd-model ├── .gitignore ├── migrations │ ├── 2023-01-20-202736_rfd │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-07-23-145951_job │ │ ├── down.sql │ │ └── up.sql │ ├── 2024-04-19-173349_add_jobs_started_index │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-11-02-161428_lock_job │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-01-20-204405_rfd_pdf │ │ ├── down.sql │ │ └── up.sql │ ├── 2024-02-28_add_labels_to_rfd_revision │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-01-20-203139_rfd_revision │ │ ├── down.sql │ │ └── up.sql │ ├── 2025-08-14-162335_rfd_revision_major_change │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-10-06-001455_add_rfd_visibility │ │ ├── down.sql │ │ └── up.sql │ ├── 2025-01-02-190923_add_commited_at_index │ │ ├── down.sql │ │ └── up.sql │ ├── 2023-11-01-171512_add_pdf_external_id │ │ ├── down.sql │ │ └── up.sql │ ├── 2024-04-12-205749_fix_revision_index │ │ ├── down.sql │ │ └── up.sql │ ├── 2024-11-25-175556_fix-revision-idx │ │ ├── down.sql │ │ └── up.sql │ ├── 00000000000000_diesel_initial_setup │ │ ├── down.sql │ │ └── up.sql │ └── 2024-11-12-141610_v_api_conversion │ │ ├── down.sql │ │ └── up.sql ├── diesel-schema.patch ├── diesel.toml ├── Cargo.toml └── src │ └── schema.rs ├── rfd-sdk ├── .gitignore ├── src │ └── generated │ │ └── mod.rs └── Cargo.toml ├── trace-request ├── .gitignore ├── Cargo.toml ├── tests │ └── test.rs └── src │ └── lib.rs ├── .cargo └── config.toml ├── dropshot-authorization-header ├── .gitignore ├── src │ ├── lib.rs │ ├── bearer.rs │ └── basic.rs └── Cargo.toml ├── parse-rfd ├── .gitignore ├── parser │ ├── package.json │ └── index.js └── Cargo.toml ├── rfd-processor ├── tests │ └── ref │ │ └── asciidoc_to_pdf.pdf ├── src │ ├── pdf.rs │ ├── updater │ │ ├── ensure_default_state.rs │ │ ├── update_search_index.rs │ │ ├── copy_images_to_storage.rs │ │ ├── update_discussion_url.rs │ │ ├── create_pull_request.rs │ │ ├── process_includes.rs │ │ ├── ensure_pr_state.rs │ │ └── update_pdfs.rs │ ├── content │ │ └── asciidoc.rs │ ├── scanner.rs │ ├── util.rs │ ├── processor.rs │ ├── main.rs │ └── search │ │ └── mod.rs ├── Cargo.toml └── config.example.toml ├── .gitignore ├── rfd-api ├── src │ ├── endpoints │ │ ├── mod.rs │ │ └── job.rs │ ├── caller.rs │ ├── util.rs │ ├── secrets.rs │ ├── error.rs │ ├── magic_link.rs │ ├── permissions.rs │ ├── config.rs │ ├── initial_data.rs │ └── server.rs ├── mappers.example.toml ├── Cargo.toml └── config.example.toml ├── .github ├── renovate.json └── workflows │ ├── license-check.yml │ ├── fmt.yml │ ├── build.yml │ ├── test.yml │ └── release.yml ├── rfd-ts ├── README.md ├── tsconfig.json ├── package.json └── src │ ├── retry.ts │ ├── util.ts │ └── http-client.ts ├── rfd-data ├── Cargo.toml └── src │ ├── content │ └── template.rs │ └── lib.rs ├── dprint.json ├── remix-auth-rfd ├── tsconfig.json ├── src │ ├── index.ts │ ├── util.ts │ └── oauth.ts └── package.json ├── .licenserc.yaml ├── rfd-installer ├── Cargo.toml └── src │ ├── main.rs │ └── lib.rs ├── rfd-github ├── Cargo.toml └── src │ ├── util.rs │ └── ext.rs ├── Cargo.toml └── SETUP.md /xtask/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /rfd-cli/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | config.toml -------------------------------------------------------------------------------- /rfd-cli/src/generated/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod cli; 2 | -------------------------------------------------------------------------------- /rfd-model/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /rfd-sdk/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /rfd-sdk/src/generated/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod sdk; 2 | -------------------------------------------------------------------------------- /trace-request/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [alias] 2 | xtask = "run --package xtask --" 3 | -------------------------------------------------------------------------------- /dropshot-authorization-header/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-202736_rfd/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE rfd -------------------------------------------------------------------------------- /rfd-model/migrations/2023-07-23-145951_job/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS job; -------------------------------------------------------------------------------- /parse-rfd/.gitignore: -------------------------------------------------------------------------------- 1 | parser/dist/css 2 | !parser/package.json 3 | !parser/package-lock.json -------------------------------------------------------------------------------- /rfd-model/migrations/2024-04-19-173349_add_jobs_started_index/down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX jobs_started; -------------------------------------------------------------------------------- /rfd-model/migrations/2023-11-02-161428_lock_job/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE job DROP COLUMN started_at; 2 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-204405_rfd_pdf/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE rfd_pdf; 2 | DROP TYPE RFD_PDF_SOURCE; -------------------------------------------------------------------------------- /rfd-model/migrations/2023-11-02-161428_lock_job/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE job ADD COLUMN started_at TIMESTAMPTZ; 2 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-02-28_add_labels_to_rfd_revision/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd_revision DROP COLUMN labels; -------------------------------------------------------------------------------- /rfd-model/migrations/2024-02-28_add_labels_to_rfd_revision/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd_revision ADD COLUMN labels VARCHAR; -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-203139_rfd_revision/down.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE rfd_revision; 2 | DROP TYPE RFD_CONTENT_FORMAT; 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2025-08-14-162335_rfd_revision_major_change/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd_revision DROP COLUMN major_change; 2 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-10-06-001455_add_rfd_visibility/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd DROP COLUMN visibility; 2 | 3 | DROP TYPE RFD_VISIBILITY; -------------------------------------------------------------------------------- /rfd-processor/tests/ref/asciidoc_to_pdf.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oxidecomputer/rfd-api/HEAD/rfd-processor/tests/ref/asciidoc_to_pdf.pdf -------------------------------------------------------------------------------- /rfd-model/migrations/2025-01-02-190923_add_commited_at_index/down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX rfd_revision_sort_no_content_idx; 2 | DROP INDEX rfd_number_idx; 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-11-01-171512_add_pdf_external_id/down.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd_pdf DROP COLUMN external_id; 2 | ALTER TABLE rfd_pdf DROP COLUMN rfd_id; 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-04-19-173349_add_jobs_started_index/up.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX jobs_started ON job (id, started_at, processed ASC, committed_at ASC, created_at ASC); -------------------------------------------------------------------------------- /rfd-model/migrations/2024-04-12-205749_fix_revision_index/down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX rfd_revision_commit_sha_idx; 2 | CREATE UNIQUE INDEX rfd_revision_sha_idx ON rfd_revision (rfd_id, sha); 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-04-12-205749_fix_revision_index/up.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX rfd_revision_sha_idx; 2 | CREATE UNIQUE INDEX rfd_revision_commit_sha_idx ON rfd_revision (rfd_id, commit_sha); 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-11-25-175556_fix-revision-idx/down.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX rfd_revision_commit_sha_idx; 2 | CREATE UNIQUE INDEX rfd_revision_commit_sha_idx ON rfd_revision (rfd_id, commit_sha); 3 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-11-01-171512_add_pdf_external_id/up.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE rfd_pdf ADD COLUMN rfd_id UUID REFERENCES rfd (id) NOT NULL; 2 | ALTER TABLE rfd_pdf ADD COLUMN external_id VARCHAR NOT NULL; 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | node_modules 4 | dist 5 | 6 | config.toml 7 | mappers.toml 8 | !.cargo/config.toml 9 | 10 | spec.toml 11 | key.json 12 | test-key.json 13 | gcp.json 14 | 15 | node_modules 16 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-10-06-001455_add_rfd_visibility/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE RFD_VISIBILITY as ENUM('public', 'private'); 2 | 3 | ALTER TABLE rfd ADD COLUMN visibility RFD_VISIBILITY NOT NULL DEFAULT 'private'; 4 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-11-25-175556_fix-revision-idx/up.sql: -------------------------------------------------------------------------------- 1 | DROP INDEX rfd_revision_commit_sha_idx; 2 | CREATE UNIQUE INDEX rfd_revision_commit_sha_idx ON rfd_revision (rfd_id, commit_sha) WHERE deleted_at IS NULL; 3 | -------------------------------------------------------------------------------- /parse-rfd/parser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "build": "ncc build index -o dist" 4 | }, 5 | "dependencies": { 6 | "@asciidoctor/core": "^2.2.6", 7 | "@vercel/ncc": "^0.38.0", 8 | "html-to-text": "^9.0.2" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-202736_rfd/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE rfd ( 2 | id UUID PRIMARY KEY, 3 | rfd_number INTEGER NOT NULL, 4 | link VARCHAR, 5 | created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 6 | updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 7 | deleted_at TIMESTAMPTZ 8 | ) -------------------------------------------------------------------------------- /dropshot-authorization-header/src/lib.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | pub mod basic; 6 | pub mod bearer; 7 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | pub mod auth; 6 | pub mod config; 7 | pub mod shortcut; 8 | -------------------------------------------------------------------------------- /rfd-model/migrations/2025-08-14-162335_rfd_revision_major_change/up.sql: -------------------------------------------------------------------------------- 1 | -- Adding the column in three steps to ensure all rows have a default. 2 | ALTER TABLE rfd_revision ADD COLUMN major_change BOOLEAN; 3 | UPDATE rfd_revision SET major_change = TRUE; 4 | ALTER TABLE rfd_revision ALTER COLUMN major_change SET NOT NULL; 5 | -------------------------------------------------------------------------------- /parse-rfd/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "parse-rfd" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | serde = { workspace = true } 10 | serde_json = { workspace = true } 11 | uuid = { workspace = true } -------------------------------------------------------------------------------- /rfd-model/migrations/2025-01-02-190923_add_commited_at_index/up.sql: -------------------------------------------------------------------------------- 1 | CREATE INDEX rfd_revision_sort_no_content_idx ON rfd_revision (rfd_id, committed_at desc) include(id, title, state, discussion, authors, content_format, sha, commit_sha, created_at, updated_at, deleted_at, labels); 2 | CREATE INDEX rfd_number_idx ON rfd (rfd_number); 3 | -------------------------------------------------------------------------------- /rfd-api/src/endpoints/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | pub static UNLIMITED: i64 = 9999999; 6 | 7 | pub mod job; 8 | pub mod rfd; 9 | pub mod webhook; 10 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "local>oxidecomputer/renovate-config", 5 | "local>oxidecomputer/renovate-config//rust/autocreate", 6 | "helpers:pinGitHubActionDigests" 7 | ], 8 | "ignorePaths": [ 9 | ".github/workflows/release.yml" 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /.github/workflows/license-check.yml: -------------------------------------------------------------------------------- 1 | name: license-check 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branch: 7 | - main 8 | 9 | jobs: 10 | license: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | - name: Check License Header 15 | uses: apache/skywalking-eyes/header@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 16 | -------------------------------------------------------------------------------- /rfd-ts/README.md: -------------------------------------------------------------------------------- 1 | # @oxide/rfd-ts 2 | 3 | Under development library for interacting with the RFD API. 4 | 5 | ## Installation 6 | 7 | ``` 8 | npm install --save @oxide/rfd.ts 9 | ``` 10 | 11 | ## Development 12 | 13 | This library is generated by [@oxide/openapi-gen-ts](https://www.npmjs.com/package/@oxide/openapi-gen-ts) based on [rfd-api-spec.json](../rfd-api-spec.json). 14 | -------------------------------------------------------------------------------- /trace-request/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "trace-request" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | proc-macro = true 8 | 9 | [dependencies] 10 | proc-macro2 = "1" 11 | quote = "1" 12 | syn = { version = "2.0", features = ["full"] } 13 | 14 | [dev-dependencies] 15 | dropshot = { workspace = true } 16 | http = { workspace = true } 17 | tracing = { workspace = true } 18 | -------------------------------------------------------------------------------- /rfd-model/migrations/00000000000000_diesel_initial_setup/down.sql: -------------------------------------------------------------------------------- 1 | -- This file was automatically created by Diesel to setup helper functions 2 | -- and other internal bookkeeping. This file is safe to edit, any future 3 | -- changes will be added to existing projects as new migrations. 4 | 5 | DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); 6 | DROP FUNCTION IF EXISTS diesel_set_updated_at(); 7 | -------------------------------------------------------------------------------- /dropshot-authorization-header/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dropshot-authorization-header" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | async-trait = { workspace = true } 10 | base64 = { workspace = true } 11 | dropshot = { workspace = true } 12 | http = { workspace = true } 13 | tracing = { workspace = true } 14 | -------------------------------------------------------------------------------- /rfd-data/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-data" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | regex = { workspace = true } 10 | rfd-model = { path = "../rfd-model" } 11 | schemars = { workspace = true } 12 | serde = { workspace = true } 13 | thiserror = { workspace = true } 14 | tracing = { workspace = true } 15 | -------------------------------------------------------------------------------- /dprint.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript": { 3 | "indentWidth": 2, 4 | "semiColons": "asi", 5 | "quoteStyle": "preferSingle", 6 | "useBraces": "always" 7 | }, 8 | "json": { 9 | }, 10 | "excludes": [ 11 | "**/node_modules", 12 | "**/*-lock.json", 13 | "**/tests" 14 | ], 15 | "plugins": [ 16 | "https://plugins.dprint.dev/typescript-0.93.0.wasm", 17 | "https://plugins.dprint.dev/json-0.19.4.wasm" 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-11-12-141610_v_api_conversion/down.sql: -------------------------------------------------------------------------------- 1 | DROP TRIGGER set_updated_at ON mapper; 2 | ALTER TABLE mapper DROP COLUMN updated_at; 3 | 4 | DROP TRIGGER set_updated_at ON login_attempt; 5 | DROP TRIGGER set_updated_at ON api_user_access_token; 6 | DROP TRIGGER set_updated_at ON api_user_provider; 7 | DROP TRIGGER set_updated_at ON api_key; 8 | DROP TRIGGER set_updated_at ON api_user; 9 | DROP TRIGGER set_updated_at ON access_groups; 10 | -------------------------------------------------------------------------------- /rfd-ts/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "noEmit": true, 5 | "allowSyntheticDefaultImports": true, 6 | "esModuleInterop": true, 7 | "experimentalDecorators": true, 8 | "lib": ["es2022", "dom", "DOM.Iterable"], 9 | "moduleResolution": "Bundler", 10 | "resolveJsonModule": true, 11 | "skipLibCheck": true, 12 | "sourceMap": true, 13 | "strict": true, 14 | "target": "ES2022" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /remix-auth-rfd/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "noEmit": true, 5 | "allowSyntheticDefaultImports": true, 6 | "esModuleInterop": true, 7 | "experimentalDecorators": true, 8 | "lib": ["es2022", "dom", "DOM.Iterable"], 9 | "moduleResolution": "Bundler", 10 | "resolveJsonModule": true, 11 | "skipLibCheck": true, 12 | "sourceMap": true, 13 | "strict": true, 14 | "target": "ES2022" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.licenserc.yaml: -------------------------------------------------------------------------------- 1 | header: 2 | license: 3 | spdx-id: MPL-2.0 4 | 5 | content: | 6 | This Source Code Form is subject to the terms of the Mozilla Public 7 | License, v. 2.0. If a copy of the MPL was not distributed with this 8 | file, You can obtain one at https://mozilla.org/MPL/2.0/. 9 | paths: 10 | - '**/*.js' 11 | - '**/*.rs' 12 | paths-ignore: 13 | - '**/generated/*.rs' 14 | - 'parse-rfd/parser/dist/index.js' 15 | 16 | comment: on-failure -------------------------------------------------------------------------------- /xtask/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "xtask" 3 | version = "0.0.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | clap = { workspace = true } 8 | newline-converter = { workspace = true } 9 | progenitor = { workspace = true } 10 | regex = { workspace = true } 11 | rustfmt-wrapper = { workspace = true } 12 | semver = { workspace = true } 13 | serde = { workspace = true } 14 | serde_json = { workspace = true } 15 | similar = { workspace = true } 16 | 17 | [package.metadata.dist] 18 | dist = false -------------------------------------------------------------------------------- /rfd-api/mappers.example.toml: -------------------------------------------------------------------------------- 1 | # Initial data to be configured during startup 2 | 3 | # Create a group to assign the initial admin user to 4 | [[groups]] 5 | name = "admin" 6 | permissions = [ 7 | # List the permissions that the admin should have 8 | ] 9 | 10 | # Add the initial mapper that will assign a user with a specific email address to the admin group 11 | [[mappers]] 12 | name = "Initial admin" 13 | rule = "email_address" 14 | email = "user@hostname" 15 | groups = [ 16 | "admin" 17 | ] 18 | -------------------------------------------------------------------------------- /rfd-installer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-installer" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "rfd_installer" 8 | path = "src/lib.rs" 9 | 10 | [[bin]] 11 | name = "rfd-installer" 12 | path = "src/main.rs" 13 | 14 | [dependencies] 15 | diesel = { workspace = true, features = ["postgres", "r2d2"] } 16 | diesel_migrations = { workspace = true, features = ["postgres"] } 17 | v-api-installer = { workspace = true } 18 | 19 | [package.metadata.dist] 20 | targets = [] 21 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-204405_rfd_pdf/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE RFD_PDF_SOURCE as ENUM('github', 'google'); 2 | 3 | CREATE TABLE rfd_pdf ( 4 | id UUID PRIMARY KEY, 5 | rfd_revision_id UUID REFERENCES rfd_revision (id) NOT NULL, 6 | source RFD_PDF_SOURCE NOT NULL, 7 | link VARCHAR NOT NULL, 8 | 9 | created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 10 | updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 11 | deleted_at TIMESTAMPTZ, 12 | 13 | CONSTRAINT revision_links_unique UNIQUE (rfd_revision_id, source, link) 14 | ); -------------------------------------------------------------------------------- /rfd-sdk/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-sdk" 3 | version = "0.12.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | chrono = { workspace = true, features = ["serde"] } 10 | progenitor-client = { workspace = true } 11 | schemars = { workspace = true, features = ["chrono", "uuid1"] } 12 | serde = { workspace = true } 13 | serde_json = { workspace = true } 14 | reqwest = { workspace = true } 15 | uuid = { workspace = true } -------------------------------------------------------------------------------- /.github/workflows/fmt.yml: -------------------------------------------------------------------------------- 1 | name: Fmt 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | check_fmt: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 12 | - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 13 | with: 14 | toolchain: stable 15 | - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 16 | with: 17 | command: fmt 18 | args: -- --check 19 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-07-23-145951_job/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE job ( 2 | id SERIAL PRIMARY KEY, 3 | owner VARCHAR NOT NULL, 4 | repository VARCHAR NOT NULL, 5 | branch VARCHAR NOT NULL, 6 | sha VARCHAR NOT NULL, 7 | rfd INTEGER NOT NULL, 8 | webhook_delivery_id UUID, 9 | committed_at TIMESTAMPTZ NOT NULL, 10 | processed BOOLEAN NOT NULL, 11 | created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 12 | 13 | UNIQUE (sha, rfd) 14 | ); 15 | 16 | CREATE INDEX jobs ON job (id, processed ASC, committed_at ASC, created_at ASC); -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 12 | - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 13 | with: 14 | toolchain: stable 15 | - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 16 | with: 17 | command: build 18 | args: --release 19 | -------------------------------------------------------------------------------- /rfd-github/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-github" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | async-trait = { workspace = true } 10 | base64 = { workspace = true } 11 | chrono = { workspace = true } 12 | http = { workspace = true } 13 | octorust = { workspace = true } 14 | regex = { workspace = true } 15 | rfd-data = { path = "../rfd-data" } 16 | rfd-model = { path = "../rfd-model" } 17 | thiserror = { workspace = true } 18 | tracing = { workspace = true } -------------------------------------------------------------------------------- /trace-request/tests/test.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use dropshot::{endpoint, HttpError, HttpResponseOk, RequestContext}; 6 | use trace_request::trace_request; 7 | 8 | #[trace_request] 9 | #[endpoint { 10 | method = GET, 11 | path = "/test" 12 | }] 13 | async fn _trace_entry_exit(rqctx: RequestContext<()>) -> Result, HttpError> { 14 | Ok(HttpResponseOk(())) 15 | } 16 | -------------------------------------------------------------------------------- /rfd-installer/src/main.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use rfd_installer::run_migrations; 6 | 7 | fn main() { 8 | if let Ok(url) = std::env::var("DATABASE_URL") { 9 | run_migrations(&url, std::env::var("V_ONLY").is_ok()); 10 | } else { 11 | println!("DATABASE_URL environment variable must be specified to run migrations and must be in the form of a connection string") 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /rfd-model/diesel-schema.patch: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | diff --git a/src/schema.rs b/src/schema.rs 5 | index 8eb4b49..cbc1526 100644 6 | --- a/src/schema.rs 7 | +++ b/src/schema.rs 8 | @@ -1,4 +1,6 @@ 9 | -// @generated automatically by Diesel CLI. 10 | +// This Source Code Form is subject to the terms of the Mozilla Public 11 | +// License, v. 2.0. If a copy of the MPL was not distributed with this 12 | +// file, You can obtain one at https://mozilla.org/MPL/2.0/. 13 | 14 | pub mod sql_types { 15 | #[derive(diesel::sql_types::SqlType)] 16 | -------------------------------------------------------------------------------- /rfd-model/diesel.toml: -------------------------------------------------------------------------------- 1 | # For documentation on how to configure this file, 2 | # see https://diesel.rs/guides/configuring-diesel-cli 3 | 4 | [print_schema] 5 | file = "src/schema.rs" 6 | patch_file = "diesel-schema.patch" 7 | 8 | [print_schema.filter] 9 | except_tables = [ 10 | "access_groups", 11 | "api_key", 12 | "api_user", 13 | "api_user_access_token", 14 | "api_user_contact_email", 15 | "api_user_provider", 16 | "link_request", 17 | "login_attempt", 18 | "magic_link_attempt", 19 | "magic_link_client", 20 | "magic_link_client_redirect_uri", 21 | "magic_link_client_secret", 22 | "mapper", 23 | "oauth_client", 24 | "oauth_client_redirect_uri", 25 | "oauth_client_secret", 26 | ] 27 | -------------------------------------------------------------------------------- /rfd-model/migrations/2023-01-20-203139_rfd_revision/up.sql: -------------------------------------------------------------------------------- 1 | CREATE TYPE RFD_CONTENT_FORMAT as ENUM('asciidoc', 'markdown'); 2 | 3 | CREATE TABLE rfd_revision ( 4 | id UUID PRIMARY KEY, 5 | rfd_id UUID REFERENCES rfd (id) NOT NULL, 6 | title VARCHAR NOT NULL, 7 | state VARCHAR, 8 | discussion VARCHAR, 9 | authors VARCHAR, 10 | content VARCHAR NOT NULL, 11 | content_format RFD_CONTENT_FORMAT NOT NULL, 12 | sha VARCHAR NOT NULL, 13 | commit_sha VARCHAR NOT NULL, 14 | committed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 15 | 16 | created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 17 | updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), 18 | deleted_at TIMESTAMPTZ 19 | ); 20 | 21 | CREATE UNIQUE INDEX rfd_revision_sha_idx ON rfd_revision (rfd_id, sha); 22 | -------------------------------------------------------------------------------- /remix-auth-rfd/src/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, you can obtain one at https://mozilla.org/MPL/2.0/. 5 | * 6 | * Copyright Oxide Computer Company 7 | */ 8 | 9 | export type RfdScope = 10 | | 'user:info:r' 11 | | 'user:info:w' 12 | | 'user:provider:w' 13 | | 'user:token:r' 14 | | 'user:token:w' 15 | | 'group:info:r' 16 | | 'mapper:r' 17 | | 'mapper:w' 18 | | 'rfd:content:r' 19 | | 'rfd:discussion:r' 20 | | 'search' 21 | 22 | export type RfdApiProvider = 'email' | 'google' | 'github' 23 | 24 | export type RfdAccessToken = { 25 | iss: string 26 | aud: string 27 | sub: string 28 | prv: string 29 | scp: string[] 30 | exp: number 31 | nbf: number 32 | jti: string 33 | } 34 | 35 | export * from './magic-link' 36 | export * from './oauth' 37 | -------------------------------------------------------------------------------- /rfd-api/src/caller.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::collections::BTreeSet; 6 | use v_model::permissions::Caller; 7 | 8 | use crate::permissions::RfdPermission; 9 | 10 | pub trait CallerExt { 11 | fn allow_rfds(&self) -> BTreeSet; 12 | } 13 | 14 | impl CallerExt for Caller { 15 | fn allow_rfds(&self) -> BTreeSet { 16 | let mut allowed = BTreeSet::new(); 17 | for permission in self.permissions.iter() { 18 | match permission { 19 | RfdPermission::GetRfd(number) => { 20 | allowed.insert(*number); 21 | } 22 | RfdPermission::GetRfds(numbers) => allowed.extend(numbers), 23 | _ => (), 24 | } 25 | } 26 | 27 | allowed 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /rfd-cli/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-cli" 3 | version = "0.12.3" 4 | edition = "2021" 5 | 6 | [features] 7 | local-dev = [] 8 | 9 | [dependencies] 10 | anyhow = { workspace = true } 11 | chrono = { workspace = true } 12 | clap = { workspace = true } 13 | config = { workspace = true } 14 | dirs = { workspace = true } 15 | futures = { workspace = true } 16 | itertools = { workspace = true } 17 | jsonwebtoken = { workspace = true } 18 | oauth2 = { workspace = true } 19 | owo-colors = { workspace = true } 20 | progenitor-client = { workspace = true } 21 | reqwest = { workspace = true } 22 | rfd-sdk = { path = "../rfd-sdk" } 23 | schemars = { workspace = true } 24 | serde = { workspace = true } 25 | serde_json = { workspace = true } 26 | tabwriter = { workspace = true } 27 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } 28 | toml = { workspace = true } 29 | uuid = { workspace = true, features = ["serde", "v4"] } 30 | 31 | [package.metadata.dist] 32 | targets = ["x86_64-unknown-linux-gnu", "aarch64-apple-darwin", "x86_64-apple-darwin"] 33 | -------------------------------------------------------------------------------- /remix-auth-rfd/src/util.ts: -------------------------------------------------------------------------------- 1 | import Api, { ApiResult } from '@oxide/rfd.ts/client' 2 | import { ApiWithRetry } from '@oxide/rfd.ts/client-retry' 3 | 4 | const retryOnReset = () => { 5 | const limit = 1 6 | let retries = 0 7 | return (err: any) => { 8 | if (retries < limit && err.type === 'system' && err.errno === 'ECONNRESET' && err.code === 'ECONNRESET') { 9 | retries += 1 10 | return true 11 | } else { 12 | return false 13 | } 14 | } 15 | } 16 | 17 | export function client(host: string, token?: string): Api { 18 | return new ApiWithRetry({ host, token, retryHandler: retryOnReset }) 19 | } 20 | 21 | export function handleApiResponse(response: ApiResult): T { 22 | if (response.type === 'success') { 23 | return response.data 24 | } else if (response.type === 'client_error') { 25 | console.error('Failed attempting to send request to RFD server', response) 26 | throw response.error as Error 27 | } else { 28 | console.error('Failed attempting to send request to RFD server', response) 29 | throw new Error(response.data.message) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /rfd-installer/src/lib.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use diesel::{ 6 | migration::{Migration, MigrationSource}, 7 | pg::Pg, 8 | r2d2::{ConnectionManager, ManageConnection}, 9 | PgConnection, 10 | }; 11 | use diesel_migrations::{embed_migrations, EmbeddedMigrations}; 12 | 13 | const MIGRATIONS: EmbeddedMigrations = embed_migrations!("../rfd-model/migrations"); 14 | 15 | pub fn run_migrations(url: &str, v_only: bool) { 16 | v_api_installer::run_migrations(&url); 17 | 18 | if !v_only { 19 | let mut conn = db_conn(&url); 20 | let migrations: Vec>> = MIGRATIONS.migrations().unwrap(); 21 | 22 | for migration in migrations { 23 | migration.run(&mut conn).unwrap(); 24 | } 25 | } 26 | } 27 | 28 | fn db_conn(url: &str) -> PgConnection { 29 | let conn: ConnectionManager = ConnectionManager::new(url); 30 | conn.connect().unwrap() 31 | } 32 | -------------------------------------------------------------------------------- /rfd-model/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-model" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [features] 9 | mock = ["mockall"] 10 | 11 | [dependencies] 12 | async-bb8-diesel = { workspace = true } 13 | async-trait = { workspace = true } 14 | bb8 = { workspace = true } 15 | chrono = { workspace = true, features = ["serde"] } 16 | diesel = { workspace = true, features = ["chrono", "uuid", "serde_json"] } 17 | mockall = { workspace = true, optional = true } 18 | newtype-uuid = { workspace = true } 19 | partial-struct = { workspace = true } 20 | schemars = { workspace = true, features = ["chrono", "uuid1"] } 21 | serde = { workspace = true, features = ["derive"] } 22 | serde_json = { workspace = true } 23 | tap = { workspace = true } 24 | thiserror = { workspace = true } 25 | tracing = { workspace = true } 26 | uuid = { workspace = true, features = ["v4", "serde"] } 27 | v-model = { workspace = true } 28 | 29 | [dev-dependencies] 30 | diesel_migrations = { version = "2.2.0", features = ["postgres"] } 31 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -------------------------------------------------------------------------------- /rfd-processor/src/pdf.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use google_drive3::Error as ClientError; 7 | use rfd_data::RfdNumber; 8 | use rfd_model::schema_ext::PdfSource; 9 | use thiserror::Error; 10 | 11 | #[derive(Debug, Error)] 12 | pub enum RfdPdfError { 13 | #[error("Upload failed to return a valid file id for {0}")] 14 | FileIdMissing(String), 15 | #[error(transparent)] 16 | Remote(#[from] ClientError), 17 | } 18 | 19 | #[derive(Debug)] 20 | pub struct PdfFileLocation { 21 | pub source: PdfSource, 22 | pub url: String, 23 | pub external_id: String, 24 | } 25 | 26 | #[async_trait] 27 | pub trait PdfStorage { 28 | async fn store_rfd_pdf( 29 | &self, 30 | external_id: Option<&str>, 31 | filename: &str, 32 | pdf: &RfdPdf, 33 | ) -> Vec>; 34 | } 35 | 36 | #[derive(Debug)] 37 | pub struct RfdPdf { 38 | pub number: RfdNumber, 39 | pub contents: Vec, 40 | } 41 | -------------------------------------------------------------------------------- /rfd-github/src/util.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use base64::{prelude::BASE64_STANDARD, DecodeError, Engine}; 6 | 7 | pub fn decode_base64(c: &str) -> Result, DecodeError> { 8 | let v = c.replace('\n', ""); 9 | let decoded = BASE64_STANDARD.decode(&v)?; 10 | Ok(decoded.trim().to_vec()) 11 | } 12 | 13 | trait SliceExt { 14 | fn trim(&self) -> Self; 15 | } 16 | 17 | impl SliceExt for Vec { 18 | fn trim(&self) -> Vec { 19 | fn is_whitespace(c: &u8) -> bool { 20 | c == &b'\t' || c == &b' ' 21 | } 22 | 23 | fn is_not_whitespace(c: &u8) -> bool { 24 | !is_whitespace(c) 25 | } 26 | 27 | if let Some(first) = self.iter().position(is_not_whitespace) { 28 | if let Some(last) = self.iter().rposition(is_not_whitespace) { 29 | self[first..last + 1].to_vec() 30 | } else { 31 | unreachable!(); 32 | } 33 | } else { 34 | vec![] 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/auth/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::Result; 6 | use chrono::{DateTime, Duration, Utc}; 7 | use clap::{Parser, Subcommand}; 8 | use oauth2::TokenResponse; 9 | use rfd_sdk::types::OAuthProviderName; 10 | use std::ops::Add; 11 | 12 | use crate::{err::format_api_err, Context}; 13 | 14 | // mod link; 15 | mod login; 16 | mod oauth; 17 | 18 | // Authenticate against the RFD API 19 | #[derive(Parser, Debug)] 20 | #[clap(name = "auth")] 21 | pub struct Auth { 22 | #[command(subcommand)] 23 | auth: AuthCommands, 24 | } 25 | 26 | #[derive(Subcommand, Debug, Clone)] 27 | enum AuthCommands { 28 | // /// Link an authentication provider to an account 29 | // Link(link::Link), 30 | /// Login via an authentication provider 31 | Login(login::Login), 32 | } 33 | 34 | impl Auth { 35 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 36 | match &self.auth { 37 | // AuthCommands::Link(link) => link.run(ctx).await, 38 | AuthCommands::Login(login) => login.run(ctx).await, 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /remix-auth-rfd/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@oxide/remix-auth-rfd", 3 | "version": "0.1.4", 4 | "engines": { 5 | "node": ">=18" 6 | }, 7 | "type": "module", 8 | "main": "./dist/index.js", 9 | "exports": { 10 | "import": "./dist/index.js", 11 | "require": "./dist/index.cjs" 12 | }, 13 | "scripts": { 14 | "build": "tsup --dts", 15 | "prepublishOnly": "npm run build", 16 | "test": "echo \"Error: no test specified\" && exit 1", 17 | "tsc": "tsc" 18 | }, 19 | "keywords": [ 20 | "remix", 21 | "remix-auth", 22 | "auth", 23 | "authentication", 24 | "strategy" 25 | ], 26 | "author": "Oxide Computer Company", 27 | "license": "MPL-2.0", 28 | "repository": { 29 | "type": "git", 30 | "url": "git+https://github.com/oxidecomputer/rfd-api.git" 31 | }, 32 | "peerDependencies": { 33 | "remix-auth": "^4.0.0", 34 | "react-router": "^7.4.0" 35 | }, 36 | "devDependencies": { 37 | "tsup": "^8.0.2", 38 | "typescript": "^5.7.2" 39 | }, 40 | "tsup": { 41 | "clean": true, 42 | "entry": [ 43 | "src/index.ts" 44 | ], 45 | "format": [ 46 | "cjs", 47 | "esm" 48 | ] 49 | }, 50 | "dependencies": { 51 | "@oxide/rfd.ts": "^0.2.1", 52 | "remix-auth-oauth2": "^3.4.0" 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /rfd-api/src/util.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | pub mod response { 6 | use dropshot::{ClientErrorStatusCode, HttpError}; 7 | use std::error::Error; 8 | use tracing::instrument; 9 | 10 | pub fn unauthorized() -> HttpError { 11 | client_error(ClientErrorStatusCode::UNAUTHORIZED, "Unauthorized") 12 | } 13 | 14 | pub fn client_error(status_code: ClientErrorStatusCode, message: S) -> HttpError 15 | where 16 | S: ToString, 17 | { 18 | HttpError::for_client_error(None, status_code, message.to_string()) 19 | } 20 | 21 | #[instrument(skip(error))] 22 | pub fn to_internal_error(error: E) -> HttpError 23 | where 24 | E: Error, 25 | { 26 | internal_error(error.to_string()) 27 | } 28 | 29 | #[instrument(skip(internal_message))] 30 | pub fn internal_error(internal_message: S) -> HttpError 31 | where 32 | S: ToString, 33 | { 34 | let internal_message = internal_message.to_string(); 35 | tracing::error!(internal_message, "Request failed"); 36 | HttpError::for_internal_error(internal_message) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rfd-model/migrations/00000000000000_diesel_initial_setup/up.sql: -------------------------------------------------------------------------------- 1 | -- This file was automatically created by Diesel to setup helper functions 2 | -- and other internal bookkeeping. This file is safe to edit, any future 3 | -- changes will be added to existing projects as new migrations. 4 | 5 | 6 | 7 | 8 | -- Sets up a trigger for the given table to automatically set a column called 9 | -- `updated_at` whenever the row is modified (unless `updated_at` was included 10 | -- in the modified columns) 11 | -- 12 | -- # Example 13 | -- 14 | -- ```sql 15 | -- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); 16 | -- 17 | -- SELECT diesel_manage_updated_at('users'); 18 | -- ``` 19 | CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ 20 | BEGIN 21 | EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s 22 | FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); 23 | END; 24 | $$ LANGUAGE plpgsql; 25 | 26 | CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ 27 | BEGIN 28 | IF ( 29 | NEW IS DISTINCT FROM OLD AND 30 | NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at 31 | ) THEN 32 | NEW.updated_at := current_timestamp; 33 | END IF; 34 | RETURN NEW; 35 | END; 36 | $$ LANGUAGE plpgsql; 37 | -------------------------------------------------------------------------------- /rfd-model/migrations/2024-11-12-141610_v_api_conversion/up.sql: -------------------------------------------------------------------------------- 1 | DROP TRIGGER IF EXISTS set_updated_at ON access_groups; 2 | SELECT diesel_manage_updated_at('access_groups'); 3 | DROP TRIGGER IF EXISTS set_updated_at ON api_user; 4 | SELECT diesel_manage_updated_at('api_user'); 5 | DROP TRIGGER IF EXISTS set_updated_at ON api_key; 6 | SELECT diesel_manage_updated_at('api_key'); 7 | DROP TRIGGER IF EXISTS set_updated_at ON api_user_provider; 8 | SELECT diesel_manage_updated_at('api_user_provider'); 9 | DROP TRIGGER IF EXISTS set_updated_at ON api_user_access_token; 10 | SELECT diesel_manage_updated_at('api_user_access_token'); 11 | DROP TRIGGER IF EXISTS set_updated_at ON login_attempt; 12 | SELECT diesel_manage_updated_at('login_attempt'); 13 | 14 | ALTER TABLE mapper ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now(); 15 | DROP TRIGGER IF EXISTS set_updated_at ON mapper; 16 | SELECT diesel_manage_updated_at('mapper'); 17 | 18 | UPDATE api_user SET permissions = '[]' WHERE permissions = '[{"ManageGroupMemberships": []}, {"ManageGroups": []}, {"GetRfds": []}, {"UpdateRfds": []}, {"GetDiscussions": []}, {"GetOAuthClients": []}, {"UpdateOAuthClients": []}, {"DeleteOAuthClients": []}]'; 19 | UPDATE api_user SET permissions = '[]' WHERE permissions = '[{"ManageGroupMemberships": []}, {"ManageGroups": []}, {"GetRfds": []}, {"GetDiscussions": []}, {"GetOAuthClients": []}, {"UpdateOAuthClients": []}, {"DeleteOAuthClients": []}]'; 20 | -------------------------------------------------------------------------------- /rfd-api/src/secrets.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use schemars::{ 6 | schema::{InstanceType, SchemaObject}, 7 | JsonSchema, 8 | }; 9 | use secrecy::{ExposeSecret, SecretString}; 10 | use serde::{Deserialize, Serialize, Serializer}; 11 | use std::borrow::Cow; 12 | 13 | #[derive(Debug, Deserialize)] 14 | pub struct OpenApiSecretString(pub SecretString); 15 | 16 | impl From for OpenApiSecretString { 17 | fn from(value: SecretString) -> Self { 18 | Self(value) 19 | } 20 | } 21 | 22 | impl JsonSchema for OpenApiSecretString { 23 | fn is_referenceable() -> bool { 24 | true 25 | } 26 | 27 | fn json_schema(_: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { 28 | SchemaObject { 29 | instance_type: Some(InstanceType::String.into()), 30 | ..Default::default() 31 | } 32 | .into() 33 | } 34 | 35 | fn schema_id() -> std::borrow::Cow<'static, str> { 36 | Cow::Borrowed("secret-string") 37 | } 38 | 39 | fn schema_name() -> String { 40 | "SecretString".to_string() 41 | } 42 | } 43 | 44 | impl Serialize for OpenApiSecretString { 45 | fn serialize(&self, serializer: S) -> Result 46 | where 47 | S: Serializer, 48 | { 49 | serializer.serialize_str(self.0.expose_secret()) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | test: 9 | runs-on: ubuntu-24.04 10 | services: 11 | postgres: 12 | image: postgres 13 | env: 14 | POSTGRES_USER: test 15 | POSTGRES_PASSWORD: test 16 | options: >- 17 | --health-cmd pg_isready 18 | --health-interval 10s 19 | --health-timeout 5s 20 | --health-retries 5 21 | ports: 22 | - 5432:5432 23 | 24 | steps: 25 | - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 26 | - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 27 | with: 28 | toolchain: stable 29 | - name: Install asciidoctor 30 | shell: bash 31 | run: | 32 | sudo apt update -y && sudo apt install -y \ 33 | asciidoctor \ 34 | ruby \ 35 | - name: Install asciidoctor-pdf, asciidoctor-mermaid 36 | shell: bash 37 | run: | 38 | sudo gem install rouge 39 | sudo gem install asciidoctor-pdf -v 2.3.2 40 | sudo gem install asciidoctor-mermaid -v 0.4.1 41 | sudo npm install -g @mermaid-js/mermaid-cli@11.4.0 42 | - name: Report versions 43 | run: | 44 | asciidoctor --version 45 | asciidoctor-pdf --version 46 | mmdc --version 47 | - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # v1 48 | with: 49 | command: test 50 | env: 51 | TEST_DATABASE: postgres://test:test@localhost 52 | RUST_LOG: v=trace,rfd=trace 53 | -------------------------------------------------------------------------------- /rfd-processor/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-processor" 3 | version = "0.12.3" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | async-trait = { workspace = true } 10 | base64 = { workspace = true } 11 | chrono = { workspace = true } 12 | config = { workspace = true } 13 | diesel = { workspace = true } 14 | futures = { workspace = true } 15 | google-drive3 = { workspace = true } 16 | google-storage1 = { workspace = true } 17 | hex = { workspace = true } 18 | hmac = { workspace = true } 19 | http = { workspace = true } 20 | meilisearch-sdk = { workspace = true } 21 | md-5 = { workspace = true } 22 | mime_guess = { workspace = true } 23 | newtype-uuid = { workspace = true } 24 | octorust = { workspace = true, features = ["httpcache"] } 25 | parse-rfd = { path = "../parse-rfd" } 26 | regex = { workspace = true } 27 | reqwest = { workspace = true } 28 | reqwest-middleware = { workspace = true } 29 | reqwest-retry = { workspace = true } 30 | reqwest-tracing = { workspace = true } 31 | rfd-data = { path = "../rfd-data" } 32 | rfd-github = { path = "../rfd-github" } 33 | rfd-model = { path = "../rfd-model" } 34 | rsa = { workspace = true } 35 | serde = { workspace = true } 36 | tap = { workspace = true } 37 | thiserror = { workspace = true } 38 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } 39 | tracing = { workspace = true } 40 | tracing-appender = { workspace = true } 41 | tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "json"] } 42 | v-model = { workspace = true } 43 | uuid = { workspace = true } 44 | yup-oauth2 = { workspace = true } 45 | 46 | [package.metadata.dist] 47 | targets = ["x86_64-unknown-linux-gnu"] 48 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/ensure_default_state.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use tracing::instrument; 7 | 8 | use crate::rfd::PersistedRfd; 9 | 10 | use super::{ 11 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 12 | RfdUpdateMode, 13 | }; 14 | 15 | #[derive(Debug)] 16 | pub struct EnsureRfdOnDefaultIsInValidState; 17 | 18 | #[async_trait] 19 | impl RfdUpdateAction for EnsureRfdOnDefaultIsInValidState { 20 | #[instrument(skip(self, ctx, new), err(Debug))] 21 | async fn run( 22 | &self, 23 | ctx: &mut RfdUpdateActionContext, 24 | new: &mut PersistedRfd, 25 | _mode: RfdUpdateMode, 26 | ) -> Result { 27 | let RfdUpdateActionContext { update, .. } = ctx; 28 | 29 | // If a RFD exists on the default branch then it should be in either the published or 30 | // abandoned state 31 | if update.location.branch == update.location.default_branch { 32 | if !new.is_state("published") 33 | && !new.is_state("committed") 34 | && !new.is_state("abandoned") 35 | { 36 | tracing::warn!(state = ?new.revision.state, "RFD on the default branch is in an invalid state. It needs to be updated to either published or abandoned"); 37 | } else { 38 | tracing::debug!( 39 | "RFD on the default branch is in a valid state. No updates are needed" 40 | ); 41 | } 42 | } 43 | 44 | Ok(RfdUpdateActionResponse::default()) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /rfd-ts/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@oxide/rfd.ts", 3 | "version": "0.2.1", 4 | "description": "TypeScript client for the RFD API", 5 | "engines": { 6 | "node": ">=18" 7 | }, 8 | "type": "module", 9 | "exports": { 10 | "./client": { 11 | "import": "./dist/Api.js", 12 | "require": "./dist/Api.cjs" 13 | }, 14 | "./client-retry": { 15 | "import": "./dist/retry.js", 16 | "require": "./dist/retry.cjs" 17 | }, 18 | "./validation": { 19 | "import": "./dist/validate.js", 20 | "require": "./dist/validate.cjs" 21 | } 22 | }, 23 | "typesVersions": { 24 | "*": { 25 | "client": [ 26 | "dist/Api.d.ts" 27 | ], 28 | "client-retry": [ 29 | "dist/retry.d.ts" 30 | ], 31 | "validation": [ 32 | "dist/validate.d.ts" 33 | ] 34 | } 35 | }, 36 | "scripts": { 37 | "build": "tsup --dts", 38 | "prepublishOnly": "npm run build", 39 | "test": "echo \"Error: no test specified\" && exit 1", 40 | "tsc": "tsc" 41 | }, 42 | "repository": { 43 | "type": "git", 44 | "url": "git+https://github.com/oxidecomputer/rfd-api.git" 45 | }, 46 | "keywords": [], 47 | "author": "Oxide Computer Company", 48 | "license": "MPL-2.0", 49 | "bugs": { 50 | "url": "https://github.com/oxidecomputer/rfd-api/issues" 51 | }, 52 | "homepage": "https://github.com/oxidecomputer/rfd-api#readme", 53 | "peerDependencies": { 54 | "zod": "^3.23.8 || ^4.0.0" 55 | }, 56 | "devDependencies": { 57 | "tsup": "^8.0.2", 58 | "typescript": "^5.7.2" 59 | }, 60 | "tsup": { 61 | "clean": true, 62 | "entry": [ 63 | "src/Api.ts", 64 | "src/retry.ts", 65 | "src/validate.ts" 66 | ], 67 | "splitting": false, 68 | "format": [ 69 | "cjs", 70 | "esm" 71 | ] 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/update_search_index.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use rfd_model::schema_ext::Visibility; 7 | use tracing::instrument; 8 | 9 | use crate::rfd::PersistedRfd; 10 | 11 | use super::{ 12 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 13 | RfdUpdateMode, 14 | }; 15 | 16 | #[derive(Debug)] 17 | pub struct UpdateSearch; 18 | 19 | #[async_trait] 20 | impl RfdUpdateAction for UpdateSearch { 21 | #[instrument(skip(self, ctx, new), err(Debug))] 22 | async fn run( 23 | &self, 24 | ctx: &mut RfdUpdateActionContext, 25 | new: &mut PersistedRfd, 26 | mode: RfdUpdateMode, 27 | ) -> Result { 28 | let RfdUpdateActionContext { ctx, .. } = ctx; 29 | 30 | for (i, index) in ctx.search.indexes.iter().enumerate() { 31 | tracing::info!("Updating search index"); 32 | 33 | if mode == RfdUpdateMode::Write { 34 | let public = match new.rfd.visibility { 35 | Visibility::Private => false, 36 | Visibility::Public => true, 37 | }; 38 | 39 | if let Err(err) = index 40 | .index_rfd(&new.rfd.rfd_number.into(), &new.revision.content, public) 41 | .await 42 | { 43 | tracing::error!(?err, search_index = i, "Failed to add RFD to search index"); 44 | } 45 | } 46 | } 47 | 48 | tracing::info!("Finished updating search indexes"); 49 | 50 | Ok(RfdUpdateActionResponse::default()) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/shortcut/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::Result; 6 | use clap::{Parser, Subcommand}; 7 | 8 | use crate::{ 9 | cmd::shortcut::{ 10 | access::{AccessShortcut, AccessShortcuts, RfdAccessShortcut, RfdAccessShortcuts}, 11 | mapper::{MapperShortcut, MapperShortcuts}, 12 | }, 13 | Context, 14 | }; 15 | 16 | use self::mapper::{EmailMapper, GitHubMapper}; 17 | 18 | mod access; 19 | mod mapper; 20 | 21 | #[derive(Debug, Parser)] 22 | #[clap(name = "shortcut", short_flag = 's')] 23 | /// Shorthand commands for commonly used features 24 | pub struct ShortcutCmd { 25 | #[clap(subcommand)] 26 | shortcut: Shortcut, 27 | } 28 | 29 | #[derive(Debug, Subcommand)] 30 | pub enum Shortcut { 31 | /// Grant and revoke access to resources for groups 32 | Access(AccessShortcut), 33 | /// Create new mappers 34 | Mapper(MapperShortcut), 35 | } 36 | 37 | impl ShortcutCmd { 38 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 39 | match &self.shortcut { 40 | Shortcut::Access(shortcut) => match &shortcut.access { 41 | AccessShortcuts::Rfd(method) => match &method.rfd { 42 | RfdAccessShortcuts::Add(cmd) => { 43 | cmd.run(ctx).await?; 44 | } 45 | RfdAccessShortcuts::Remove(cmd) => { 46 | cmd.run(ctx).await?; 47 | } 48 | }, 49 | }, 50 | Shortcut::Mapper(shortcut) => match &shortcut.mapper { 51 | MapperShortcuts::Email(cmd) => cmd.run(ctx).await?, 52 | MapperShortcuts::GitHub(cmd) => cmd.run(ctx).await?, 53 | }, 54 | } 55 | 56 | Ok(()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /parse-rfd/parser/index.js: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | let Asciidoctor = require('@asciidoctor/core') 6 | let convert = require('html-to-text').convert 7 | 8 | const asciidoc = Asciidoctor() 9 | 10 | const parse = (content) => { 11 | const doc = asciidoc.load(content) 12 | 13 | const sections = doc 14 | .getSections() 15 | .map((section) => formatSection(section, content)) 16 | .reduce((acc, prev) => [...acc, ...prev], []) 17 | 18 | const title = doc.getTitle() 19 | 20 | return { 21 | title: (title || '') 22 | .replace('RFD', '') 23 | .replace('# ', '') 24 | .replace('= ', '') 25 | .trim() 26 | .split(' ') 27 | .slice(1) 28 | .join(' '), 29 | sections, 30 | } 31 | } 32 | 33 | const formatSection = (section, content) => { 34 | const formattedSections = [] 35 | for (const s of section.getSections()) { 36 | formattedSections.push(...formatSection(s, content)) 37 | } 38 | const parentSections = [] 39 | let level = section.getLevel() - 1 40 | let currSection = section.getParent() 41 | 42 | while (level-- && currSection) { 43 | if (typeof currSection.getName === 'function') { 44 | parentSections.push(currSection.getName()) 45 | } 46 | currSection = currSection.getParent() 47 | } 48 | 49 | return [ 50 | { 51 | section_id: section.getId(), 52 | name: section.getName(), 53 | content: convert( 54 | section 55 | .getBlocks() 56 | .filter((block) => block.context !== 'section') 57 | .map((block) => block.convert()) 58 | .join(''), 59 | ), 60 | parents: parentSections, 61 | }, 62 | ...formattedSections, 63 | ] 64 | } 65 | 66 | let content = require('fs').readFileSync(0, 'utf-8') 67 | console.log(JSON.stringify(parse(content))) 68 | -------------------------------------------------------------------------------- /rfd-ts/src/retry.ts: -------------------------------------------------------------------------------- 1 | import Api from './Api' 2 | import { ApiConfig, ApiResult, FullParams, handleResponse, mergeParams, toQueryString } from './http-client' 3 | import { snakeify } from './util' 4 | 5 | export type RetryHandler = (err: any) => boolean 6 | export type RetryHandlerFactory = (url: RequestInfo | URL, init: RequestInit) => RetryHandler 7 | 8 | export type ApiConfigWithRetry = ApiConfig & { retryHandler?: RetryHandlerFactory } 9 | 10 | export class ApiWithRetry extends Api { 11 | retryHandler: RetryHandlerFactory 12 | 13 | constructor({ host = '', baseParams = {}, token, retryHandler }: ApiConfigWithRetry = {}) { 14 | super({ host, baseParams, token }) 15 | this.retryHandler = retryHandler ? retryHandler : () => () => false 16 | } 17 | 18 | public async request({ 19 | body, 20 | path, 21 | query, 22 | host, 23 | ...fetchParams 24 | }: FullParams): Promise> { 25 | const url = (host || this.host) + path + toQueryString(query) 26 | const init = { 27 | ...mergeParams(this.baseParams, fetchParams), 28 | body: JSON.stringify(snakeify(body), replacer), 29 | } 30 | return fetchWithRetry(fetch, url, init, this.retryHandler(url, init)) 31 | } 32 | } 33 | 34 | export async function fetchWithRetry( 35 | fetch: any, 36 | url: string, 37 | init: RequestInit, 38 | retry: RetryHandler, 39 | ): Promise> { 40 | try { 41 | return handleResponse(await fetch(url, init)) 42 | } catch (err) { 43 | if (retry(err)) { 44 | return await fetchWithRetry(fetch, url, init, retry) 45 | } 46 | 47 | throw err 48 | } 49 | } 50 | 51 | /** 52 | * Convert `Date` to ISO string. Leave other values alone. Used for both request 53 | * body and query params. 54 | */ 55 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 56 | function replacer(_key: string, value: any) { 57 | if (value instanceof Date) { 58 | return value.toISOString() 59 | } 60 | return value 61 | } 62 | -------------------------------------------------------------------------------- /remix-auth-rfd/src/oauth.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, you can obtain one at https://mozilla.org/MPL/2.0/. 5 | * 6 | * Copyright Oxide Computer Company 7 | */ 8 | 9 | import Api, { GetUserResponse_for_RfdPermission } from '@oxide/rfd.ts/client' 10 | import { redirect, type SessionData, type SessionStorage } from 'react-router' 11 | import { OAuth2Strategy } from 'remix-auth-oauth2' 12 | import { Strategy } from 'remix-auth/strategy' 13 | import { RfdApiProvider, RfdScope } from '.' 14 | import { client, handleApiResponse } from './util' 15 | 16 | export type RfdOAuthStrategyOptions = { 17 | host: string 18 | clientId: string 19 | clientSecret: string 20 | redirectURI: string 21 | remoteProvider: RfdApiProvider 22 | /** 23 | * @default "user:info:r" 24 | */ 25 | scopes?: RfdScope[] 26 | } 27 | 28 | export type ExpiringUser = { 29 | expiresAt: Date 30 | } 31 | 32 | export type RfdVerifyCallback = Strategy.VerifyFunction 33 | 34 | export class RfdOAuthStrategy extends OAuth2Strategy< 35 | User 36 | > { 37 | public name = `rfd` 38 | protected readonly userInfoUrl 39 | protected readonly host 40 | 41 | constructor( 42 | { 43 | host, 44 | clientId, 45 | clientSecret, 46 | redirectURI, 47 | remoteProvider, 48 | scopes, 49 | }: RfdOAuthStrategyOptions, 50 | verify: RfdVerifyCallback, 51 | ) { 52 | super( 53 | { 54 | clientId, 55 | clientSecret, 56 | redirectURI, 57 | authorizationEndpoint: `${host}/login/oauth/${remoteProvider}/code/authorize`, 58 | tokenEndpoint: `${host}/login/oauth/${remoteProvider}/code/token`, 59 | scopes: scopes ?? ['user:info:r'], 60 | }, 61 | verify, 62 | ) 63 | this.name = `${this.name}-${remoteProvider}` 64 | this.host = host 65 | this.userInfoUrl = `${host}/self` 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /rfd-ts/src/util.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, you can obtain one at https://mozilla.org/MPL/2.0/. 5 | * 6 | * Copyright Oxide Computer Company 7 | */ 8 | 9 | export const camelToSnake = (s: string) => s.replace(/[A-Z]/g, (l) => '_' + l.toLowerCase()) 10 | 11 | export const snakeToCamel = (s: string) => s.replace(/_./g, (l) => l[1].toUpperCase()) 12 | 13 | export const isObjectOrArray = (o: unknown) => 14 | typeof o === 'object' 15 | && !(o instanceof Date) 16 | && !(o instanceof RegExp) 17 | && !(o instanceof Error) 18 | && o !== null 19 | 20 | /** 21 | * Recursively map (k, v) pairs using Object.entries 22 | * 23 | * Note that value transform function takes both k and v so we can use the key 24 | * to decide whether to transform the value. 25 | * 26 | * @param kf maps key to key 27 | * @param vf maps key + value to value 28 | */ 29 | export const mapObj = ( 30 | kf: (k: string) => string, 31 | vf: (k: string | undefined, v: unknown) => unknown = (k, v) => v, 32 | ) => 33 | (o: unknown): unknown => { 34 | if (!isObjectOrArray(o)) { return o } 35 | 36 | if (Array.isArray(o)) { return o.map(mapObj(kf, vf)) } 37 | 38 | const newObj: Record = {} 39 | for (const [k, v] of Object.entries(o as Record)) { 40 | newObj[kf(k)] = isObjectOrArray(v) ? mapObj(kf, vf)(v) : vf(k, v) 41 | } 42 | return newObj 43 | } 44 | 45 | export const parseIfDate = (k: string | undefined, v: unknown) => { 46 | if (typeof v === 'string' && (k?.startsWith('time_') || k === 'timestamp')) { 47 | const d = new Date(v) 48 | if (isNaN(d.getTime())) { return v } 49 | return d 50 | } 51 | return v 52 | } 53 | 54 | export const snakeify = mapObj(camelToSnake) 55 | 56 | export const processResponseBody = mapObj(snakeToCamel, parseIfDate) 57 | 58 | export function isNotNull(value: T): value is NonNullable { 59 | return value != null 60 | } 61 | 62 | export const uniqueItems = [ 63 | (arr: T[]) => new Set(arr).size === arr.length, 64 | { message: 'Items must be unique' }, 65 | ] as const 66 | -------------------------------------------------------------------------------- /rfd-data/src/content/template.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use serde::{Deserialize, Serialize}; 6 | use std::collections::HashMap; 7 | use thiserror::Error; 8 | 9 | #[derive(Debug, Error)] 10 | pub enum TemplateError { 11 | #[error("Template is missing some of the required values")] 12 | MissingRequiredFields { 13 | template: RfdTemplate, 14 | values: Vec, 15 | }, 16 | } 17 | 18 | #[derive(Clone, Debug, Deserialize, Serialize, Default)] 19 | pub struct RfdTemplate { 20 | template: String, 21 | #[serde(default)] 22 | values: HashMap, 23 | required_fields: Vec, 24 | } 25 | 26 | #[derive(Clone, Debug)] 27 | pub struct RenderableRfdTemplate(RfdTemplate); 28 | 29 | impl RfdTemplate { 30 | pub fn field(mut self, field: String, value: String) -> Self { 31 | self.values.insert(field, value); 32 | self 33 | } 34 | 35 | pub fn build(self) -> Result { 36 | let set_fields = self.values.keys().collect::>(); 37 | let missing_fields = self 38 | .required_fields 39 | .iter() 40 | .filter(|field| !set_fields.contains(field)) 41 | .cloned() 42 | .collect::>(); 43 | 44 | if missing_fields.len() == 0 { 45 | Ok(RenderableRfdTemplate(self)) 46 | } else { 47 | Err(TemplateError::MissingRequiredFields { 48 | template: self, 49 | values: missing_fields, 50 | }) 51 | } 52 | } 53 | } 54 | 55 | impl RenderableRfdTemplate { 56 | pub fn render(self) -> String { 57 | let mut rendered = self.0.template; 58 | 59 | for field in self.0.required_fields { 60 | rendered = rendered.replace(&format!("{{{}}}", field), self.0.values.get(&field).expect("Renderable template is missing a required field. This is a bug, please report.")); 61 | } 62 | 63 | rendered 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /rfd-processor/src/content/asciidoc.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use rfd_data::content::{RfdAsciidoc, RfdDocument}; 7 | use std::{path::PathBuf, process::Command}; 8 | 9 | use crate::util::write_file; 10 | 11 | use super::{RenderableRfdError, RenderedPdf, RfdOutputError, RfdRenderedFormat}; 12 | 13 | #[async_trait] 14 | impl<'a> RfdRenderedFormat> for RenderedPdf { 15 | async fn render(content: &RfdAsciidoc, content_dir: PathBuf) -> Result { 16 | let file_path = content_dir.join("contents.adoc"); 17 | 18 | // Write the contents to a temporary file. 19 | write_file(&file_path, content.raw().as_bytes()).await?; 20 | tracing::info!("Wrote file to temp dir"); 21 | 22 | let mut command = Command::new("asciidoctor-pdf"); 23 | command.current_dir(content_dir.clone()).args([ 24 | "-o", 25 | "-", 26 | "-r", 27 | "asciidoctor-mermaid/pdf", 28 | "-a", 29 | "source-highlighter=rouge", 30 | file_path.to_str().unwrap(), 31 | ]); 32 | 33 | let cmd_output = tokio::task::spawn_blocking(move || { 34 | tracing::info!(?file_path, "Shelling out to asciidoctor"); 35 | 36 | // Verify the expected resources exist 37 | tracing::info!(?file_path, exists = file_path.exists(), "Check document"); 38 | 39 | let out = command.output(); 40 | 41 | match &out { 42 | Ok(_) => tracing::info!(?file_path, "Command succeeded"), 43 | Err(err) => tracing::info!(?file_path, ?err, "Command failed"), 44 | }; 45 | 46 | out 47 | }) 48 | .await??; 49 | 50 | tracing::info!("Completed asciidoc rendering"); 51 | 52 | if cmd_output.status.success() { 53 | Ok(cmd_output.stdout.into()) 54 | } else { 55 | Err(RenderableRfdError::ParserFailed(String::from_utf8( 56 | cmd_output.stderr, 57 | )))? 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /rfd-api/src/error.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use dropshot::HttpError; 6 | use octorust::ClientError as OctorustError; 7 | use reqwest::Error as ReqwestError; 8 | use rfd_github::GitHubError; 9 | use thiserror::Error; 10 | use v_api::response::{conflict, forbidden, internal_error, not_found, ResourceError}; 11 | use v_model::storage::StoreError; 12 | 13 | #[derive(Debug, Error)] 14 | pub enum AppError { 15 | #[error("Failed to construct HTTP client")] 16 | ClientConstruction(ReqwestError), 17 | #[error("GitHub communication error")] 18 | GitHub(#[from] GitHubError), 19 | #[error("Invalid GitHub private key")] 20 | InvalidGitHubPrivateKey(#[from] rsa::pkcs1::Error), 21 | #[error("A template for new RFDs must be defined")] 22 | MissingNewRfdTemplate, 23 | #[error("At least one JWT signing key must be configured")] 24 | NoConfiguredJwtKeys, 25 | #[error("Failed to construct GitHub client")] 26 | Octorust(#[from] OctorustError), 27 | } 28 | 29 | #[derive(Debug, Error)] 30 | pub enum ApiError { 31 | #[error("Conflict with an existing resource")] 32 | Conflict, 33 | #[error("Caller does not have the required permissions")] 34 | Forbidden, 35 | #[error("Resource could not be found")] 36 | NotFound, 37 | #[error("Internal storage failed {0}")] 38 | Storage(#[from] StoreError), 39 | } 40 | 41 | impl From for HttpError { 42 | fn from(error: ApiError) -> Self { 43 | match error { 44 | ApiError::Conflict => conflict(), 45 | ApiError::Forbidden => forbidden(), 46 | ApiError::NotFound => not_found(""), 47 | ApiError::Storage(_) => internal_error("Internal storage failed"), 48 | } 49 | } 50 | } 51 | 52 | impl From> for ApiError { 53 | fn from(value: ResourceError) -> Self { 54 | match value { 55 | ResourceError::Conflict => ApiError::Conflict, 56 | ResourceError::DoesNotExist => ApiError::NotFound, 57 | ResourceError::InternalError(err) => ApiError::Storage(err), 58 | ResourceError::Restricted => ApiError::Forbidden, 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rfd-api/src/endpoints/job.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use dropshot::{endpoint, ClientErrorStatusCode, HttpError, HttpResponseOk, Query, RequestContext}; 6 | use rfd_model::{storage::JobFilter, Job}; 7 | use schemars::JsonSchema; 8 | use serde::Deserialize; 9 | use trace_request::trace_request; 10 | use tracing::instrument; 11 | use v_api::{response::client_error, ApiContext}; 12 | use v_model::{permissions::Caller, storage::ListPagination}; 13 | 14 | use crate::{context::RfdContext, permissions::RfdPermission}; 15 | 16 | use super::UNLIMITED; 17 | 18 | // Read Endpoints 19 | 20 | #[derive(Debug, Deserialize, JsonSchema)] 21 | struct ListJobsQuery { 22 | rfd: String, 23 | limit: Option, 24 | offset: Option, 25 | } 26 | 27 | /// List all jobs for a RFD 28 | #[trace_request] 29 | #[endpoint { 30 | method = GET, 31 | path = "/job", 32 | }] 33 | #[instrument(skip(rqctx), fields(request_id = rqctx.request_id), err(Debug))] 34 | pub async fn list_jobs( 35 | rqctx: RequestContext, 36 | query: Query, 37 | ) -> Result>, HttpError> { 38 | let ctx = rqctx.context(); 39 | let caller = ctx.v_ctx().get_caller(&rqctx).await?; 40 | let query = query.into_inner(); 41 | list_jobs_op(ctx, &caller, query).await 42 | } 43 | 44 | // Read operation 45 | 46 | #[instrument(skip(ctx, caller), fields(caller = ?caller.id), err(Debug))] 47 | async fn list_jobs_op( 48 | ctx: &RfdContext, 49 | caller: &Caller, 50 | query: ListJobsQuery, 51 | ) -> Result>, HttpError> { 52 | if let Ok(rfd_number) = query.rfd.parse::() { 53 | let jobs = ctx 54 | .list_jobs( 55 | caller, 56 | Some(JobFilter::default().rfd(Some(vec![rfd_number]))), 57 | &ListPagination::default() 58 | .limit(query.limit.unwrap_or(UNLIMITED)) 59 | .offset(query.offset.unwrap_or(0)), 60 | ) 61 | .await?; 62 | Ok(HttpResponseOk(jobs)) 63 | } else { 64 | Err(client_error( 65 | ClientErrorStatusCode::BAD_REQUEST, 66 | "Malformed RFD number", 67 | )) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release CLI 2 | 3 | on: 4 | push: 5 | tags: 6 | - '**[0-9]+.[0-9]+.[0-9]+*' 7 | 8 | jobs: 9 | release-linux: 10 | runs-on: ubuntu-24.04 11 | permissions: 12 | contents: write 13 | steps: 14 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 15 | 16 | - name: Install Rust 17 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 18 | with: 19 | toolchain: stable 20 | targets: x86_64-unknown-linux-gnu 21 | 22 | - name: Build Release Binary 23 | run: | 24 | cargo build --release --package rfd-cli 25 | mv target/release/rfd-cli target/release/rfd-cli-x86_64-unknown-linux-gnu 26 | 27 | - uses: ncipollo/release-action@bcfe5470707e8832e12347755757cec0eb3c22af # v1 28 | with: 29 | allowUpdates: true 30 | artifacts: "target/release/rfd-cli-x86_64-unknown-linux-gnu" 31 | release-macos: 32 | runs-on: macos-15 33 | permissions: 34 | contents: write 35 | steps: 36 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 37 | 38 | - name: Install Rust 39 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 40 | with: 41 | toolchain: stable 42 | targets: aarch64-apple-darwin 43 | 44 | - name: Build Release Binary 45 | run: | 46 | cargo build --release --package rfd-cli 47 | mv target/release/rfd-cli target/release/rfd-cli-aarch64-apple-darwin 48 | 49 | - uses: ncipollo/release-action@bcfe5470707e8832e12347755757cec0eb3c22af # v1 50 | with: 51 | allowUpdates: true 52 | artifacts: "target/release/rfd-cli-aarch64-apple-darwin" 53 | 54 | release-windows: 55 | runs-on: windows-2025 56 | permissions: 57 | contents: write 58 | steps: 59 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 60 | 61 | - name: Install Rust 62 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 63 | with: 64 | toolchain: stable 65 | targets: x86_64-pc-windows-msvc 66 | 67 | - name: Build Release Binary 68 | run: | 69 | cargo build --release --package rfd-cli 70 | move target\release\rfd-cli.exe target\release\rfd-cli-x86_64-pc-windows-msvc.exe 71 | 72 | - uses: ncipollo/release-action@bcfe5470707e8832e12347755757cec0eb3c22af # v1 73 | with: 74 | allowUpdates: true 75 | artifacts: "target\\/release\\/rfd-cli-x86_64-pc-windows-msvc.exe" 76 | -------------------------------------------------------------------------------- /dropshot-authorization-header/src/bearer.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use dropshot::{ 7 | ApiEndpointBodyContentType, ExtensionMode, ExtractorMetadata, HttpError, RequestContext, 8 | ServerContext, SharedExtractor, 9 | }; 10 | 11 | /// A token used for bearer authentication 12 | pub struct BearerAuth(Option); 13 | 14 | impl BearerAuth { 15 | pub fn new(token: String) -> Self { 16 | Self(Some(token)) 17 | } 18 | 19 | pub fn key(&self) -> Option<&str> { 20 | self.0.as_ref().map(|s| s.as_str()) 21 | } 22 | 23 | pub fn consume(self) -> Option { 24 | self.0 25 | } 26 | } 27 | 28 | /// Extracting a bearer token should never fail, it should always return `Ok(BearerAuth(Some(token))))` 29 | /// or `Ok(BearerAuth(None))`. `None` will be returned in any of the cases that a valid string can not 30 | /// be extracted. This extractor is not responsible for checking the value of the token. 31 | #[async_trait] 32 | impl SharedExtractor for BearerAuth { 33 | async fn from_request( 34 | rqctx: &RequestContext, 35 | ) -> Result { 36 | // Similarly we only care about the presence of the Authorization header 37 | let header_value = rqctx 38 | .request 39 | .headers() 40 | .get("Authorization") 41 | .and_then(|header| { 42 | // If the value provided is not a readable string we will throw it out 43 | header 44 | .to_str() 45 | .map(|s| s.to_string()) 46 | .map_err(|err| { 47 | tracing::info!("Failed to turn Authorization header into string"); 48 | err 49 | }) 50 | .ok() 51 | }); 52 | 53 | // Finally ensure that the value we found is properly formed 54 | let contents = header_value.and_then(|value| { 55 | let parts = value.split_once(' '); 56 | 57 | match parts { 58 | Some(("Bearer", token)) => Some(token.to_string()), 59 | _ => None, 60 | } 61 | }); 62 | 63 | Ok(BearerAuth(contents)) 64 | } 65 | 66 | fn metadata(_body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata { 67 | ExtractorMetadata { 68 | parameters: vec![], 69 | extension_mode: ExtensionMode::None, 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/auth/link.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use core::panic; 6 | 7 | use anyhow::Result; 8 | use clap::{Parser, Subcommand}; 9 | use jsonwebtoken::{Algorithm, DecodingKey, Validation}; 10 | use oauth2::TokenResponse; 11 | use rfd_sdk::types::OAuthProviderName; 12 | use serde::Deserialize; 13 | use uuid::Uuid; 14 | 15 | use crate::{ 16 | cmd::auth::{login::AuthenticationMode, oauth}, 17 | Context, 18 | }; 19 | 20 | use super::login::LoginProvider; 21 | 22 | // Authenticates and generates an access token for interacting with the api 23 | #[derive(Parser, Debug, Clone)] 24 | #[clap(name = "link")] 25 | pub struct Link { 26 | #[arg(value_enum)] 27 | provider: LoginProvider, 28 | } 29 | 30 | #[derive(Debug, Deserialize)] 31 | struct Claims { 32 | prv: Uuid, 33 | } 34 | 35 | impl Link { 36 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 37 | // Determine the user id of the currently authenticated user 38 | let self_id = ctx.client()?.get_self().send().await?.info.id; 39 | 40 | let access_token = self 41 | .provider 42 | .run(ctx, &AuthenticationMode::Identity) 43 | .await?; 44 | 45 | // Fetch the public JWKS from the remote API 46 | let jwks = ctx.client()?.jwks_json().send().await?.into_inner(); 47 | let jwk = &jwks.keys[0]; 48 | 49 | // Decode the access token to extract the provider token 50 | let jwt = jsonwebtoken::decode::( 51 | &access_token, 52 | &DecodingKey::from_rsa_components(&jwk.n, &jwk.e)?, 53 | &Validation::new(Algorithm::RS256), 54 | )?; 55 | 56 | // An account linking request can only be generated by the owning account. Therefore we 57 | // need to use the sdk to generate a new client 58 | let client = ctx.new_client(Some(&access_token))?; 59 | 60 | // This needs to be the id of the provider the client just logged in with 61 | let link_token = client 62 | .create_link_token() 63 | .identifier(jwt.claims.prv) 64 | .body_map(|body| body.user_identifier(self_id)) 65 | .send() 66 | .await? 67 | .into_inner() 68 | .token; 69 | 70 | ctx.client()? 71 | .link_provider() 72 | .identifier(self_id) 73 | .body_map(|body| body.token(link_token)) 74 | .send() 75 | .await?; 76 | 77 | println!("Successfully linked provider"); 78 | 79 | Ok(()) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/config/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::Result; 6 | use clap::{Parser, Subcommand}; 7 | 8 | use crate::{Context, FormatStyle}; 9 | 10 | #[derive(Debug, Parser)] 11 | #[clap(name = "config")] 12 | pub struct ConfigCmd { 13 | #[clap(subcommand)] 14 | setting: SettingCmd, 15 | } 16 | 17 | #[derive(Debug, Subcommand)] 18 | pub enum SettingCmd { 19 | /// Gets a setting 20 | #[clap(subcommand, name = "get")] 21 | Get(GetCmd), 22 | /// Sets a setting 23 | #[clap(subcommand, name = "set")] 24 | Set(SetCmd), 25 | } 26 | 27 | #[derive(Debug, Subcommand)] 28 | pub enum GetCmd { 29 | /// Get the default formatter to use when printing results 30 | #[clap(name = "format")] 31 | Format, 32 | /// Get the configured API host in use 33 | #[clap(name = "host")] 34 | Host, 35 | /// Get the configured access token 36 | #[clap(name = "token")] 37 | Token, 38 | } 39 | 40 | #[derive(Debug, Subcommand)] 41 | pub enum SetCmd { 42 | /// Set the default formatter to use when printing results 43 | #[clap(name = "format")] 44 | Format { format: FormatStyle }, 45 | /// Set the configured API host to use 46 | #[clap(name = "host")] 47 | Host { host: String }, 48 | } 49 | 50 | impl ConfigCmd { 51 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 52 | match &self.setting { 53 | SettingCmd::Get(get) => get.run(ctx).await?, 54 | SettingCmd::Set(set) => set.run(ctx).await?, 55 | } 56 | 57 | Ok(()) 58 | } 59 | } 60 | 61 | impl GetCmd { 62 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 63 | match &self { 64 | GetCmd::Format => { 65 | println!("{}", ctx.config.format_style()); 66 | } 67 | GetCmd::Host => { 68 | println!("{}", ctx.config.host().unwrap_or("None")); 69 | } 70 | GetCmd::Token => { 71 | println!("{}", ctx.config.token().unwrap_or("None")); 72 | } 73 | } 74 | 75 | Ok(()) 76 | } 77 | } 78 | 79 | impl SetCmd { 80 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 81 | match &self { 82 | SetCmd::Format { format } => { 83 | ctx.config.set_format(format.clone()); 84 | ctx.config.save()?; 85 | } 86 | SetCmd::Host { host } => { 87 | ctx.config.set_host(host.to_string()); 88 | ctx.config.save()?; 89 | } 90 | } 91 | 92 | Ok(()) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /rfd-cli/src/store/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::{ 6 | fs::{create_dir_all, File as StdFile, OpenOptions}, 7 | io::Write, 8 | path::PathBuf, 9 | }; 10 | 11 | use anyhow::{anyhow, Result}; 12 | use config::{Config, Environment, File}; 13 | use serde::{Deserialize, Serialize}; 14 | 15 | use crate::FormatStyle; 16 | 17 | #[derive(Debug, Deserialize, Serialize)] 18 | pub struct CliConfig { 19 | host: Option, 20 | token: Option, 21 | default_format: Option, 22 | } 23 | 24 | impl CliConfig { 25 | pub fn new() -> Result { 26 | let (path, _) = Self::file(false)?; 27 | let config = Config::builder() 28 | .add_source(File::from(path)) 29 | .add_source(Environment::default()) 30 | .build()?; 31 | 32 | Ok(config.try_deserialize()?) 33 | } 34 | 35 | fn file(clear: bool) -> Result<(PathBuf, StdFile)> { 36 | let mut path = dirs::config_dir().expect("Failed to determine configs path"); 37 | path.push("rfd-cli"); 38 | create_dir_all(&path).expect("Failed to create configs path"); 39 | 40 | path.push("config.toml"); 41 | let file = OpenOptions::new() 42 | .write(true) 43 | .create(true) 44 | .truncate(clear) 45 | .open(&path)?; 46 | 47 | Ok((path, file)) 48 | } 49 | 50 | pub fn host(&self) -> Result<&str> { 51 | self.host.as_ref().map(|s| &**s).ok_or_else(|| { 52 | anyhow!("Host must either be configured via a configuration file or the environment") 53 | }) 54 | } 55 | 56 | pub fn set_host(&mut self, host: String) { 57 | self.host = Some(host); 58 | } 59 | 60 | pub fn token(&self) -> Result<&str> { 61 | self.token.as_ref().map(|s| &**s).ok_or_else(|| { 62 | anyhow!("Token must either be configured via a configuration file or the environment") 63 | }) 64 | } 65 | 66 | pub fn set_token(&mut self, token: String) { 67 | self.token = Some(token); 68 | } 69 | 70 | pub fn format_style(&self) -> FormatStyle { 71 | self.default_format 72 | .as_ref() 73 | .cloned() 74 | .unwrap_or(FormatStyle::Json) 75 | } 76 | 77 | pub fn set_format(&mut self, format: FormatStyle) { 78 | self.default_format = Some(format); 79 | } 80 | 81 | pub fn save(&self) -> Result<()> { 82 | let (_, mut file) = Self::file(true)?; 83 | let _ = file.write_all(toml::to_string(&self)?.as_bytes())?; 84 | 85 | println!("Configuration updated"); 86 | Ok(()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /rfd-cli/src/err.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::{anyhow, Error}; 6 | use rfd_sdk::{types::Error as ApiError, ProgenitorClientError}; 7 | 8 | use crate::{Context, VerbosityLevel}; 9 | 10 | pub fn format_api_err(ctx: &Context, client_err: ProgenitorClientError) -> Error { 11 | let mut err = anyhow!("API Request failed"); 12 | 13 | match client_err { 14 | ProgenitorClientError::CommunicationError(inner) => { 15 | if ctx.verbosity >= VerbosityLevel::All { 16 | err = err.context("Communication Error").context(inner); 17 | } 18 | } 19 | ProgenitorClientError::ErrorResponse(response) => { 20 | if ctx.verbosity >= VerbosityLevel::All { 21 | err = err.context(format!("Status: {}", response.status())); 22 | err = err.context(format!("Headers {:?}", response.headers())); 23 | } 24 | 25 | let response_message = response.into_inner(); 26 | 27 | if ctx.verbosity >= VerbosityLevel::All { 28 | err = err.context(format!("Request {}", response_message.request_id)); 29 | } 30 | 31 | err = err.context(format!( 32 | "Code: {}", 33 | response_message 34 | .error_code 35 | .as_ref() 36 | .map(|s| &**s) 37 | .unwrap_or("") 38 | )); 39 | err = err.context(response_message.message); 40 | } 41 | ProgenitorClientError::InvalidRequest(message) => { 42 | err = err.context("Invalid request").context(message); 43 | } 44 | ProgenitorClientError::InvalidResponsePayload(_, inner) => { 45 | err = err.context("Invalid response").context(inner); 46 | } 47 | ProgenitorClientError::UnexpectedResponse(response) => { 48 | err = err 49 | .context("Unexpected response") 50 | .context(format!("Status: {}", response.status())); 51 | 52 | if ctx.verbosity >= VerbosityLevel::All { 53 | err = err.context(format!("Headers {:?}", response.headers())); 54 | } 55 | } 56 | ProgenitorClientError::ResponseBodyError(inner) => { 57 | err = err.context("Invalid response").context(inner); 58 | } 59 | ProgenitorClientError::InvalidUpgrade(inner) => { 60 | err = err.context("Invalid upgrade").context(inner) 61 | } 62 | ProgenitorClientError::Custom(inner) => { 63 | err = err.context("Internl progenitor error").context(inner) 64 | } 65 | } 66 | 67 | err 68 | } 69 | -------------------------------------------------------------------------------- /rfd-processor/src/scanner.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use diesel::result::{DatabaseErrorKind, Error as DieselError}; 6 | use rfd_github::{GitHubError, GitHubRfdUpdate}; 7 | use rfd_model::{storage::JobStore, NewJob}; 8 | use std::sync::Arc; 9 | use thiserror::Error; 10 | use tokio::time::interval; 11 | use v_model::storage::StoreError; 12 | 13 | use crate::context::Context; 14 | 15 | #[derive(Debug, Error)] 16 | pub enum ScannerError { 17 | #[error(transparent)] 18 | GitHub(#[from] GitHubError), 19 | #[error(transparent)] 20 | Storage(#[from] StoreError), 21 | } 22 | 23 | pub async fn scanner(ctx: Arc) -> Result<(), ScannerError> { 24 | let mut interval = interval(ctx.scanner.interval); 25 | interval.tick().await; 26 | 27 | loop { 28 | if ctx.scanner.enabled { 29 | let updates = ctx 30 | .github 31 | .repository 32 | .get_rfd_sync_updates(&ctx.github.client) 33 | .await?; 34 | 35 | for update in updates { 36 | match JobStore::upsert(&ctx.db.storage, update.clone().into_job()).await { 37 | Ok(job) => tracing::trace!(?job.id, "Added job to the queue"), 38 | Err(err) => { 39 | match err { 40 | StoreError::Db(DieselError::DatabaseError( 41 | DatabaseErrorKind::UniqueViolation, 42 | _, 43 | )) => { 44 | // Nothing to do here, we expect uniqueness conflicts. It is expected 45 | // that the scanner picks ups redundant jobs for RFDs that have not 46 | // changed since the last scan 47 | } 48 | err => { 49 | tracing::warn!(?err, ?update, "Failed to add job") 50 | } 51 | } 52 | } 53 | } 54 | } 55 | } 56 | 57 | interval.tick().await; 58 | } 59 | } 60 | 61 | pub trait IntoJob { 62 | fn into_job(self) -> NewJob; 63 | } 64 | 65 | impl IntoJob for GitHubRfdUpdate { 66 | fn into_job(self) -> NewJob { 67 | NewJob { 68 | owner: self.location.owner, 69 | repository: self.location.repo, 70 | branch: self.location.branch, 71 | sha: self.location.commit.into(), 72 | rfd: self.number.into(), 73 | webhook_delivery_id: None, 74 | committed_at: self.committed_at, 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /rfd-api/src/magic_link.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use minijinja::{context, Environment}; 7 | use reqwest::Url; 8 | use resend_rs::{types::CreateEmailBaseOptions, Resend}; 9 | use v_api::{ 10 | messenger::{Message, Messenger, MessengerError}, 11 | MagicLinkMessage, 12 | }; 13 | 14 | pub struct MagicLinkMessageBuilder { 15 | pub env: Environment<'static>, 16 | } 17 | 18 | impl MagicLinkMessage for MagicLinkMessageBuilder { 19 | fn create_message(&self, recipient: &str, token: &str, url: &Url) -> Option { 20 | let subject = self.env.get_template("subject").ok(); 21 | let text = self.env.get_template("text").ok(); 22 | let html = self.env.get_template("html").ok(); 23 | 24 | if let (Some(subject), Some(text)) = (subject, text) { 25 | Some(Message { 26 | recipient: recipient.to_string(), 27 | subject: subject 28 | .render(context! { 29 | recipient => recipient, 30 | url => url, 31 | }) 32 | .ok(), 33 | text: text 34 | .render(context! { 35 | recipient => recipient, 36 | token => token, 37 | url => url, 38 | }) 39 | .ok()?, 40 | html: html.and_then(|html| { 41 | html.render(context! { 42 | recipient => recipient, 43 | token => token, 44 | url => url, 45 | }) 46 | .ok() 47 | }), 48 | }) 49 | } else { 50 | None 51 | } 52 | } 53 | } 54 | 55 | pub struct ResendMagicLink { 56 | client: Resend, 57 | from: String, 58 | } 59 | 60 | impl ResendMagicLink { 61 | pub fn new(key: String, from: String) -> Self { 62 | Self { 63 | client: Resend::new(&key), 64 | from, 65 | } 66 | } 67 | } 68 | 69 | #[async_trait] 70 | impl Messenger for ResendMagicLink { 71 | async fn send(&self, message: Message) -> Result<(), MessengerError> { 72 | let mut email = CreateEmailBaseOptions::new( 73 | &self.from, 74 | [&message.recipient], 75 | message.subject.unwrap_or_default(), 76 | ); 77 | email = email.with_text(&message.text); 78 | 79 | if let Some(html) = &message.html { 80 | email = email.with_html(html); 81 | } 82 | 83 | self.client.emails.send(email).await?; 84 | 85 | Ok(()) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/auth/oauth.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::time::Duration; 6 | 7 | use anyhow::Result; 8 | use oauth2::basic::BasicClient; 9 | use oauth2::basic::{BasicErrorResponseType, BasicTokenType}; 10 | use oauth2::{ 11 | AuthType, AuthUrl, ClientId, DeviceAuthorizationUrl, EmptyExtraTokenFields, EndpointNotSet, 12 | EndpointSet, RequestTokenError, Scope, StandardErrorResponse, StandardTokenResponse, TokenUrl, 13 | }; 14 | use oauth2::{DeviceCodeErrorResponseType, StandardDeviceAuthorizationResponse}; 15 | use rfd_sdk::types::OAuthProviderInfo; 16 | 17 | type DeviceClient = BasicClient< 18 | // HasAuthUrl 19 | EndpointSet, 20 | // HasDeviceAuthUrl 21 | EndpointSet, 22 | // HasIntrospectionUrl 23 | EndpointNotSet, 24 | // HasRevocationUrl 25 | EndpointNotSet, 26 | // HasTokenUrl 27 | EndpointSet, 28 | >; 29 | 30 | pub struct DeviceOAuth { 31 | client: DeviceClient, 32 | http: reqwest::Client, 33 | scopes: Vec, 34 | } 35 | 36 | impl DeviceOAuth { 37 | pub fn new(provider: OAuthProviderInfo) -> Result { 38 | let device_auth_url = DeviceAuthorizationUrl::new(provider.device_code_endpoint)?; 39 | 40 | let client = BasicClient::new(ClientId::new(provider.client_id)) 41 | .set_auth_uri(AuthUrl::new(provider.auth_url_endpoint)?) 42 | .set_auth_type(AuthType::RequestBody) 43 | .set_token_uri(TokenUrl::new(provider.token_endpoint)?) 44 | .set_device_authorization_url(device_auth_url); 45 | 46 | Ok(Self { 47 | client, 48 | http: reqwest::ClientBuilder::new() 49 | .redirect(reqwest::redirect::Policy::none()) 50 | .build() 51 | .unwrap(), 52 | scopes: provider.scopes, 53 | }) 54 | } 55 | 56 | pub async fn login( 57 | &self, 58 | details: &StandardDeviceAuthorizationResponse, 59 | ) -> Result> { 60 | let token = self 61 | .client 62 | .exchange_device_access_token(&details) 63 | .set_max_backoff_interval(details.interval()) 64 | .request_async(&self.http, tokio::time::sleep, Some(details.expires_in())) 65 | .await; 66 | 67 | Ok(token?) 68 | } 69 | 70 | pub async fn get_device_authorization(&self) -> Result { 71 | let mut req = self.client.exchange_device_code(); 72 | 73 | for scope in &self.scopes { 74 | req = req.add_scope(Scope::new(scope.to_string())); 75 | } 76 | 77 | let res = req.request_async(&self.http).await; 78 | 79 | Ok(res?) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /trace-request/src/lib.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | #![allow(dead_code, unused_imports)] 6 | 7 | extern crate proc_macro; 8 | #[macro_use] 9 | extern crate quote; 10 | 11 | use std::result::Iter; 12 | 13 | use proc_macro::TokenStream; 14 | use proc_macro2::{Delimiter, Group, Span, TokenTree}; 15 | use quote::ToTokens; 16 | use syn::{ 17 | bracketed, 18 | parse::{Parse, ParseStream, Parser}, 19 | parse_macro_input, 20 | punctuated::Punctuated, 21 | Block, DeriveInput, Ident, ItemFn, LitStr, Result, Token, Type, 22 | }; 23 | 24 | #[proc_macro_attribute] 25 | pub fn trace_request(_attr: TokenStream, input: TokenStream) -> TokenStream { 26 | let mut input = parse_macro_input!(input as ItemFn); 27 | let body_block = input.block; 28 | 29 | let wrapped_body_block: TokenStream = quote! { 30 | { 31 | use tracing::Instrument; 32 | 33 | fn get_status(res: &Result) -> http::StatusCode where T: dropshot::HttpCodedResponse { 34 | match res { 35 | Ok(_) => T::STATUS_CODE, 36 | Err(err) => err.status_code.as_status(), 37 | } 38 | } 39 | 40 | let request_id = &rqctx.request_id; 41 | let req = &rqctx.request; 42 | let uri = req.uri(); 43 | let method = req.method(); 44 | 45 | async { 46 | tracing::info!("Request handler start"); 47 | 48 | let start = std::time::Instant::now(); 49 | let result = async #body_block.await; 50 | let end = std::time::Instant::now(); 51 | 52 | let status = get_status(&result); 53 | let duration = end - start; 54 | 55 | match &result { 56 | Ok(_) => tracing::info!( 57 | ?status, 58 | ?duration, 59 | "Request handler complete" 60 | ), 61 | Err(err) => { 62 | tracing::info!( 63 | ?status, 64 | ?duration, 65 | external_message = ?err.external_message, 66 | internal_message = ?err.internal_message, 67 | "Request handler complete" 68 | ) 69 | } 70 | }; 71 | 72 | result 73 | }.instrument(tracing::info_span!("handler", ?request_id, ?method, ?uri)).await 74 | } 75 | }.into(); 76 | 77 | input.block = Box::new(parse_macro_input!(wrapped_body_block as Block)); 78 | 79 | quote! { 80 | #input 81 | } 82 | .into() 83 | } 84 | -------------------------------------------------------------------------------- /rfd-processor/config.example.toml: -------------------------------------------------------------------------------- 1 | # Allowed values: json, pretty 2 | log_format = "json" 3 | 4 | # Controls if the processor should run 5 | processor_enabled = true 6 | 7 | # How many jobs should be processed at once 8 | processor_batch_size = 10 9 | 10 | # How often to select a batch of jobs to process 11 | processor_interval = 30 12 | 13 | # A control mode for all processor actions, designating if the action should persist data to remote 14 | # services or only generate what would be persisted. 15 | processor_update_mode = "read" 16 | 17 | # Maximum number of RFD jobs to process concurrently 18 | processor_capacity = 4 19 | 20 | # Controls if the scanner should run 21 | scanner_enabled = true 22 | 23 | # How often the processor scanner should check the remote GitHub repo for RFDs 24 | scanner_interval = 900 25 | 26 | # The internal database url to store RFD information 27 | database_url = "postgres://:@/" 28 | 29 | # The list of actions that should be run for each processing job 30 | actions = [ 31 | # "CopyImagesToStorage", 32 | # "UpdateSearch", 33 | # "UpdatePdfs", 34 | # "CreatePullRequest", 35 | # "UpdatePullRequest", 36 | # "UpdateDiscussionUrl", 37 | # "EnsureRfdWithPullRequestIsInValidState", 38 | # "EnsureRfdOnDefaultIsInValidState", 39 | ] 40 | 41 | # The method for authenticating to GitHub. This requires one of two authentication styles: 42 | # 1. A GitHub App installation that is defined by an app_id, installation_id, and private_key 43 | # 2. A GitHub access token 44 | # Exactly one authentication must be specified 45 | 46 | # App Installation 47 | [auth.github] 48 | # Numeric GitHub App id 49 | app_id = 1111111 50 | # Numeric GitHub App installation id corresponding to the organization that the configured repo 51 | # belongs to 52 | installation_id = 2222222 53 | # PEM encoded private key for the GitHub App 54 | private_key = """""" 55 | 56 | # Access Token 57 | [auth.github] 58 | # This may be any GitHub access token that has permission to the configured repo 59 | token = "" 60 | 61 | # The GitHub repository to use to read and write RFDs 62 | [source] 63 | # GitHub user or organization 64 | owner = "" 65 | # GitHub repository name 66 | repo = "" 67 | # Path within the repository where RFDs are stored 68 | path = "" 69 | # Branch to use as the default branch of the repository 70 | default_branch = "" 71 | 72 | # Bucket to push static assets pulled from RFDs to (currently only GCP Storage buckets are supported) 73 | [[static_storage]] 74 | # Name of the bucket 75 | bucket = "" 76 | 77 | # Location to store generated PDFs (currently on Google Drive Shared Drives are supported) 78 | [pdf_storage] 79 | # Shared Drive id 80 | drive = "" 81 | # Folder id within the Shared Drive 82 | folder = "" 83 | 84 | # Search backend for indexing RFD contents (currently on Meilisearch is supported) 85 | [[search_storage]] 86 | # Https endpoint of the search instance 87 | host = "" 88 | # API Key for reading and writing documents 89 | key = "" 90 | # Search index to store documents in 91 | index = "" 92 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/copy_images_to_storage.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use google_storage1::api::Object; 7 | use tracing::instrument; 8 | 9 | use crate::{rfd::PersistedRfd, util::decode_base64}; 10 | 11 | use super::{ 12 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 13 | RfdUpdateMode, 14 | }; 15 | 16 | #[derive(Debug)] 17 | pub struct CopyImagesToStorage; 18 | 19 | #[async_trait] 20 | impl RfdUpdateAction for CopyImagesToStorage { 21 | #[instrument(skip(self, ctx, _new), err(Debug))] 22 | async fn run( 23 | &self, 24 | ctx: &mut RfdUpdateActionContext, 25 | _new: &mut PersistedRfd, 26 | mode: RfdUpdateMode, 27 | ) -> Result { 28 | let RfdUpdateActionContext { ctx, update, .. } = ctx; 29 | 30 | let images = update 31 | .location 32 | .download_supporting_documents(&ctx.github.client, &update.number) 33 | .await 34 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 35 | 36 | for image in images { 37 | let sub_path = image 38 | .path 39 | .replace(&format!("rfd/{}/", update.number.as_number_string()), ""); 40 | let object_name = format!("rfd/{}/latest/{}", update.number, sub_path); 41 | let mime_type = mime_guess::from_path(&object_name).first_or_octet_stream(); 42 | let data = decode_base64(&image.content) 43 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 44 | 45 | tracing::info!( 46 | ?object_name, 47 | ?mime_type, 48 | size = data.len(), 49 | "Writing file to storage buckets" 50 | ); 51 | 52 | let cursor = std::io::Cursor::new(data); 53 | 54 | for location in &ctx.assets.locations { 55 | tracing::info!(bucket = ?location.bucket, ?object_name, "Writing to location"); 56 | 57 | if mode == RfdUpdateMode::Write { 58 | // TODO: Move implementation to a trait and abstract over different storage systems 59 | if let Err(err) = ctx 60 | .assets 61 | .client 62 | .objects() 63 | .insert(Object::default(), &location.bucket) 64 | .name(&object_name) 65 | .upload(cursor.clone(), mime_type.clone()) 66 | .await 67 | { 68 | tracing::error!(?err, "Failed to upload static file to GCP"); 69 | } 70 | } 71 | } 72 | } 73 | 74 | Ok(RfdUpdateActionResponse::default()) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /rfd-api/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rfd-api" 3 | version = "0.12.3" 4 | edition = "2021" 5 | repository = "https://github.com/oxidecomputer/rfd-api" 6 | 7 | [features] 8 | local-dev = ["v-api/local-dev"] 9 | 10 | [dependencies] 11 | anyhow = { workspace = true } 12 | async-bb8-diesel = { workspace = true } 13 | async-trait = { workspace = true } 14 | base64 = { workspace = true } 15 | bb8 = { workspace = true } 16 | chrono = { workspace = true, features = ["serde"] } 17 | config = { workspace = true } 18 | cookie = { workspace = true } 19 | crc32c = { workspace = true } 20 | diesel = { workspace = true } 21 | dropshot = { workspace = true } 22 | dropshot-authorization-header = { path = "../dropshot-authorization-header" } 23 | dropshot-verified-body = { workspace = true, features = ["github"] } 24 | hex = { workspace = true } 25 | http = { workspace = true } 26 | hyper = { workspace = true } 27 | jsonwebtoken = { workspace = true } 28 | meilisearch-sdk = { workspace = true } 29 | minijinja = { workspace = true } 30 | newtype-uuid = { workspace = true } 31 | oauth2 = { workspace = true } 32 | octorust = { workspace = true, features = ["httpcache"] } 33 | partial-struct = { workspace = true } 34 | rand = { workspace = true, features = ["std"] } 35 | rand_core = { workspace = true, features = ["std"] } 36 | regex = { workspace = true } 37 | reqwest = { workspace = true } 38 | reqwest-middleware = { workspace = true } 39 | reqwest-retry = { workspace = true } 40 | reqwest-tracing = { workspace = true } 41 | resend-rs = { workspace = true } 42 | ring = { workspace = true } 43 | rfd-data = { path = "../rfd-data" } 44 | rfd-github = { path = "../rfd-github" } 45 | rfd-model = { path = "../rfd-model" } 46 | rsa = { workspace = true, features = ["sha2"] } 47 | schemars = { workspace = true, features = ["chrono"] } 48 | secrecy = { workspace = true, features = ["serde"] } 49 | semver = { workspace = true } 50 | serde = { workspace = true, features = ["derive"] } 51 | serde_json = { workspace = true } 52 | serde_urlencoded = { workspace = true } 53 | sha2 = { workspace = true } 54 | slog = { workspace = true } 55 | slog-async = { workspace = true } 56 | tap = { workspace = true } 57 | thiserror = { workspace = true } 58 | tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } 59 | toml = { workspace = true } 60 | trace-request = { path = "../trace-request" } 61 | tracing = { workspace = true } 62 | tracing-appender = { workspace = true } 63 | tracing-slog = { workspace = true } 64 | tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "json"] } 65 | uuid = { workspace = true, features = ["v4", "serde"] } 66 | v-api = { workspace = true } 67 | v-model = { workspace = true } 68 | v-api-permission-derive = { workspace = true } 69 | yup-oauth2 = { workspace = true } 70 | 71 | [dev-dependencies] 72 | async-trait = { workspace = true } 73 | mockall = { workspace = true } 74 | rfd-model = { path = "../rfd-model", features = ["mock"] } 75 | rsa = { workspace = true, features = ["pem"] } 76 | 77 | [package.metadata.dist] 78 | targets = ["x86_64-unknown-linux-gnu"] 79 | -------------------------------------------------------------------------------- /rfd-model/src/schema.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | pub mod sql_types { 6 | #[derive(diesel::sql_types::SqlType)] 7 | #[diesel(postgres_type(name = "rfd_content_format"))] 8 | pub struct RfdContentFormat; 9 | 10 | #[derive(diesel::sql_types::SqlType)] 11 | #[diesel(postgres_type(name = "rfd_pdf_source"))] 12 | pub struct RfdPdfSource; 13 | 14 | #[derive(diesel::sql_types::SqlType)] 15 | #[diesel(postgres_type(name = "rfd_visibility"))] 16 | pub struct RfdVisibility; 17 | } 18 | 19 | diesel::table! { 20 | job (id) { 21 | id -> Int4, 22 | owner -> Varchar, 23 | repository -> Varchar, 24 | branch -> Varchar, 25 | sha -> Varchar, 26 | rfd -> Int4, 27 | webhook_delivery_id -> Nullable, 28 | committed_at -> Timestamptz, 29 | processed -> Bool, 30 | created_at -> Timestamptz, 31 | started_at -> Nullable, 32 | } 33 | } 34 | 35 | diesel::table! { 36 | use diesel::sql_types::*; 37 | use super::sql_types::RfdVisibility; 38 | 39 | rfd (id) { 40 | id -> Uuid, 41 | rfd_number -> Int4, 42 | link -> Nullable, 43 | created_at -> Timestamptz, 44 | updated_at -> Timestamptz, 45 | deleted_at -> Nullable, 46 | visibility -> RfdVisibility, 47 | } 48 | } 49 | 50 | diesel::table! { 51 | use diesel::sql_types::*; 52 | use super::sql_types::RfdPdfSource; 53 | 54 | rfd_pdf (id) { 55 | id -> Uuid, 56 | rfd_revision_id -> Uuid, 57 | source -> RfdPdfSource, 58 | link -> Varchar, 59 | created_at -> Timestamptz, 60 | updated_at -> Timestamptz, 61 | deleted_at -> Nullable, 62 | rfd_id -> Uuid, 63 | external_id -> Varchar, 64 | } 65 | } 66 | 67 | diesel::table! { 68 | use diesel::sql_types::*; 69 | use super::sql_types::RfdContentFormat; 70 | 71 | rfd_revision (id) { 72 | id -> Uuid, 73 | rfd_id -> Uuid, 74 | title -> Varchar, 75 | state -> Nullable, 76 | discussion -> Nullable, 77 | authors -> Nullable, 78 | content -> Varchar, 79 | content_format -> RfdContentFormat, 80 | sha -> Varchar, 81 | commit_sha -> Varchar, 82 | committed_at -> Timestamptz, 83 | created_at -> Timestamptz, 84 | updated_at -> Timestamptz, 85 | deleted_at -> Nullable, 86 | labels -> Nullable, 87 | major_change -> Bool, 88 | } 89 | } 90 | 91 | diesel::joinable!(rfd_pdf -> rfd (rfd_id)); 92 | diesel::joinable!(rfd_pdf -> rfd_revision (rfd_revision_id)); 93 | diesel::joinable!(rfd_revision -> rfd (rfd_id)); 94 | 95 | diesel::allow_tables_to_appear_in_same_query!( 96 | job, 97 | rfd, 98 | rfd_pdf, 99 | rfd_revision, 100 | ); 101 | -------------------------------------------------------------------------------- /rfd-data/src/lib.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::fmt::Display; 6 | 7 | use schemars::JsonSchema; 8 | use serde::{Deserialize, Serialize}; 9 | 10 | pub mod content; 11 | 12 | #[derive(Debug, Copy, Clone)] 13 | pub struct RfdNumber(i32); 14 | 15 | impl RfdNumber { 16 | /// Get the path to where the source contents of this RFD exists in the RFD repo. 17 | pub fn repo_path(&self) -> String { 18 | format!("/rfd/{}", self.as_number_string()) 19 | } 20 | 21 | /// Get a RFD number in its expanded form with leading 0s 22 | pub fn as_number_string(&self) -> String { 23 | let mut number_string = self.0.to_string(); 24 | while number_string.len() < 4 { 25 | number_string = format!("0{}", number_string); 26 | } 27 | 28 | number_string 29 | } 30 | } 31 | 32 | impl Display for RfdNumber { 33 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 34 | write!(f, "{}", self.0) 35 | } 36 | } 37 | 38 | impl From for RfdNumber { 39 | fn from(num: i32) -> Self { 40 | Self(num) 41 | } 42 | } 43 | 44 | impl From<&i32> for RfdNumber { 45 | fn from(num: &i32) -> Self { 46 | Self(*num) 47 | } 48 | } 49 | 50 | impl From for i32 { 51 | fn from(num: RfdNumber) -> Self { 52 | num.0 53 | } 54 | } 55 | 56 | impl From<&RfdNumber> for i32 { 57 | fn from(num: &RfdNumber) -> Self { 58 | num.0 59 | } 60 | } 61 | 62 | #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] 63 | #[serde(rename_all = "kebab-case")] 64 | pub enum RfdState { 65 | Abandoned, 66 | Committed, 67 | Discussion, 68 | Ideation, 69 | Prediscussion, 70 | Published, 71 | } 72 | 73 | impl Display for RfdState { 74 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 75 | match self { 76 | RfdState::Abandoned => write!(f, "abandoned"), 77 | RfdState::Committed => write!(f, "committed"), 78 | RfdState::Discussion => write!(f, "discussion"), 79 | RfdState::Ideation => write!(f, "ideation"), 80 | RfdState::Prediscussion => write!(f, "prediscussion"), 81 | RfdState::Published => write!(f, "published"), 82 | } 83 | } 84 | } 85 | 86 | #[derive(Debug)] 87 | pub struct InvalidRfdState<'a>(pub &'a str); 88 | 89 | impl<'a> TryFrom<&'a str> for RfdState { 90 | type Error = InvalidRfdState<'a>; 91 | fn try_from(value: &'a str) -> Result { 92 | match value { 93 | "abandoned" => Ok(RfdState::Abandoned), 94 | "committed" => Ok(RfdState::Committed), 95 | "discussion" => Ok(RfdState::Discussion), 96 | "ideation" => Ok(RfdState::Ideation), 97 | "prediscussion" => Ok(RfdState::Prediscussion), 98 | "published" => Ok(RfdState::Published), 99 | _ => Err(InvalidRfdState(value)), 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/shortcut/mapper.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::Result; 6 | use clap::{Parser, Subcommand}; 7 | use progenitor_client::Error; 8 | use rfd_sdk::types::RfdPermission; 9 | use serde_json::json; 10 | use uuid::Uuid; 11 | 12 | use crate::{printer::CliOutput, Context}; 13 | 14 | #[derive(Debug, Parser)] 15 | pub struct MapperShortcut { 16 | #[clap(subcommand)] 17 | pub mapper: MapperShortcuts, 18 | } 19 | 20 | #[derive(Debug, Subcommand)] 21 | pub enum MapperShortcuts { 22 | Email(EmailMapper), 23 | #[command(name = "github")] 24 | GitHub(GitHubMapper), 25 | } 26 | 27 | #[derive(Debug, Parser)] 28 | /// Add a new one-time mapping for for a GitHub user 29 | pub struct GitHubMapper { 30 | /// The GitHub username of the user to create a mapping rule for 31 | #[clap(index = 1)] 32 | username: String, 33 | /// The group to add this user to 34 | #[clap(index = 2)] 35 | group: String, 36 | } 37 | 38 | #[derive(Debug, Parser)] 39 | /// Add a new one-time mapping for for an email address 40 | pub struct EmailMapper { 41 | /// The email of the user to create a mapping rule for 42 | #[clap(index = 1)] 43 | email: String, 44 | /// The group to add this user to 45 | #[clap(index = 2)] 46 | group: String, 47 | } 48 | 49 | impl GitHubMapper { 50 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 51 | let mut request = ctx.client()?.create_mapper(); 52 | request = request.body_map(|body| { 53 | body.max_activations(1) 54 | .name(format!("map-github-{}", Uuid::new_v4())) 55 | .rule(json!({ 56 | "rule": "github_username", 57 | "github_username": self.username.clone(), 58 | "groups": vec![self.group.to_string()], 59 | "permissions": Vec::::new(), 60 | })) 61 | }); 62 | 63 | let result = request.send().await; 64 | match result { 65 | Ok(r) => ctx.printer()?.output_mapper(r.into_inner()), 66 | // Err(r) => ctx.printer()?.output_create_mapper(Err(r)), 67 | _ => (), 68 | } 69 | 70 | Ok(()) 71 | } 72 | } 73 | 74 | impl EmailMapper { 75 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 76 | let mut request = ctx.client()?.create_mapper(); 77 | request = request.body_map(|body| { 78 | body.max_activations(1) 79 | .name(format!("map-email-{}", Uuid::new_v4())) 80 | .rule(json!({ 81 | "rule": "email_address", 82 | "email": self.email.clone(), 83 | "groups": vec![self.group.to_string()], 84 | "permissions": Vec::::new(), 85 | })) 86 | }); 87 | 88 | let result = request.send().await; 89 | match result { 90 | Ok(r) => ctx.printer()?.output_mapper(r.into_inner()), 91 | // Err(r) => ctx.printer()?.output_create_mapper(Err(r)), 92 | _ => (), 93 | } 94 | 95 | Ok(()) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /rfd-github/src/ext.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use base64::{prelude::BASE64_STANDARD, DecodeError, Engine}; 7 | use octorust::{repos::Repos, types::ContentFile, ClientError}; 8 | use tracing::instrument; 9 | 10 | #[async_trait] 11 | pub trait ReposExt { 12 | async fn get_content_blob( 13 | &self, 14 | owner: &str, 15 | repo: &str, 16 | branch: &str, 17 | file: &str, 18 | ) -> Result; 19 | } 20 | 21 | #[async_trait] 22 | impl ReposExt for Repos { 23 | #[instrument(skip(self))] 24 | async fn get_content_blob( 25 | &self, 26 | owner: &str, 27 | repo: &str, 28 | ref_: &str, 29 | file: &str, 30 | ) -> Result { 31 | tracing::trace!("Fetching content from GitHub"); 32 | let mut file = self.get_content_file(owner, repo, file, ref_).await?.body; 33 | 34 | // If the content is empty and the encoding is none then we likely hit a "too large" file case. 35 | // Try requesting the blob directly 36 | if file.content.is_empty() && file.encoding == "none" { 37 | let blob = self 38 | .client 39 | .git() 40 | .get_blob(owner, repo, &file.sha) 41 | .await? 42 | .body; 43 | 44 | // We are only interested in the copying over the content and encoding fields, everything 45 | // else from the original response should still be valid 46 | file.content = blob.content; 47 | file.encoding = blob.encoding; 48 | } 49 | 50 | Ok(file) 51 | } 52 | } 53 | 54 | pub trait ContentFileExt { 55 | fn is_empty(&self) -> bool; 56 | fn decode(&self) -> Result, DecodeError>; 57 | fn to_text(&self) -> Option; 58 | } 59 | 60 | impl ContentFileExt for ContentFile { 61 | fn is_empty(&self) -> bool { 62 | self.content.is_empty() && self.sha.is_empty() 63 | } 64 | 65 | fn decode(&self) -> Result, DecodeError> { 66 | BASE64_STANDARD 67 | .decode(self.content.replace('\n', "")) 68 | .map(|data| data.trim().to_vec()) 69 | } 70 | 71 | fn to_text(&self) -> Option { 72 | self.decode() 73 | .ok() 74 | .and_then(|content| String::from_utf8(content).ok()) 75 | } 76 | } 77 | 78 | trait SliceExt { 79 | fn trim(&self) -> Self; 80 | } 81 | 82 | impl SliceExt for Vec { 83 | fn trim(&self) -> Vec { 84 | fn is_whitespace(c: &u8) -> bool { 85 | c == &b'\t' || c == &b' ' 86 | } 87 | 88 | fn is_not_whitespace(c: &u8) -> bool { 89 | !is_whitespace(c) 90 | } 91 | 92 | if let Some(first) = self.iter().position(is_not_whitespace) { 93 | if let Some(last) = self.iter().rposition(is_not_whitespace) { 94 | self[first..last + 1].to_vec() 95 | } else { 96 | unreachable!(); 97 | } 98 | } else { 99 | vec![] 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /rfd-api/src/permissions.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use schemars::JsonSchema; 6 | use serde::{Deserialize, Serialize}; 7 | use std::collections::BTreeSet; 8 | use v_api::permissions::VPermission; 9 | use v_api_permission_derive::v_api; 10 | 11 | #[v_api(From(VPermission))] 12 | #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] 13 | pub enum RfdPermission { 14 | #[v_api( 15 | contract(kind = append, variant = GetRfds), 16 | scope(to = "rfd:content:r") 17 | )] 18 | GetRfd(i32), 19 | #[v_api( 20 | contract(kind = extend, variant = GetRfds), 21 | expand(kind = iter, variant = GetRfd) 22 | scope(to = "rfd:content:r") 23 | )] 24 | GetRfds(BTreeSet), 25 | #[v_api( 26 | expand(kind = alias, variant = GetRfd, source = actor), 27 | scope(to = "rfd:content:r", from = "rfd:content:r") 28 | )] 29 | GetRfdsAssigned, 30 | #[v_api(scope(to = "rfd:content:r", from = "rfd:content:r"))] 31 | GetRfdsAll, 32 | #[v_api(scope(to = "rfd:content:w", from = "rfd:content:w"))] 33 | CreateRfd, 34 | #[v_api( 35 | contract(kind = append, variant = UpdateRfds), 36 | scope(to = "rfd:content:w") 37 | )] 38 | UpdateRfd(i32), 39 | #[v_api( 40 | contract(kind = extend, variant = UpdateRfds), 41 | expand(kind = iter, variant = UpdateRfd) 42 | scope(to = "rfd:content:w") 43 | )] 44 | UpdateRfds(BTreeSet), 45 | #[v_api( 46 | expand(kind = alias, variant = UpdateRfd, source = actor), 47 | scope(to = "rfd:content:w", from = "rfd:content:w") 48 | )] 49 | UpdateRfdsAssigned, 50 | #[v_api(scope(to = "rfd:content:w", from = "rfd:content:w"))] 51 | UpdateRfdsAll, 52 | #[v_api( 53 | contract(kind = append, variant = ManageRfdsVisibility), 54 | scope(to = "rfd:visibility:w") 55 | )] 56 | ManageRfdVisibility(i32), 57 | #[v_api( 58 | contract(kind = extend, variant = ManageRfdsVisibility), 59 | expand(kind = iter, variant = ManageRfdVisibility) 60 | scope(to = "rfd:visibility:w") 61 | )] 62 | ManageRfdsVisibility(BTreeSet), 63 | #[v_api( 64 | expand(kind = alias, variant = ManageRfdVisibility, source = actor), 65 | scope(to = "rfd:visibility:w", from = "rfd:visibility:w") 66 | )] 67 | ManageRfdsVisibilityAssigned, 68 | #[v_api(scope(to = "rfd:visibility:w", from = "rfd:visibility:w"))] 69 | ManageRfdsVisibilityAll, 70 | #[v_api( 71 | contract(kind = append, variant = GetDiscussions), 72 | scope(to = "rfd:discussion:r") 73 | )] 74 | GetDiscussion(i32), 75 | #[v_api( 76 | contract(kind = extend, variant = GetDiscussions), 77 | expand(kind = iter, variant = GetDiscussion) 78 | scope(to = "rfd:discussion:r") 79 | )] 80 | GetDiscussions(BTreeSet), 81 | #[v_api( 82 | expand(kind = alias, variant = GetDiscussion, source = actor), 83 | scope(to = "rfd:discussion:r", from = "rfd:discussion:r") 84 | )] 85 | GetDiscussionsAssigned, 86 | #[v_api(scope(to = "rfd:discussion:r", from = "rfd:discussion:r"))] 87 | GetDiscussionsAll, 88 | #[v_api(scope(to = "search", from = "search"))] 89 | SearchRfds, 90 | } 91 | -------------------------------------------------------------------------------- /rfd-api/src/config.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::{collections::HashMap, path::PathBuf}; 6 | 7 | use config::{Config, ConfigError, Environment, File}; 8 | use rfd_data::content::RfdTemplate; 9 | use serde::Deserialize; 10 | use v_api::config::{AsymmetricKey, AuthnProviders, JwtConfig}; 11 | use v_model::schema_ext::MagicLinkMedium; 12 | 13 | use crate::server::SpecConfig; 14 | 15 | #[derive(Debug, Deserialize)] 16 | pub struct AppConfig { 17 | pub log_format: ServerLogFormat, 18 | pub log_directory: Option, 19 | pub initial_mappers: Option, 20 | pub public_url: String, 21 | pub server_port: u16, 22 | pub database_url: String, 23 | pub keys: Vec, 24 | pub jwt: JwtConfig, 25 | pub spec: Option, 26 | pub authn: AuthnProviders, 27 | pub magic_link: MagicLinkConfig, 28 | pub search: SearchConfig, 29 | pub content: ContentConfig, 30 | pub services: ServicesConfig, 31 | } 32 | 33 | #[derive(Debug, Deserialize)] 34 | #[serde(rename_all = "kebab-case")] 35 | pub enum ServerLogFormat { 36 | Json, 37 | Pretty, 38 | } 39 | 40 | #[derive(Debug, Default, Deserialize)] 41 | pub struct SearchConfig { 42 | pub host: String, 43 | pub key: String, 44 | pub index: String, 45 | } 46 | 47 | #[derive(Debug, Default, Deserialize)] 48 | pub struct ContentConfig { 49 | pub templates: HashMap, 50 | } 51 | 52 | #[derive(Debug, Deserialize)] 53 | pub struct ServicesConfig { 54 | pub github: GitHubConfig, 55 | } 56 | 57 | #[derive(Debug, Deserialize)] 58 | pub struct GitHubConfig { 59 | pub auth: GitHubAuthConfig, 60 | pub owner: String, 61 | pub path: String, 62 | pub repo: String, 63 | pub default_branch: String, 64 | } 65 | 66 | #[derive(Debug, Deserialize)] 67 | #[serde(untagged)] 68 | pub enum GitHubAuthConfig { 69 | Installation { 70 | app_id: i64, 71 | installation_id: i64, 72 | private_key: String, 73 | }, 74 | User { 75 | token: String, 76 | }, 77 | } 78 | 79 | #[derive(Debug, Deserialize)] 80 | pub struct MagicLinkConfig { 81 | pub templates: Vec, 82 | pub email_service: Option, 83 | } 84 | 85 | #[derive(Debug, Deserialize)] 86 | pub struct MagicLinkTemplate { 87 | pub medium: MagicLinkMedium, 88 | pub channel: String, 89 | pub from: String, 90 | pub subject: Option, 91 | pub text: String, 92 | pub html: Option, 93 | } 94 | 95 | #[derive(Debug, Deserialize)] 96 | #[serde(rename_all = "lowercase")] 97 | pub enum EmailService { 98 | Resend { key: String }, 99 | } 100 | 101 | impl AppConfig { 102 | pub fn new(config_sources: Option>) -> Result { 103 | let mut config = Config::builder() 104 | .add_source(File::with_name("config.toml").required(false)) 105 | .add_source(File::with_name("rfd-api/config.toml").required(false)); 106 | 107 | for source in config_sources.unwrap_or_default() { 108 | config = config.add_source(File::with_name(&source).required(false)); 109 | } 110 | 111 | config 112 | .add_source(Environment::default()) 113 | .build()? 114 | .try_deserialize() 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/update_discussion_url.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use std::cmp::Ordering; 7 | use tracing::instrument; 8 | 9 | use crate::rfd::PersistedRfd; 10 | 11 | use super::{ 12 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 13 | RfdUpdateMode, 14 | }; 15 | 16 | #[derive(Debug)] 17 | pub struct UpdateDiscussionUrl; 18 | 19 | #[async_trait] 20 | impl RfdUpdateAction for UpdateDiscussionUrl { 21 | #[instrument(skip(self, ctx, new), err(Debug))] 22 | async fn run( 23 | &self, 24 | ctx: &mut RfdUpdateActionContext, 25 | new: &mut PersistedRfd, 26 | _mode: RfdUpdateMode, 27 | ) -> Result { 28 | let RfdUpdateActionContext { pull_requests, .. } = ctx; 29 | 30 | let mut requires_source_commit = false; 31 | 32 | // We only want to operate on open pull requests 33 | let open_prs = pull_requests 34 | .iter() 35 | .filter(|pr| pr.state == "open") 36 | .collect::>(); 37 | 38 | // Explicitly we will only update a pull request if it is the only open pull request for the 39 | // branch that we are working on 40 | match open_prs.len().cmp(&1) { 41 | Ordering::Equal => { 42 | if let Some(pull_request) = open_prs.get(0) { 43 | tracing::debug!(current = ?new.revision.discussion, pr = ?pull_request.html_url, "Found discussion url for pull request. Testing if it matches the current url"); 44 | 45 | // If the stored discussion link does not match the PR we found, then and 46 | // update is required 47 | if !pull_request.html_url.is_empty() 48 | && new 49 | .revision 50 | .discussion 51 | .as_ref() 52 | .map(|link| link != &pull_request.html_url) 53 | .unwrap_or(true) 54 | { 55 | tracing::info!( 56 | new.revision.discussion, 57 | pull_request.html_url, 58 | "Stored discussion link does not match the pull request found" 59 | ); 60 | 61 | new.update_discussion(&pull_request.html_url) 62 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 63 | 64 | tracing::info!("Updated RFD file in GitHub with discussion link change"); 65 | 66 | requires_source_commit = true; 67 | } 68 | } 69 | } 70 | Ordering::Greater => { 71 | tracing::warn!( 72 | "Found multiple open pull requests for RFD. Unable to update discussion url" 73 | ); 74 | } 75 | Ordering::Less => { 76 | // Nothing to do, there are no PRs 77 | tracing::debug!( 78 | "No pull requests found for RFD. Discussion url can not be updated" 79 | ); 80 | } 81 | } 82 | 83 | Ok(RfdUpdateActionResponse { 84 | requires_source_commit, 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/create_pull_request.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use tracing::instrument; 7 | 8 | use crate::rfd::PersistedRfd; 9 | 10 | use super::{ 11 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 12 | RfdUpdateMode, 13 | }; 14 | 15 | #[derive(Debug)] 16 | pub struct CreatePullRequest; 17 | 18 | #[async_trait] 19 | impl RfdUpdateAction for CreatePullRequest { 20 | #[instrument(skip(self, ctx, new), err(Debug))] 21 | async fn run( 22 | &self, 23 | ctx: &mut RfdUpdateActionContext, 24 | new: &mut PersistedRfd, 25 | mode: RfdUpdateMode, 26 | ) -> Result { 27 | let RfdUpdateActionContext { 28 | ctx, 29 | update, 30 | pull_requests, 31 | previous, 32 | .. 33 | } = ctx; 34 | 35 | // We only ever create pull requests if the RFD is in the discussion state, we are not 36 | // handling an update on the default branch, and there are no previous pull requests for 37 | // for this branch. This includes Closed pull requests, therefore this action will not 38 | // re-open or create a new pull request for a branch that previously had an open PR 39 | if update.location.branch != update.location.default_branch 40 | && new.is_state("discussion") 41 | && pull_requests.is_empty() 42 | { 43 | tracing::info!("RFD is in the discussion state but there are no open pull requests, creating a new pull request"); 44 | 45 | if mode == RfdUpdateMode::Write { 46 | let pull = ctx 47 | .github 48 | .client 49 | .pulls() 50 | .create( 51 | &update.location.owner, 52 | &update.location.repo, 53 | &octorust::types::PullsCreateRequest { 54 | title: new.name(), 55 | head: format!( 56 | "{}:{}", 57 | ctx.github.repository.owner, update.location.branch 58 | ), 59 | base: update.location.default_branch.to_string(), 60 | body: "Automatically opening the pull request since the document \ 61 | is marked as being in discussion. If you wish to not have \ 62 | a pull request open, change the state of your document and \ 63 | close this pull request." 64 | .to_string(), 65 | draft: Some(false), 66 | maintainer_can_modify: Some(true), 67 | issue: 0, 68 | }, 69 | ) 70 | .await 71 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))? 72 | .body; 73 | 74 | tracing::info!( 75 | old_state = ?previous.map(|rfd| &rfd.revision.state), new_state = new.revision.state, new_pr = pull.number, 76 | "Opened new pull request for discussion" 77 | ); 78 | 79 | // Add the newly created pull request into the context for future actions 80 | pull_requests.push(pull.into()); 81 | } 82 | } else { 83 | tracing::debug!("RFD does not require a pull request or one already exists"); 84 | } 85 | 86 | Ok(RfdUpdateActionResponse::default()) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/process_includes.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use rfd_data::content::{RfdContent, RfdDocument}; 7 | use rfd_github::ext::ContentFileExt; 8 | use rfd_model::schema_ext::ContentFormat; 9 | use tracing::instrument; 10 | 11 | use crate::rfd::PersistedRfd; 12 | 13 | use super::{ 14 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 15 | RfdUpdateMode, 16 | }; 17 | 18 | #[derive(Debug)] 19 | pub struct ProcessIncludes; 20 | 21 | #[async_trait] 22 | impl RfdUpdateAction for ProcessIncludes { 23 | #[instrument(skip(self, ctx, new), err(Debug))] 24 | async fn run( 25 | &self, 26 | ctx: &mut RfdUpdateActionContext, 27 | new: &mut PersistedRfd, 28 | _mode: RfdUpdateMode, 29 | ) -> Result { 30 | tracing::info!("Processing inculdes"); 31 | let RfdUpdateActionContext { ctx, update, .. } = ctx; 32 | 33 | let content = new 34 | .content() 35 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 36 | let (format, new_content) = match &content.content { 37 | RfdContent::Asciidoc(adoc) => { 38 | let mut raw = adoc.raw().to_string(); 39 | let includes = adoc.includes(); 40 | 41 | // Ensure that we only do the work of downloading supporting documents if there 42 | // are include macros to process 43 | if includes.len() > 0 { 44 | let documents = update 45 | .location 46 | .download_supporting_documents(&ctx.github.client, &update.number) 47 | .await 48 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 49 | 50 | tracing::trace!(?documents, "Retrieved supporting documents from GitHub"); 51 | for include in includes { 52 | tracing::info!(?include, "Handle include"); 53 | 54 | if let Some(document) = documents.iter().find(|document| { 55 | let trimmed_path = document 56 | .path 57 | .trim_start_matches(&ctx.github.repository.path) 58 | .trim_start_matches('/') 59 | .trim_start_matches(&update.number.as_number_string()) 60 | .trim_start_matches('/'); 61 | 62 | tracing::debug!(path = ?document.path, ?trimmed_path, name = include.name(), "Test include name against path"); 63 | trimmed_path == include.name() 64 | }) { 65 | if let Some(content) = document.to_text() { 66 | tracing::info!(name = include.name(), file = document.name, "Found include match. Replacing contents"); 67 | raw = include.perform_replacement(&raw, &content); 68 | } else { 69 | tracing::warn!(?include, "Non UTF-8 files can not be included") 70 | } 71 | } 72 | } 73 | } 74 | 75 | (ContentFormat::Asciidoc, raw) 76 | } 77 | RfdContent::Markdown(_) => (ContentFormat::Markdown, content.raw().to_string()), 78 | }; 79 | 80 | tracing::trace!("Processed all includes"); 81 | new.set_content(format, &new_content); 82 | 83 | Ok(RfdUpdateActionResponse { 84 | requires_source_commit: false, 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "dropshot-authorization-header", 4 | "parse-rfd", 5 | "rfd-api", 6 | "rfd-cli", 7 | "rfd-data", 8 | "rfd-github", 9 | "rfd-installer", 10 | "rfd-model", 11 | "rfd-processor", 12 | "rfd-sdk", 13 | "trace-request", 14 | "xtask" 15 | ] 16 | resolver = "2" 17 | 18 | [workspace.dependencies] 19 | anyhow = "1.0.100" 20 | async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel" } 21 | async-trait = "0.1.88" 22 | base64 = "0.22" 23 | bb8 = "0.8.6" 24 | chrono = "0.4.41" 25 | clap = { version = "4.5.42", features = ["derive", "string", "env"] } 26 | config = { version = "0.15.13", features = ["toml"] } 27 | cookie = { version = "0.18.1" } 28 | crc32c = "0.6.8" 29 | diesel = { version = "2.2.12", features = ["postgres"] } 30 | diesel_migrations = { version = "2.2.0" } 31 | dirs = "6.0.0" 32 | dropshot = "0.16" 33 | dropshot-verified-body = { git = "https://github.com/oxidecomputer/dropshot-verified-body" } 34 | futures = "0.3.31" 35 | google-drive3 = "6" 36 | google-storage1 = "6" 37 | hex = "0.4.3" 38 | hmac = "0.12.1" 39 | http = "1.3.1" 40 | hyper = "1.6.0" 41 | itertools = "0.13.0" 42 | jsonwebtoken = "9" 43 | meilisearch-sdk = "0.28.0" 44 | md-5 = "0.10.6" 45 | mime_guess = "2.0.5" 46 | minijinja = { version = "2.11.0", features = ["loader"] } 47 | mockall = "0.13.1" 48 | newline-converter = "0.3.0" 49 | newtype-uuid = { version = "1.2.4", features = ["schemars08", "serde", "v4"] } 50 | oauth2 = { version = "5.0.0", default-features = false, features = ["rustls-tls"] } 51 | octorust = "0.10.0" 52 | owo-colors = "4.2.2" 53 | partial-struct = { git = "https://github.com/oxidecomputer/partial-struct" } 54 | progenitor = { git = "https://github.com/oxidecomputer/progenitor" } 55 | progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" } 56 | rand = "0.8.5" 57 | rand_core = "0.6" 58 | regex = "1.11.1" 59 | reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] } 60 | reqwest-middleware = "0.4" 61 | reqwest-retry = "0.7" 62 | reqwest-tracing = "0.5" 63 | resend-rs = { version = "0.15.0", default-features = false, features = ["rustls-tls"] } 64 | ring = "0.17.14" 65 | rsa = "0.9.8" 66 | rustfmt-wrapper = "0.2.1" 67 | schemars = "0.8.22" 68 | secrecy = "0.10.3" 69 | semver = "1.0.26" 70 | serde = "1" 71 | serde_bytes = "0.11.17" 72 | serde_json = "1" 73 | serde_urlencoded = "0.7.1" 74 | sha2 = "0.10.9" 75 | similar = "2.7.0" 76 | slog = "2.7.0" 77 | slog-async = "2.8.0" 78 | tabwriter = "1.4.1" 79 | tap = "1.0.1" 80 | textwrap = "0.16.2" 81 | thiserror = "2" 82 | tokio = "1.47.1" 83 | toml = "0.9.8" 84 | tracing = "0.1.41" 85 | tracing-appender = "0.2.3" 86 | tracing-slog = { git = "https://github.com/oxidecomputer/tracing-slog", default-features = false } 87 | tracing-subscriber = "0.3.20" 88 | uuid = { version = "1.17.0", features = ["serde"] } 89 | valuable = "0.1.1" 90 | v-api = { git = "https://github.com/oxidecomputer/v-api" } 91 | v-api-installer = { git = "https://github.com/oxidecomputer/v-api" } 92 | v-model = { git = "https://github.com/oxidecomputer/v-api" } 93 | v-api-permission-derive = { git = "https://github.com/oxidecomputer/v-api" } 94 | yup-oauth2 = { version = "11.0.0" } 95 | 96 | # Config for 'cargo dist' 97 | [workspace.metadata.dist] 98 | # The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) 99 | cargo-dist-version = "0.12.2" 100 | # CI backends to support 101 | ci = ["github"] 102 | # The installers to generate for each app 103 | installers = [] 104 | # Target platforms to build apps for (Rust target-triple syntax) 105 | targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] 106 | # Publish jobs to run in CI 107 | pr-run-mode = "skip" 108 | 109 | # The profile that 'cargo dist' will build with 110 | [profile.dist] 111 | inherits = "release" 112 | lto = "thin" 113 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/shortcut/access.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use anyhow::{bail, Ok, Result}; 6 | use clap::{Parser, Subcommand}; 7 | use rfd_sdk::types::RfdPermission; 8 | 9 | use crate::Context; 10 | 11 | #[derive(Debug, Parser)] 12 | pub struct AccessShortcut { 13 | #[clap(subcommand)] 14 | pub access: AccessShortcuts, 15 | } 16 | 17 | #[derive(Debug, Subcommand)] 18 | pub enum AccessShortcuts { 19 | /// Grant and revoke access to RFDs 20 | Rfd(RfdAccessShortcut), 21 | } 22 | 23 | #[derive(Debug, Parser)] 24 | pub struct RfdAccessShortcut { 25 | #[clap(subcommand)] 26 | pub rfd: RfdAccessShortcuts, 27 | } 28 | 29 | #[derive(Debug, Subcommand)] 30 | pub enum RfdAccessShortcuts { 31 | /// Grant access to an RFD 32 | Add(AddRfdAccessShortcut), 33 | /// Revoke access to an RFD 34 | Remove(RemoveRfdAccessShortcut), 35 | } 36 | 37 | #[derive(Debug, Parser)] 38 | pub struct AddRfdAccessShortcut { 39 | /// Group name or id to grant access to 40 | pub group: String, 41 | /// RFD to grant access to 42 | pub number: i32, 43 | } 44 | 45 | #[derive(Debug, Parser)] 46 | pub struct RemoveRfdAccessShortcut { 47 | /// Group name or id to revoke access to 48 | pub group: String, 49 | /// RFD to revoke access to 50 | pub number: i32, 51 | } 52 | 53 | impl AddRfdAccessShortcut { 54 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 55 | let client = ctx.client()?; 56 | let groups = client.get_groups().send().await?.into_inner(); 57 | let group = groups 58 | .iter() 59 | .find(|g| g.id.to_string() == self.group) 60 | .or_else(|| groups.iter().find(|g| g.name == self.group)); 61 | 62 | if let Some(mut group) = group.cloned() { 63 | group.permissions.0.push(RfdPermission::GetRfd(self.number)); 64 | let response = client 65 | .update_group() 66 | .group_id(group.id) 67 | .body_map(|body| body.name(group.name).permissions(group.permissions)) 68 | .send() 69 | .await? 70 | .into_inner(); 71 | 72 | println!("Added access to RFD {} to {}", self.number, self.group); 73 | Ok(()) 74 | } else { 75 | bail!("Unable to find requested group") 76 | } 77 | } 78 | } 79 | 80 | impl RemoveRfdAccessShortcut { 81 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 82 | let client = ctx.client()?; 83 | let groups = client.get_groups().send().await?.into_inner(); 84 | let group = groups 85 | .iter() 86 | .find(|g| g.id.to_string() == self.group) 87 | .or_else(|| groups.iter().find(|g| g.name == self.group)); 88 | 89 | if let Some(mut group) = group.cloned() { 90 | group.permissions.0 = group 91 | .permissions 92 | .0 93 | .into_iter() 94 | .filter(|permission| match permission { 95 | RfdPermission::GetRfd(number) if *number == self.number => false, 96 | _ => true, 97 | }) 98 | .collect::>(); 99 | 100 | let response = client 101 | .update_group() 102 | .group_id(group.id) 103 | .body_map(|body| body.name(group.name).permissions(group.permissions)) 104 | .send() 105 | .await? 106 | .into_inner(); 107 | 108 | println!("Removed access to RFD {} from {}", self.number, self.group); 109 | Ok(()) 110 | } else { 111 | bail!("Unable to find requested group") 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /SETUP.md: -------------------------------------------------------------------------------- 1 | # rfd-api 2 | 3 | Backend services and tools for processing and managing RFDs 4 | 5 | ## Setup 6 | 7 | To start with there are few dependencies for running the RFD API. Notably the RFD API is broken up 8 | in to two applications, the `rfd-api` and the `rfd-processor`. See [README.md] for more information 9 | on the distinctions. 10 | 11 | ### Dependencies 12 | 13 | General 14 | * Rust 1.77 15 | * Postgres 14 16 | 17 | Processor 18 | * Asciidoctor 2.0.16 19 | * Node 20.11.0 20 | * NPM Packages: 21 | * Ruby 3.0.2 22 | * Ruby Gems: 23 | * asciidoctor-mermaid 0.4.1 24 | * asciidoctor-pdf 2.3.10 25 | * mmdc 10.6.1 26 | * rouge 4.2.0 27 | 28 | Optional 29 | * Meilisearch (if search is to be supported) 30 | 31 | ### Build 32 | 33 | Run `cargo build --release` to build all of the project binaries. Two binaries will be generated and 34 | emitted at: 35 | 36 | * `target/release/rfd-api` 37 | * `target/release/rfd-processor` 38 | 39 | ### Installation 40 | 41 | Once all of the dependencies have been installed, database migrations will need to be run to prepare 42 | the database tables. These can be run using the `rfd-installer` tool: 43 | 44 | ```sh 45 | cd rfd-model 46 | V_ONLY=1 DATABASE_URL= cargo run -p rfd-installer 47 | DATABASE_URL= diesel migration run 48 | ``` 49 | 50 | Replace `` with the url to the Postgres instance that the API and processor will be 51 | configured to run against. 52 | 53 | ### Running database migrations 54 | 55 | After the initial migration described above, any future database migration can 56 | be executed with the following commands: 57 | 58 | ```sh 59 | cd rfd-model 60 | DATABASE_URL= diesel migration run 61 | ``` 62 | 63 | > [!NOTE] 64 | > 65 | > If the generated `schema.rs` includes additional tables in its diff, it means 66 | > v-api added more tables of its own. You should exclude them in 67 | > `rfd-model/diesel.toml` and re-run migrations. The extraneous tables should 68 | > then disappear from `schema.rs`. 69 | 70 | ### Configuration 71 | 72 | Each part (the api and processor) has its own configuration file, and the respective application 73 | directories have example files called `config.example.toml`. Where possible the define either 74 | default values, or disabled options. 75 | 76 | #### API 77 | 78 | The two sections in the API configuration to pay attention to are the `[[keys]]` and `[[authn]]` 79 | configurations. 80 | 81 | `[[keys]]` define the key pairs used to sign API keys and session tokens. Two sources are supported 82 | for keys, either local or GCP Cloud KMS. Placeholder configurations are provided for both sources as 83 | examples. During local development it is recommended to generate a new RSA key pair locally for use. 84 | 85 | `[[authn]]` is a list of authentication providers that should be enabled. Google and GitHub are the 86 | only authentication providers supported currently and placeholders are available in the API example 87 | configuration. 88 | 89 | For local development remote authentication can be bypassed by using the `local-dev` feature. 90 | Instead of using a remote authentication provider, the API can be run via: 91 | 92 | ```sh 93 | cargo run -p rfd-api --features local-dev 94 | ``` 95 | 96 | This will run the API with a [`POST /login/local`](rfd-api/src/endpoints/login/local/mod.rs) endpoint 97 | exposed. This endpoint allows for logging in with an arbitrary email and user supplied unique 98 | identifier. To use this with the CLI, the `local-dev` feature will need to be passed to the CLI 99 | build. 100 | 101 | ```sh 102 | cargo run -p rfd-cli --features local-dev 103 | ``` 104 | 105 | #### Processor 106 | 107 | The processor has multiple jobs that are able to be run, and configuration is only required for 108 | jobs that are going to be run. The `actions` key defines the jobs that should be run. By default 109 | all jobs are disabled. In this this mode the processor will only construct a database of RFDs. 110 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/ensure_pr_state.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use std::cmp::Ordering; 7 | use tracing::instrument; 8 | 9 | use crate::rfd::PersistedRfd; 10 | 11 | use super::{ 12 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 13 | RfdUpdateMode, 14 | }; 15 | 16 | #[derive(Debug)] 17 | pub struct EnsureRfdWithPullRequestIsInValidState; 18 | 19 | #[async_trait] 20 | impl RfdUpdateAction for EnsureRfdWithPullRequestIsInValidState { 21 | #[instrument(skip(self, ctx, new), err(Debug))] 22 | async fn run( 23 | &self, 24 | ctx: &mut RfdUpdateActionContext, 25 | new: &mut PersistedRfd, 26 | _mode: RfdUpdateMode, 27 | ) -> Result { 28 | let RfdUpdateActionContext { pull_requests, .. } = ctx; 29 | 30 | let mut requires_source_commit = false; 31 | 32 | // We only want to operate on open pull requests 33 | let open_prs = pull_requests.iter().filter(|pr| pr.state == "open"); 34 | 35 | // Explicitly we will only update a pull request if it is the only open pull request for the 36 | // branch that we are working on 37 | match open_prs.count().cmp(&1) { 38 | Ordering::Equal => { 39 | // If there is a pull request open for this branch, then check to ensure that it is in one 40 | // of these valid states: 41 | // * published - A RFD may be in this state if it had previously been published and an 42 | // an update is being made, Or the RFD may be in the process of being 43 | // published 44 | // * committed - A RFD may be in this state if it had previously been committed and an 45 | // an update is being made. Or the RFD may be in the process of being 46 | // committed 47 | // * discussion - The default state for a RFD that has an open pull request and has yet to 48 | // to be merged. If the document on this branch is found to be in an 49 | // invalid state, it will be set back to the discussion state 50 | // * ideation - An alternative state to discussion where the RFD is not yet merged, but 51 | // may not be ready for discussion. A pull request is being used to share 52 | // initial thoughts on an idea 53 | // * abandoned - A RFD may be in this state if it had previously been abandoned or is in 54 | // the process of being abandoned 55 | if !new.is_state("discussion") 56 | && !new.is_state("published") 57 | && !new.is_state("committed") 58 | && !new.is_state("ideation") 59 | && !new.is_state("abandoned") 60 | { 61 | new.update_state("discussion") 62 | .map_err(|err| RfdUpdateActionErr::Stop(Box::new(err)))?; 63 | requires_source_commit = true; 64 | } else { 65 | tracing::debug!("RFD is in a valid state and does not need to be updated"); 66 | } 67 | } 68 | Ordering::Greater => { 69 | tracing::info!( 70 | "Found multiple pull requests for RFD. Unable to update state to discussion" 71 | ); 72 | } 73 | Ordering::Less => { 74 | // Nothing to do, there are no PRs 75 | tracing::debug!("No pull requests found for RFD. State can not be updated"); 76 | } 77 | } 78 | 79 | Ok(RfdUpdateActionResponse { 80 | requires_source_commit, 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /dropshot-authorization-header/src/basic.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use base64::{prelude::BASE64_STANDARD, Engine}; 7 | use dropshot::{ 8 | ApiEndpointBodyContentType, ExtensionMode, ExtractorMetadata, HttpError, RequestContext, 9 | ServerContext, SharedExtractor, 10 | }; 11 | 12 | /// Credentials for basic authentication 13 | pub struct BasicAuth { 14 | username: Option, 15 | password: Option, 16 | } 17 | 18 | impl BasicAuth { 19 | pub fn new(username: String, password: String) -> Self { 20 | Self { 21 | username: Some(username), 22 | password: Some(password), 23 | } 24 | } 25 | 26 | pub fn username(&self) -> Option<&String> { 27 | self.username.as_ref() 28 | } 29 | 30 | pub fn password(&self) -> Option<&String> { 31 | self.password.as_ref() 32 | } 33 | } 34 | 35 | /// Extracting a basic token should never fail, it should always return either `Ok(Some(BasicAuth))` 36 | /// or `Ok(None)`. `None` will be returned in any of the cases that a valid string can not be extracted. 37 | /// This extractor is not responsible for checking the value of the token. 38 | #[async_trait] 39 | impl SharedExtractor for BasicAuth { 40 | async fn from_request( 41 | rqctx: &RequestContext, 42 | ) -> Result { 43 | // Similarly we only care about the presence of the Authorization header 44 | let header_value = rqctx 45 | .request 46 | .headers() 47 | .get("Authorization") 48 | .and_then(|header| { 49 | // If the value provided is not a readable string we will throw it out 50 | header 51 | .to_str() 52 | .map(|s| s.to_string()) 53 | .map_err(|err| { 54 | tracing::info!("Failed to turn Authorization header into string"); 55 | err 56 | }) 57 | .ok() 58 | }); 59 | 60 | // Finally ensure that the value we found is properly formed 61 | let contents = header_value.and_then(|value| { 62 | let parts = value.split_once(' '); 63 | 64 | match parts { 65 | Some(("Basic", token)) => BASE64_STANDARD 66 | .decode(token) 67 | .map_err(|err| { 68 | tracing::info!("Failed to decode basic auth header"); 69 | err 70 | }) 71 | .ok() 72 | .and_then(|decoded| { 73 | String::from_utf8(decoded) 74 | .map_err(|err| { 75 | tracing::info!( 76 | "Failed to interpret decoded bytes from authorization header" 77 | ); 78 | err 79 | }) 80 | .ok() 81 | }) 82 | .and_then(|parsed| match parsed.split_once(':') { 83 | Some((username, password)) => { 84 | Some((username.to_string(), password.to_string())) 85 | } 86 | _ => None, 87 | }), 88 | _ => None, 89 | } 90 | }); 91 | 92 | if let Some((username, password)) = contents { 93 | Ok(BasicAuth { 94 | username: Some(username), 95 | password: Some(password), 96 | }) 97 | } else { 98 | Ok(BasicAuth { 99 | username: None, 100 | password: None, 101 | }) 102 | } 103 | } 104 | 105 | fn metadata(_body_content_type: ApiEndpointBodyContentType) -> ExtractorMetadata { 106 | ExtractorMetadata { 107 | parameters: vec![], 108 | extension_mode: ExtensionMode::None, 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /rfd-processor/src/util.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use base64::{prelude::BASE64_STANDARD, DecodeError, Engine}; 6 | use google_drive3::{ 7 | hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}, 8 | hyper_util::{ 9 | client::legacy::{connect::HttpConnector, Client}, 10 | rt::TokioExecutor, 11 | }, 12 | DriveHub, 13 | }; 14 | use std::path::Path; 15 | use thiserror::Error; 16 | use tokio::{fs, io::AsyncWriteExt}; 17 | 18 | #[derive(Debug, Error)] 19 | pub enum FileIoError { 20 | #[error("Failed to write file {0}")] 21 | Io(#[from] std::io::Error), 22 | #[error("Expected file path to have a parent path")] 23 | MissingParent, 24 | } 25 | 26 | pub async fn write_file(file: &Path, contents: &[u8]) -> Result<(), FileIoError> { 27 | if let Some(parent) = file.parent() { 28 | // create each directory. 29 | fs::create_dir_all(parent).await?; 30 | 31 | // Write to the file. 32 | let mut f = fs::File::create(file).await?; 33 | f.write_all(contents).await?; 34 | 35 | tracing::info!(?file, "Wrote file"); 36 | 37 | Ok(()) 38 | } else { 39 | Err(FileIoError::MissingParent) 40 | } 41 | } 42 | 43 | pub fn decode_base64(c: &str) -> Result, DecodeError> { 44 | let v = c.replace('\n', ""); 45 | let decoded = BASE64_STANDARD.decode(&v)?; 46 | Ok(decoded.trim().to_vec()) 47 | } 48 | 49 | trait SliceExt { 50 | fn trim(&self) -> Self; 51 | } 52 | 53 | impl SliceExt for Vec { 54 | fn trim(&self) -> Vec { 55 | fn is_whitespace(c: &u8) -> bool { 56 | c == &b'\t' || c == &b' ' 57 | } 58 | 59 | fn is_not_whitespace(c: &u8) -> bool { 60 | !is_whitespace(c) 61 | } 62 | 63 | if let Some(first) = self.iter().position(is_not_whitespace) { 64 | if let Some(last) = self.iter().rposition(is_not_whitespace) { 65 | self[first..last + 1].to_vec() 66 | } else { 67 | unreachable!(); 68 | } 69 | } else { 70 | vec![] 71 | } 72 | } 73 | } 74 | 75 | #[derive(Debug, Error)] 76 | pub enum GDriveError { 77 | #[error("Failed to construct GDrive error: {0}")] 78 | RemoteKeyAuthMissing(#[source] std::io::Error), 79 | } 80 | 81 | pub async fn gdrive_client() -> Result>, GDriveError> { 82 | let opts = yup_oauth2::ApplicationDefaultCredentialsFlowOpts::default(); 83 | 84 | tracing::trace!(?opts, "Request GCP credentials"); 85 | 86 | let gcp_credentials = 87 | yup_oauth2::ApplicationDefaultCredentialsAuthenticator::builder(opts).await; 88 | 89 | tracing::trace!("Retrieved GCP credentials"); 90 | 91 | let gcp_auth = match gcp_credentials { 92 | yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::ServiceAccount(auth) => { 93 | tracing::debug!("Create GCP service account based credentials"); 94 | 95 | auth.build().await.map_err(|err| { 96 | tracing::error!( 97 | ?err, 98 | "Failed to construct Google Drive credentials from service account" 99 | ); 100 | GDriveError::RemoteKeyAuthMissing(err) 101 | })? 102 | } 103 | yup_oauth2::authenticator::ApplicationDefaultCredentialsTypes::InstanceMetadata(auth) => { 104 | tracing::debug!("Create GCP instance based credentials"); 105 | 106 | auth.build().await.map_err(|err| { 107 | tracing::error!( 108 | ?err, 109 | "Failed to construct Google Drive credentials from instance metadata" 110 | ); 111 | GDriveError::RemoteKeyAuthMissing(err) 112 | })? 113 | } 114 | }; 115 | 116 | Ok(DriveHub::new( 117 | Client::builder(TokioExecutor::new()).build( 118 | HttpsConnectorBuilder::new() 119 | .with_native_roots() 120 | .unwrap() 121 | .https_only() 122 | .enable_http2() 123 | .build(), 124 | ), 125 | gcp_auth, 126 | )) 127 | } 128 | 129 | #[cfg(test)] 130 | pub mod test_util { 131 | use tracing_subscriber::EnvFilter; 132 | 133 | #[allow(dead_code)] 134 | pub fn start_tracing() { 135 | let _subscriber = tracing_subscriber::fmt() 136 | .with_file(false) 137 | .with_line_number(false) 138 | .with_env_filter(EnvFilter::from_default_env()) 139 | .pretty() 140 | .init(); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /rfd-api/config.example.toml: -------------------------------------------------------------------------------- 1 | # Allowed values: json, pretty 2 | log_format = "" 3 | 4 | # Optional directory to write log files to. If not specified, logs are written to stdout 5 | log_directory = "" 6 | 7 | # Config file to load initial mappers and groups from. See mappers.example.toml for how to 8 | # construct this file 9 | initial_mappers = "" 10 | 11 | # Public url of the service. This should include the protocol, host, and port that a user would use 12 | # to connect to the server on 13 | public_url = "" 14 | 15 | # Port for the server to run on. This does not have to match the public_url (i.e. in the case that 16 | # the server is running behind a proxy) 17 | server_port = 8080 18 | 19 | # Full url of the Postgres database to connect to 20 | database_url = "postgres://:@/" 21 | 22 | # Settings for JWT management 23 | [jwt] 24 | 25 | # Duration that a JWT is valid for 26 | default_expiration = 3600 27 | 28 | # Keys for signing JWTs and generating secrets. GCP Cloud KMS keys and local static keys 29 | # are supported. At least one signer and one verifier key must be configured. 30 | 31 | # Cloud KMS - Signer 32 | [[keys]] 33 | kind = "ckms_signer" # Static identifier indicating Cloud KMS 34 | kid = "" # Unique key identifer, that will be used in JWKS 35 | version = 1 # KMS key version 36 | key = "" # KMS key name 37 | keyring = "" # KMS keying name 38 | location = "" # KMS region 39 | project = "" # GCP project containing Cloud KMS 40 | 41 | # Cloud KMS - Verifier 42 | [[keys]] 43 | kind = "ckms_verifier" # Static identifier indicating Cloud KMS 44 | kid = "" # Unique key identifer, that will be used in JWKS 45 | version = 1 # KMS key version 46 | key = "" # KMS key name 47 | keyring = "" # KMS keying name 48 | location = "" # KMS region 49 | project = "" # GCP project containing Cloud KMS 50 | 51 | # Local key - Signer 52 | [[keys]] 53 | kind = "local_signer" # Static identifier indicating a local signing key 54 | kid = "" # Unique key identifier, that will be used in JWKS 55 | private = """""" # PEM encoded private key 56 | 57 | # Local key - Verifier 58 | [[keys]] 59 | kind = "local_verifier" # Static identifier indicating a local verification key 60 | kid = "" # Unique key identifier, that will be used in JWKS (must match signer kid) 61 | public = """""" # PEM encoded public key 62 | 63 | # OAuth Providers 64 | # Google and GitHub are supported. An OAuth provider needs to have both a web and device config. 65 | # At least one OAuth provider must be configured 66 | 67 | [authn.oauth.google.device] 68 | client_id = "" 69 | client_secret = "" 70 | 71 | [authn.oauth.google.web] 72 | client_id = "" 73 | client_secret = "" 74 | redirect_uri = "https:///login/oauth/google/code/callback" 75 | 76 | [authn.oauth.github.device] 77 | client_id = "" 78 | client_secret = "" 79 | 80 | [authn.oauth.github.web] 81 | client_id = "" 82 | client_secret = "" 83 | redirect_uri = "https:///login/oauth/github/code/callback" 84 | 85 | # Search configuration 86 | [search] 87 | # Remote url of the search service 88 | host = "" 89 | # Read-only search key 90 | key = "" 91 | # Index to perform searches against 92 | index = "" 93 | 94 | # Fields for use in generating the OpenAPI spec file 95 | [spec] 96 | title = "" 97 | description = "" 98 | contact_url = "" 99 | contact_email = "" 100 | output_path = "" 101 | 102 | # Templated for creating new RFDs. The 'placeholder' and 'new' templates are the only two templates 103 | # available and are both required 104 | 105 | # Template used when creating a new RFD without specifying a body 106 | [content.templates.placeholder] 107 | template = """""" 108 | required_fields = [] 109 | 110 | # Template used when creating a new RFD while specifying a body 111 | [content.templates.new] 112 | template = """""" 113 | required_fields = [] 114 | 115 | # The GitHub repository to use to write RFDs 116 | [services.github] 117 | # GitHub user or organization 118 | owner = "" 119 | # GitHub repository name 120 | repo = "" 121 | # Path within the repository where RFDs are stored 122 | path = "" 123 | # Branch to use as the default branch of the repository 124 | default_branch = "" 125 | 126 | # The method for authenticating to GitHub. This requires one of two authentication styles: 127 | # 1. A GitHub App installation that is defined by an app_id, installation_id, and private_key 128 | # 2. A GitHub access token 129 | # Exactly one authentication must be specified 130 | 131 | # App Installation 132 | [services.github.auth] 133 | # Numeric GitHub App id 134 | app_id = 1111111 135 | # Numeric GitHub App installation id corresponding to the organization that the configured repo 136 | # belongs to 137 | installation_id = 2222222 138 | # PEM encoded private key for the GitHub App 139 | private_key = """""" 140 | 141 | # Access Token 142 | [services.github.auth] 143 | # This may be any GitHub access token that has permission to the configured repo 144 | token = "" 145 | -------------------------------------------------------------------------------- /rfd-api/src/initial_data.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use config::{Config, ConfigError, Environment, File}; 6 | use newtype_uuid::TypedUuid; 7 | use serde::Deserialize; 8 | use thiserror::Error; 9 | use tracing::Instrument; 10 | use v_api::{mapper::MappingRulesData, response::ResourceError, VContext}; 11 | use v_model::{storage::StoreError, NewAccessGroup, NewMapper, Permissions}; 12 | 13 | use crate::permissions::RfdPermission; 14 | 15 | #[derive(Debug, Deserialize)] 16 | pub struct InitialData { 17 | pub groups: Vec, 18 | pub mappers: Vec, 19 | } 20 | 21 | #[derive(Debug, Deserialize)] 22 | pub struct InitialGroup { 23 | pub name: String, 24 | pub permissions: Permissions, 25 | } 26 | 27 | #[derive(Debug, Deserialize)] 28 | pub struct InitialMapper { 29 | pub name: String, 30 | #[serde(flatten)] 31 | pub rule: MappingRulesData, 32 | pub max_activations: Option, 33 | } 34 | 35 | #[derive(Debug, Error)] 36 | pub enum InitError { 37 | #[error("Failed to parse configuration file for initial data: {0}")] 38 | Config(#[from] ConfigError), 39 | #[error("Resource operation failed")] 40 | Resource(#[from] ResourceError), 41 | #[error("Failed to serialize rule for storage: {0}")] 42 | Rule(#[from] serde_json::Error), 43 | #[error("Failed to store initial rule: {0}")] 44 | Storage(#[from] StoreError), 45 | } 46 | 47 | impl InitialData { 48 | pub fn new(config_sources: Option>) -> Result { 49 | let mut config = Config::builder() 50 | .add_source(File::with_name("mappers.toml").required(false)) 51 | .add_source(File::with_name("rfd-api/mappers.toml").required(false)); 52 | 53 | for source in config_sources.unwrap_or_default() { 54 | config = config.add_source(File::with_name(&source).required(false)); 55 | } 56 | 57 | Ok(config 58 | .add_source(Environment::default()) 59 | .build()? 60 | .try_deserialize()?) 61 | } 62 | 63 | pub async fn initialize(self, ctx: &VContext) -> Result<(), InitError> { 64 | let existing_groups = ctx 65 | .group 66 | .get_groups(&ctx.builtin_registration_user()) 67 | .await?; 68 | 69 | for group in self.groups { 70 | let span = tracing::info_span!("Initializing group", group = ?group); 71 | 72 | async { 73 | let id = existing_groups 74 | .iter() 75 | .find(|g| g.name == group.name) 76 | .map(|g| g.id) 77 | .unwrap_or_else(|| TypedUuid::new_v4()); 78 | 79 | ctx.group 80 | .create_group( 81 | &ctx.builtin_registration_user(), 82 | NewAccessGroup { 83 | id, 84 | name: group.name, 85 | permissions: group.permissions, 86 | }, 87 | ) 88 | .await 89 | .map(|_| ()) 90 | .or_else(handle_unique_violation_error) 91 | } 92 | .instrument(span) 93 | .await? 94 | } 95 | 96 | for mapper in self.mappers { 97 | let span = tracing::info_span!("Initializing mapper", mapper = ?mapper); 98 | async { 99 | let new_mapper = NewMapper { 100 | id: TypedUuid::new_v4(), 101 | name: mapper.name, 102 | rule: serde_json::to_value(&mapper.rule)?, 103 | activations: None, 104 | max_activations: mapper.max_activations.map(|i| i as i32), 105 | }; 106 | 107 | ctx.mapping 108 | .add_mapper(&ctx.builtin_registration_user(), &new_mapper) 109 | .await 110 | .map(|_| ()) 111 | .or_else(handle_unique_violation_error)?; 112 | 113 | Ok::<(), InitError>(()) 114 | } 115 | .instrument(span) 116 | .await?; 117 | } 118 | 119 | Ok(()) 120 | } 121 | } 122 | 123 | fn handle_unique_violation_error( 124 | err: ResourceError, 125 | ) -> Result<(), ResourceError> { 126 | match err { 127 | ResourceError::Conflict => { 128 | tracing::info!("Record already exists. Skipping."); 129 | Ok(()) 130 | } 131 | err => { 132 | tracing::error!(?err, "Failed to store record"); 133 | Err(err) 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /rfd-processor/src/processor.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use futures::TryFutureExt; 6 | use rfd_github::{GitHubRfdLocation, GitHubRfdUpdate}; 7 | use rfd_model::{ 8 | storage::{JobFilter, JobStore}, 9 | Job, 10 | }; 11 | use std::sync::Arc; 12 | use tap::TapFallible; 13 | use thiserror::Error; 14 | use tokio::{ 15 | sync::{AcquireError, Semaphore}, 16 | time::interval, 17 | }; 18 | use tracing::instrument; 19 | use v_model::storage::{ListPagination, StoreError}; 20 | 21 | use crate::{context::Context, updater::RfdUpdater}; 22 | 23 | #[derive(Debug, Error)] 24 | pub enum JobError { 25 | #[error("Failed to acquire permit")] 26 | Permit(#[from] AcquireError), 27 | #[error(transparent)] 28 | Storage(#[from] StoreError), 29 | } 30 | 31 | pub async fn processor(ctx: Arc) -> Result<(), JobError> { 32 | let mut interval = interval(ctx.processor.interval); 33 | let pagination = ListPagination::default().limit(ctx.processor.batch_size); 34 | let capacity = Arc::new(Semaphore::new(ctx.processor.capacity as usize)); 35 | 36 | tracing::info!(?interval, ?pagination, "Starting processor"); 37 | 38 | interval.tick().await; 39 | 40 | loop { 41 | if ctx.processor.enabled { 42 | let jobs = JobStore::list( 43 | &ctx.db.storage, 44 | vec![JobFilter::default() 45 | .processed(Some(false)) 46 | .started(Some(false))], 47 | &pagination, 48 | ) 49 | .await?; 50 | 51 | tracing::info!(jobs = ?jobs.iter().map(|job| job.id).collect::>(), "Spawning jobs"); 52 | 53 | for job in jobs { 54 | let job_id = job.id; 55 | tracing::info!("Starting job processing"); 56 | match JobStore::start(&ctx.db.storage, job.id).await { 57 | Ok(Some(job)) => { 58 | tracing::info!(job = ?job_id, "Spawning job"); 59 | let capacity = capacity.clone(); 60 | let ctx = ctx.clone(); 61 | 62 | tokio::spawn( 63 | async move { 64 | tracing::info!(job = ?job_id, "Acquiring permit to run job"); 65 | let permit = capacity.acquire().await?; 66 | let result = run_job(ctx, job).await; 67 | 68 | // Hold the permit until the job has reached a conclusion 69 | drop(permit); 70 | 71 | result 72 | } 73 | .or_else(move |err| async move { 74 | tracing::error!(id = ?job_id, ?err, "Spawned job failed"); 75 | Err(err) 76 | }), 77 | ); 78 | } 79 | Ok(None) => { 80 | tracing::error!(?job, "Job that was scheduled to run has gone missing! Was it started by a different task?"); 81 | } 82 | Err(err) => { 83 | tracing::warn!( 84 | ?job, 85 | ?err, 86 | "Failed to start job. Was it previously started?" 87 | ); 88 | } 89 | } 90 | } 91 | } 92 | 93 | interval.tick().await; 94 | } 95 | } 96 | 97 | #[instrument(skip(ctx, job), fields(id = job.id, rfd = job.rfd, sha = ?job.sha, commited_at = ?job.committed_at))] 98 | async fn run_job(ctx: Arc, job: Job) -> Result<(), JobError> { 99 | tracing::info!("Running job"); 100 | 101 | let location = GitHubRfdLocation { 102 | client: ctx.github.client.clone(), 103 | owner: job.owner.clone(), 104 | repo: job.repository.clone(), 105 | branch: job.branch.clone(), 106 | commit: job.sha.clone(), 107 | default_branch: ctx.github.repository.default_branch.clone(), 108 | }; 109 | 110 | let update = GitHubRfdUpdate { 111 | location, 112 | number: job.rfd.into(), 113 | committed_at: job.committed_at, 114 | }; 115 | 116 | let updater = RfdUpdater::new(&ctx.actions, ctx.processor.update_mode); 117 | 118 | match updater.handle(&ctx, &[update]).await { 119 | Ok(_) => { 120 | let _ = JobStore::complete(&ctx.db.storage, job.id) 121 | .await 122 | .tap_err(|err| tracing::error!(?err, "Failed to mark job as completed")); 123 | } 124 | Err(err) => { 125 | tracing::error!(?err, "RFD update failed"); 126 | 127 | // TODO: Mark job as failed or retry? 128 | } 129 | } 130 | 131 | Ok::<_, JobError>(()) 132 | } 133 | -------------------------------------------------------------------------------- /rfd-cli/src/printer/json.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use rfd_sdk::types; 6 | use serde::Serialize; 7 | 8 | use super::CliOutput; 9 | 10 | #[derive(Debug, Clone)] 11 | pub struct RfdJsonPrinter; 12 | 13 | impl CliOutput for RfdJsonPrinter { 14 | fn output_api_user(&self, value: types::ApiUserForRfdPermission) { 15 | println!("{}", serde_json::to_string(&value).unwrap()) 16 | } 17 | 18 | fn output_api_user_list(&self, value: Vec) { 19 | println!("{}", serde_json::to_string(&value).unwrap()) 20 | } 21 | 22 | fn output_user(&self, value: types::GetUserResponseForRfdPermission) { 23 | println!("{}", serde_json::to_string(&value).unwrap()) 24 | } 25 | 26 | fn output_api_key_list(&self, value: Vec) { 27 | println!("{}", serde_json::to_string(&value).unwrap()) 28 | } 29 | 30 | fn output_api_key_initial(&self, value: types::InitialApiKeyResponseForRfdPermission) { 31 | println!("{}", serde_json::to_string(&value).unwrap()) 32 | } 33 | 34 | fn output_api_key(&self, value: types::ApiKeyResponseForRfdPermission) { 35 | println!("{}", serde_json::to_string(&value).unwrap()) 36 | } 37 | 38 | fn output_group_list(&self, value: Vec) { 39 | println!("{}", serde_json::to_string(&value).unwrap()) 40 | } 41 | 42 | fn output_group(&self, value: types::AccessGroupForRfdPermission) { 43 | println!("{}", serde_json::to_string(&value).unwrap()) 44 | } 45 | 46 | fn output_mapper_list(&self, value: Vec) { 47 | println!("{}", serde_json::to_string(&value).unwrap()) 48 | } 49 | 50 | fn output_mapper(&self, value: types::Mapper) { 51 | println!("{}", serde_json::to_string(&value).unwrap()) 52 | } 53 | 54 | fn output_oauth_client_list(&self, value: Vec) { 55 | println!("{}", serde_json::to_string(&value).unwrap()) 56 | } 57 | 58 | fn output_oauth_client(&self, value: types::OAuthClient) { 59 | println!("{}", serde_json::to_string(&value).unwrap()) 60 | } 61 | 62 | fn output_oauth_redirect_uri(&self, value: types::OAuthClientRedirectUri) { 63 | println!("{}", serde_json::to_string(&value).unwrap()) 64 | } 65 | 66 | fn output_oauth_secret_initial(&self, value: types::InitialOAuthClientSecretResponse) { 67 | println!("{}", serde_json::to_string(&value).unwrap()) 68 | } 69 | 70 | fn output_oauth_secret(&self, value: types::OAuthClientSecret) { 71 | println!("{}", serde_json::to_string(&value).unwrap()) 72 | } 73 | 74 | fn output_rfd_meta(&self, value: types::RfdWithoutContent) { 75 | println!("{}", serde_json::to_string(&value).unwrap()) 76 | } 77 | 78 | fn output_rfd_list(&self, value: Vec) { 79 | println!("{}", serde_json::to_string(&value).unwrap()) 80 | } 81 | 82 | fn output_rfd_full(&self, value: types::RfdWithRaw) { 83 | println!("{}", serde_json::to_string(&value).unwrap()) 84 | } 85 | 86 | fn output_rfd(&self, value: types::Rfd) { 87 | println!("{}", serde_json::to_string(&value).unwrap()) 88 | } 89 | 90 | fn output_search_results(&self, value: types::SearchResults) { 91 | println!("{}", serde_json::to_string(&value).unwrap()) 92 | } 93 | 94 | fn output_rfd_attr(&self, value: types::RfdAttr) { 95 | println!("{}", serde_json::to_string(&value).unwrap()) 96 | } 97 | 98 | fn output_rfd_revision_meta(&self, value: types::RfdRevisionMeta) { 99 | println!("{}", serde_json::to_string(&value).unwrap()) 100 | } 101 | 102 | fn output_rfd_revision_meta_list(&self, value: Vec) { 103 | println!("{}", serde_json::to_string(&value).unwrap()) 104 | } 105 | 106 | fn output_reserved_rfd(&self, value: types::ReserveRfdResponse) { 107 | println!("{}", serde_json::to_string(&value).unwrap()) 108 | } 109 | 110 | fn output_job_list(&self, value: Vec) { 111 | println!("{}", serde_json::to_string(&value).unwrap()) 112 | } 113 | 114 | fn output_magic_link_client_list(&self, value: Vec) { 115 | println!("{}", serde_json::to_string(&value).unwrap()) 116 | } 117 | 118 | fn output_magic_link_client(&self, value: types::MagicLink) { 119 | println!("{}", serde_json::to_string(&value).unwrap()) 120 | } 121 | 122 | fn output_magic_link_redirect_uri(&self, value: types::MagicLinkRedirectUri) { 123 | println!("{}", serde_json::to_string(&value).unwrap()) 124 | } 125 | 126 | fn output_magic_link_secret_initial(&self, value: types::InitialMagicLinkSecretResponse) { 127 | println!("{}", serde_json::to_string(&value).unwrap()) 128 | } 129 | 130 | fn output_magic_link_secret(&self, value: types::MagicLinkSecret) { 131 | println!("{}", serde_json::to_string(&value).unwrap()) 132 | } 133 | 134 | fn output_error(&self, value: &progenitor_client::Error) 135 | where 136 | T: schemars::JsonSchema + serde::Serialize + std::fmt::Debug, 137 | { 138 | eprintln!("{}", value) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /rfd-ts/src/http-client.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This Source Code Form is subject to the terms of the Mozilla Public 3 | * License, v. 2.0. If a copy of the MPL was not distributed with this 4 | * file, you can obtain one at https://mozilla.org/MPL/2.0/. 5 | * 6 | * Copyright Oxide Computer Company 7 | */ 8 | 9 | import { camelToSnake, isNotNull, processResponseBody, snakeify } from './util' 10 | 11 | /** Success responses from the API */ 12 | export type ApiSuccess = { 13 | type: 'success' 14 | response: Response 15 | data: Data 16 | } 17 | 18 | // HACK: this has to match what comes from the API in the `Error` schema. We put 19 | // our own copy here so we can test this file statically without generating 20 | // anything 21 | export type ErrorBody = { 22 | errorCode?: string | null 23 | message: string 24 | requestId: string 25 | } 26 | 27 | export type ErrorResult = 28 | // 4xx and 5xx responses from the API 29 | | { 30 | type: 'error' 31 | response: Response 32 | data: ErrorBody 33 | } 34 | // JSON parsing or processing errors within the client. Includes raised Error 35 | // and response body as a string for debugging. 36 | | { 37 | type: 'client_error' 38 | response: Response 39 | error: Error 40 | text: string 41 | } 42 | 43 | export type ApiResult = ApiSuccess | ErrorResult 44 | 45 | /** 46 | * Convert `Date` to ISO string. Leave other values alone. Used for both request 47 | * body and query params. 48 | */ 49 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 50 | function replacer(_key: string, value: any) { 51 | if (value instanceof Date) { 52 | return value.toISOString() 53 | } 54 | return value 55 | } 56 | 57 | function encodeQueryParam(key: string, value: unknown) { 58 | return `${encodeURIComponent(camelToSnake(key))}=${ 59 | encodeURIComponent( 60 | replacer(key, value), 61 | ) 62 | }` 63 | } 64 | 65 | export async function handleResponse( 66 | response: Response, 67 | ): Promise> { 68 | const respText = await response.text() 69 | 70 | // catch JSON parse or processing errors 71 | let respJson 72 | try { 73 | // don't bother trying to parse empty responses like 204s 74 | // TODO: is empty object what we want here? 75 | respJson = respText.length > 0 ? processResponseBody(JSON.parse(respText)) : {} 76 | } catch (e) { 77 | return { 78 | type: 'client_error', 79 | response, 80 | error: e as Error, 81 | text: respText, 82 | } 83 | } 84 | 85 | if (!response.ok) { 86 | return { 87 | type: 'error', 88 | response, 89 | data: respJson as ErrorBody, 90 | } 91 | } 92 | 93 | // don't validate respJson, just assume it matches the type 94 | return { 95 | type: 'success', 96 | response, 97 | data: respJson as Data, 98 | } 99 | } 100 | 101 | // has to be any. the particular query params types don't like unknown 102 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 103 | type QueryParams = Record 104 | 105 | /** 106 | * Params that get passed to `fetch`. This ends up as an optional second 107 | * argument to each generated request method. Properties are a subset of 108 | * `RequestInit`. 109 | */ 110 | export interface FetchParams extends Omit {} 111 | 112 | /** All arguments to `request()` */ 113 | export interface FullParams extends FetchParams { 114 | path: string 115 | query?: QueryParams 116 | body?: unknown 117 | host?: string 118 | method?: string 119 | } 120 | 121 | export interface ApiConfig { 122 | /** 123 | * No host means requests will be sent to the current host. This is used in 124 | * the web console. 125 | */ 126 | host?: string 127 | token?: string 128 | baseParams?: FetchParams 129 | } 130 | 131 | export class HttpClient { 132 | host: string 133 | token?: string 134 | baseParams: FetchParams 135 | 136 | constructor({ host = '', baseParams = {}, token }: ApiConfig = {}) { 137 | this.host = host 138 | this.token = token 139 | 140 | const headers = new Headers({ 'Content-Type': 'application/json' }) 141 | if (token) { 142 | headers.append('Authorization', `Bearer ${token}`) 143 | } 144 | this.baseParams = mergeParams({ headers }, baseParams) 145 | } 146 | 147 | public async request({ 148 | body, 149 | path, 150 | query, 151 | host, 152 | ...fetchParams 153 | }: FullParams): Promise> { 154 | const url = (host || this.host) + path + toQueryString(query) 155 | const init = { 156 | ...mergeParams(this.baseParams, fetchParams), 157 | body: JSON.stringify(snakeify(body), replacer), 158 | } 159 | return handleResponse(await fetch(url, init)) 160 | } 161 | } 162 | 163 | export function mergeParams(a: FetchParams, b: FetchParams): FetchParams { 164 | // calling `new Headers()` normalizes `HeadersInit`, which could be a Headers 165 | // object, a plain object, or an array of tuples 166 | const headers = new Headers(a.headers) 167 | for (const [key, value] of new Headers(b.headers).entries()) { 168 | headers.set(key, value) 169 | } 170 | return { ...a, ...b, headers } 171 | } 172 | 173 | /** Query params with null values filtered out. `"?"` included. */ 174 | export function toQueryString(rawQuery?: QueryParams): string { 175 | const qs = Object.entries(rawQuery || {}) 176 | .filter(([_key, value]) => isNotNull(value)) 177 | .map(([key, value]) => 178 | Array.isArray(value) 179 | ? value.map((item) => encodeQueryParam(key, item)).join('&') 180 | : encodeQueryParam(key, value) 181 | ) 182 | .join('&') 183 | return qs ? '?' + qs : '' 184 | } 185 | -------------------------------------------------------------------------------- /rfd-api/src/server.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use dropshot::{ 6 | ApiDescription, ConfigDropshot, EndpointTagPolicy, ServerBuilder, TagConfig, TagDetails, 7 | }; 8 | use semver::Version; 9 | use serde::Deserialize; 10 | use slog::Drain; 11 | use std::{collections::HashMap, error::Error, fs::File, net::SocketAddr, path::PathBuf}; 12 | use tracing_slog::TracingSlogDrain; 13 | use v_api::{inject_endpoints, v_system_endpoints}; 14 | 15 | use crate::{ 16 | context::RfdContext, 17 | endpoints::{ 18 | job::list_jobs, 19 | rfd::{ 20 | discuss_rfd, list_rfd_revisions, list_rfds, publish_rfd, reserve_rfd, search_rfds, 21 | set_rfd_attr, set_rfd_content, set_rfd_document, update_rfd_revision, 22 | update_rfd_visibility, view_rfd, view_rfd_attr, view_rfd_discussion, view_rfd_meta, 23 | view_rfd_pdf, view_rfd_revision, view_rfd_revision_attr, view_rfd_revision_discussion, 24 | view_rfd_revision_meta, view_rfd_revision_pdf, 25 | }, 26 | webhook::github_webhook, 27 | }, 28 | permissions::RfdPermission, 29 | }; 30 | 31 | #[derive(Clone, Debug, Deserialize)] 32 | pub struct SpecConfig { 33 | pub title: String, 34 | pub description: String, 35 | pub contact_url: String, 36 | pub contact_email: String, 37 | pub output_path: PathBuf, 38 | } 39 | 40 | pub struct ServerConfig { 41 | pub context: RfdContext, 42 | pub server_address: SocketAddr, 43 | pub spec_output: Option, 44 | } 45 | 46 | v_system_endpoints!(RfdContext, RfdPermission); 47 | 48 | pub fn server( 49 | config: ServerConfig, 50 | ) -> Result, Box> { 51 | let mut config_dropshot = ConfigDropshot::default(); 52 | config_dropshot.bind_address = config.server_address; 53 | config_dropshot.default_request_body_max_bytes = 1024 * 1024; 54 | 55 | // Construct a shim to pipe dropshot logs into the global tracing logger 56 | let dropshot_logger = { 57 | let level_drain = slog::LevelFilter(TracingSlogDrain, slog::Level::Debug).fuse(); 58 | let async_drain = slog_async::Async::new(level_drain).build().fuse(); 59 | slog::Logger::root(async_drain, slog::o!()) 60 | }; 61 | 62 | let mut tags = HashMap::new(); 63 | tags.insert( 64 | "hidden".to_string(), 65 | TagDetails { 66 | description: Some("Internal endpoints".to_string()), 67 | external_docs: None, 68 | }, 69 | ); 70 | 71 | let mut api = ApiDescription::new().tag_config(TagConfig { 72 | allow_other_tags: false, 73 | policy: EndpointTagPolicy::Any, 74 | tags, 75 | }); 76 | 77 | inject_endpoints!(api); 78 | 79 | // RFDs 80 | api.register(list_rfds) 81 | .expect("Failed to register endpoint"); 82 | 83 | api.register(view_rfd_meta) 84 | .expect("Failed to register endpoint"); 85 | api.register(view_rfd).expect("Failed to register endpoint"); 86 | api.register(view_rfd_pdf) 87 | .expect("Failed to register endpoint"); 88 | api.register(view_rfd_attr) 89 | .expect("Failed to register endpoint"); 90 | api.register(view_rfd_discussion) 91 | .expect("Failed to register endpoint"); 92 | 93 | api.register(list_rfd_revisions) 94 | .expect("Failed to register endpoint"); 95 | api.register(view_rfd_revision_meta) 96 | .expect("Failed to register endpoint"); 97 | api.register(view_rfd_revision) 98 | .expect("Failed to register endpoint"); 99 | api.register(view_rfd_revision_pdf) 100 | .expect("Failed to register endpoint"); 101 | api.register(view_rfd_revision_attr) 102 | .expect("Failed to register endpoint"); 103 | api.register(view_rfd_revision_discussion) 104 | .expect("Failed to register endpoint"); 105 | 106 | api.register(search_rfds) 107 | .expect("Failed to register endpoint"); 108 | 109 | api.register(reserve_rfd) 110 | .expect("Failed to register endpoint"); 111 | api.register(set_rfd_document) 112 | .expect("Failed to register endpoint"); 113 | api.register(set_rfd_content) 114 | .expect("Failed to register endpoint"); 115 | api.register(set_rfd_attr) 116 | .expect("Failed to register endpoint"); 117 | api.register(discuss_rfd) 118 | .expect("Failed to register endpoint"); 119 | api.register(publish_rfd) 120 | .expect("Failed to register endpoint"); 121 | api.register(update_rfd_visibility) 122 | .expect("Failed to register endpoint"); 123 | api.register(update_rfd_revision) 124 | .expect("Failed to register endpoint"); 125 | 126 | api.register(list_jobs) 127 | .expect("Failed to register endpoint"); 128 | 129 | // Webhooks 130 | api.register(github_webhook) 131 | .expect("Failed to register endpoint"); 132 | 133 | if let Some(spec) = config.spec_output { 134 | // TODO: How do we validate that spec_output can be read or written? Currently File::create 135 | // panics if the file path is not a valid path 136 | 137 | // Create the API schema. 138 | let mut api_definition = 139 | &mut api.openapi(spec.title, Version::parse(env!("CARGO_PKG_VERSION"))?); 140 | api_definition = api_definition 141 | .description(spec.description) 142 | .contact_url(spec.contact_url) 143 | .contact_email(spec.contact_email); 144 | 145 | let mut buffer = File::create(spec.output_path).unwrap(); 146 | api_definition.write(&mut buffer).unwrap(); 147 | } 148 | 149 | Ok(ServerBuilder::new(api, config.context, dropshot_logger).config(config_dropshot)) 150 | } 151 | -------------------------------------------------------------------------------- /rfd-processor/src/main.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use config::{Config, ConfigError, Environment, File}; 6 | use processor::{processor, JobError}; 7 | use serde::{Deserialize, Serialize}; 8 | use std::sync::Arc; 9 | use thiserror::Error; 10 | use tokio::select; 11 | use tracing_appender::non_blocking::NonBlocking; 12 | use tracing_subscriber::EnvFilter; 13 | use updater::RfdUpdateMode; 14 | 15 | use crate::{ 16 | context::{Context, Database}, 17 | scanner::{scanner, ScannerError}, 18 | }; 19 | 20 | mod content; 21 | mod context; 22 | // mod github; 23 | mod pdf; 24 | mod processor; 25 | mod rfd; 26 | mod scanner; 27 | mod search; 28 | mod updater; 29 | mod util; 30 | 31 | #[derive(Debug, Deserialize)] 32 | pub struct AppConfig { 33 | pub log_directory: Option, 34 | #[serde(default)] 35 | pub log_format: LogFormat, 36 | pub processor_enabled: bool, 37 | pub processor_batch_size: i64, 38 | pub processor_interval: u64, 39 | pub processor_capacity: u64, 40 | pub processor_update_mode: RfdUpdateMode, 41 | pub scanner_enabled: bool, 42 | pub scanner_interval: u64, 43 | pub database_url: String, 44 | pub actions: Vec, 45 | pub auth: AuthConfig, 46 | pub source: GitHubSourceRepo, 47 | #[serde(default)] 48 | pub static_storage: Vec, 49 | #[serde(default)] 50 | pub pdf_storage: Option, 51 | #[serde(default)] 52 | pub search_storage: Vec, 53 | } 54 | 55 | #[derive(Debug, Default, Deserialize)] 56 | #[serde(rename_all = "kebab-case")] 57 | pub enum LogFormat { 58 | Pretty, 59 | // The default value is used to avoid breaking old configuration files. 60 | #[default] 61 | Json, 62 | } 63 | 64 | #[derive(Debug, Error)] 65 | pub enum AppError { 66 | #[error("Job task failed")] 67 | Job(#[source] JobError), 68 | #[error("Scanner task failed")] 69 | Scanner(#[source] ScannerError), 70 | } 71 | 72 | #[derive(Debug, Deserialize, Serialize)] 73 | pub struct AuthConfig { 74 | pub github: GitHubAuthConfig, 75 | } 76 | 77 | #[derive(Debug, Deserialize, Serialize)] 78 | #[serde(untagged)] 79 | pub enum GitHubAuthConfig { 80 | Installation { 81 | app_id: i64, 82 | installation_id: i64, 83 | private_key: String, 84 | }, 85 | User { 86 | token: String, 87 | }, 88 | } 89 | 90 | #[derive(Debug, Deserialize, Serialize)] 91 | pub struct GitHubSourceRepo { 92 | pub owner: String, 93 | pub repo: String, 94 | pub path: String, 95 | pub default_branch: String, 96 | } 97 | 98 | #[derive(Debug, Deserialize, Serialize)] 99 | pub struct StaticStorageConfig { 100 | pub bucket: String, 101 | } 102 | 103 | #[derive(Debug, Deserialize, Serialize)] 104 | pub struct PdfStorageConfig { 105 | pub drive: Option, 106 | pub folder: String, 107 | } 108 | 109 | #[derive(Debug, Deserialize, Serialize)] 110 | pub struct SearchConfig { 111 | pub host: String, 112 | pub key: String, 113 | pub index: String, 114 | } 115 | 116 | impl AppConfig { 117 | pub fn new(config_sources: Option>) -> Result { 118 | let mut config = Config::builder() 119 | .add_source(File::with_name("config.toml").required(false)) 120 | .add_source(File::with_name("rfd-processor/config.toml").required(false)); 121 | 122 | for source in config_sources.unwrap_or_default() { 123 | config = config.add_source(File::with_name(&source).required(false)); 124 | } 125 | 126 | config 127 | .add_source(Environment::default()) 128 | .build()? 129 | .try_deserialize() 130 | } 131 | } 132 | 133 | #[tokio::main] 134 | async fn main() -> Result<(), Box> { 135 | let mut args = std::env::args(); 136 | let _ = args.next(); 137 | let config_path = args.next(); 138 | 139 | let config = AppConfig::new(config_path.map(|path| vec![path]))?; 140 | 141 | let (writer, _guard) = if let Some(log_directory) = &config.log_directory { 142 | let file_appender = tracing_appender::rolling::daily(log_directory, "rfd-processor.log"); 143 | tracing_appender::non_blocking(file_appender) 144 | } else { 145 | NonBlocking::new(std::io::stdout()) 146 | }; 147 | 148 | let subscriber = tracing_subscriber::fmt() 149 | .with_file(false) 150 | .with_line_number(false) 151 | .with_env_filter(EnvFilter::from_default_env()) 152 | .with_writer(writer); 153 | 154 | match config.log_format { 155 | LogFormat::Pretty => subscriber.pretty().init(), 156 | LogFormat::Json => subscriber.json().init(), 157 | } 158 | 159 | let ctx = Arc::new(Context::new(Database::new(&config.database_url).await, &config).await?); 160 | 161 | let scanner_ctx = ctx.clone(); 162 | let scanner_handle = tokio::spawn(async move { 163 | scanner(scanner_ctx).await?; 164 | Ok::<_, ScannerError>(()) 165 | }); 166 | 167 | let processor_ctx = ctx.clone(); 168 | let processor_handle = tokio::spawn(async move { 169 | processor(processor_ctx).await?; 170 | Ok::<_, JobError>(()) 171 | }); 172 | 173 | // Tasks should run for the lifetime of the program. If any of them complete for any reason 174 | // then the entire application should exit 175 | let error = select! { 176 | value = processor_handle => { 177 | tracing::info!(?value, "Processor task exited"); 178 | value?.map_err(AppError::Job) 179 | } 180 | value = scanner_handle => { 181 | tracing::info!(?value, "Scanner task exited"); 182 | value?.map_err(AppError::Scanner) 183 | } 184 | }; 185 | 186 | Ok(error?) 187 | } 188 | -------------------------------------------------------------------------------- /rfd-processor/src/updater/update_pdfs.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use async_trait::async_trait; 6 | use newtype_uuid::TypedUuid; 7 | use rfd_github::GitHubRfdUpdate; 8 | use rfd_model::{ 9 | schema_ext::PdfSource, 10 | storage::{DbError, RfdPdfStore}, 11 | NewRfdPdf, 12 | }; 13 | use tracing::instrument; 14 | use v_model::storage::StoreError; 15 | 16 | use crate::{ 17 | content::RfdOutputError, 18 | context::Context, 19 | pdf::{PdfFileLocation, PdfStorage}, 20 | rfd::PersistedRfd, 21 | }; 22 | 23 | use super::{ 24 | RfdUpdateAction, RfdUpdateActionContext, RfdUpdateActionErr, RfdUpdateActionResponse, 25 | RfdUpdateMode, 26 | }; 27 | 28 | #[derive(Debug)] 29 | pub struct UpdatePdfs; 30 | 31 | impl UpdatePdfs { 32 | async fn upload( 33 | ctx: &Context, 34 | update: &GitHubRfdUpdate, 35 | new: &mut PersistedRfd, 36 | mode: RfdUpdateMode, 37 | ) -> Result, RfdOutputError> { 38 | // Generate the PDFs for the RFD 39 | let pdf = match new 40 | .content() 41 | .map_err(|err| RfdOutputError::ContentFailure(err))? 42 | .to_pdf(&ctx.github.client, &update.number, &update.location) 43 | .await 44 | { 45 | Ok(pdf) => { 46 | tracing::info!("Generated RFD pdf"); 47 | pdf 48 | } 49 | Err(err) => { 50 | match &err { 51 | RfdOutputError::FormatNotSupported => { 52 | tracing::info!("RFD is not in a format that supports PDF output"); 53 | 54 | // If a RFD does not support PDF output than we do not want to report an 55 | // error. We return early instead 56 | return Ok(vec![]); 57 | } 58 | inner => { 59 | tracing::error!(?inner, "Failed trying to generate PDF for RFD"); 60 | return Err(err); 61 | } 62 | } 63 | } 64 | }; 65 | 66 | // Upload the generate PDF 67 | tracing::info!(existing_id = ?new.pdf_external_id, filename = ?new.get_pdf_filename(), ?pdf.number, "Uploading PDF version"); 68 | 69 | let store_results = match mode { 70 | RfdUpdateMode::Read => Vec::new(), 71 | RfdUpdateMode::Write => { 72 | ctx.pdf 73 | .store_rfd_pdf( 74 | new.pdf_external_id.as_ref().map(|s| s.as_str()), 75 | &new.get_pdf_filename(), 76 | &pdf, 77 | ) 78 | .await 79 | } 80 | }; 81 | 82 | Ok(store_results 83 | .into_iter() 84 | .enumerate() 85 | .filter_map(|(i, result)| match result { 86 | Ok(file) => Some(file), 87 | Err(err) => { 88 | tracing::error!(?err, storage_index = i, "Failed to store PDF"); 89 | None 90 | } 91 | }) 92 | .collect::>()) 93 | } 94 | } 95 | 96 | #[async_trait] 97 | impl RfdUpdateAction for UpdatePdfs { 98 | #[instrument(skip(self, ctx, new), err(Debug))] 99 | async fn run( 100 | &self, 101 | ctx: &mut RfdUpdateActionContext, 102 | new: &mut PersistedRfd, 103 | mode: RfdUpdateMode, 104 | ) -> Result { 105 | // TODO: This updater should not upload a new version if there were no material changes to 106 | // the RFD. This is slightly tricky as we need to consider the contents of the RFD itself 107 | // as well as any external documents that may become embedded in it. It would be great if 108 | // we could hash the generated PDF, but from past attempts PDFs generated via asciidoctor-pdf 109 | // are not deterministic across systems 110 | 111 | let RfdUpdateActionContext { ctx, update, .. } = ctx; 112 | 113 | // On each update a PDF is uploaded (possibly overwriting an existing file) 114 | let pdf_locations = Self::upload(ctx, update, new, mode) 115 | .await 116 | .map_err(|err| RfdUpdateActionErr::Continue(Box::new(err)))?; 117 | 118 | // Under the assumption that the PDF filename only changes if the revision id has also 119 | // changed, then this upsert will only create a new rows per revision. In any other case, 120 | // the upsert will hit a constraint conflict and drop the insert. The upsert call itself 121 | // should handle this case. 122 | for pdf_location in pdf_locations { 123 | tracing::trace!(?new.revision.id, source = ?PdfSource::Google, ?pdf_location, "Attempt to upsert PDF record"); 124 | 125 | let response = RfdPdfStore::upsert( 126 | &ctx.db.storage, 127 | NewRfdPdf { 128 | id: TypedUuid::new_v4(), 129 | rfd_revision_id: new.revision.id, 130 | source: PdfSource::Google, 131 | link: pdf_location.url, 132 | rfd_id: new.rfd.id, 133 | external_id: pdf_location.external_id, 134 | }, 135 | ) 136 | .await; 137 | 138 | match response { 139 | Ok(_) => 140 | /* Upsert succeeded, nothing to do */ 141 | { 142 | () 143 | } 144 | 145 | // A not found error will be returned in the case of a conflict. This is expected 146 | // and should not cause the function to return 147 | Err(StoreError::Db(DbError::NotFound)) => { 148 | tracing::debug!("Dropping not found database response"); 149 | } 150 | Err(err) => { 151 | tracing::warn!(?err, "Updating RFD pdf link records failed"); 152 | return Err(RfdUpdateActionErr::Continue(Box::new(err))); 153 | } 154 | } 155 | } 156 | 157 | Ok(RfdUpdateActionResponse::default()) 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /rfd-cli/src/cmd/auth/login.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use std::ops::Add; 6 | 7 | use anyhow::Result; 8 | use chrono::{Duration, NaiveDate, Utc}; 9 | use clap::{Parser, Subcommand, ValueEnum}; 10 | use futures::stream::StreamExt; 11 | use oauth2::{basic::BasicTokenType, EmptyExtraTokenFields, StandardTokenResponse, TokenResponse}; 12 | use rfd_sdk::types::OAuthProviderName; 13 | 14 | use crate::{cmd::auth::oauth, Context}; 15 | 16 | // Authenticates and generates an access token for interacting with the api 17 | #[derive(Parser, Debug, Clone)] 18 | #[clap(name = "login")] 19 | pub struct Login { 20 | #[command(subcommand)] 21 | provider: LoginProviderCommand, 22 | #[arg(short = 'm', default_value = "id")] 23 | mode: AuthenticationMode, 24 | } 25 | 26 | impl Login { 27 | pub async fn run(&self, ctx: &mut Context) -> Result<()> { 28 | let access_token = self.provider.run(ctx, &self.mode).await?; 29 | 30 | ctx.config.set_token(access_token); 31 | ctx.config.save()?; 32 | 33 | Ok(()) 34 | } 35 | } 36 | 37 | #[derive(Debug, Clone, Subcommand)] 38 | pub enum LoginProviderCommand { 39 | #[clap(name = "github")] 40 | /// Login via GitHub 41 | GitHub, 42 | /// Login via Google 43 | Google, 44 | /// Login with arbitrary details for local development 45 | #[cfg(feature = "local-dev")] 46 | Local { 47 | /// The email to authenticate as 48 | email: String, 49 | /// An arbitrary external id to uniquely identify this user 50 | external_id: String, 51 | }, 52 | } 53 | 54 | #[derive(ValueEnum, Debug, Clone, PartialEq)] 55 | pub enum AuthenticationMode { 56 | /// Retrieve and store an identity token. Identity mode is the default and should be used to 57 | /// when you do not require extended (multi-day) access 58 | #[value(name = "id")] 59 | Identity, 60 | /// Retrieve and store an api token. Token mode should be used when you want to authenticate 61 | /// a machine for continued access. This requires the permission to create api tokens 62 | #[value(name = "token")] 63 | Token, 64 | } 65 | 66 | pub struct OAuthProviderRunner(OAuthProviderName); 67 | 68 | #[cfg(feature = "local-dev")] 69 | 70 | pub struct LocalProviderRunner { 71 | email: String, 72 | external_id: String, 73 | } 74 | 75 | pub trait ProviderRunner { 76 | async fn run(&self, ctx: &mut Context, mode: &AuthenticationMode) -> Result; 77 | } 78 | 79 | impl ProviderRunner for OAuthProviderRunner { 80 | async fn run(&self, ctx: &mut Context, mode: &AuthenticationMode) -> Result { 81 | let provider = ctx 82 | .client()? 83 | .get_device_provider() 84 | .provider(self.0.to_string()) 85 | .send() 86 | .await?; 87 | 88 | let oauth_client = oauth::DeviceOAuth::new(provider.into_inner())?; 89 | let details = oauth_client.get_device_authorization().await?; 90 | 91 | println!( 92 | "To complete login visit: {} and enter {}", 93 | details.verification_uri().as_str(), 94 | details.user_code().secret() 95 | ); 96 | 97 | let token_response = oauth_client.login(&details).await; 98 | 99 | let identity_token = match token_response { 100 | Ok(token) => Ok(token.access_token().to_owned()), 101 | Err(err) => Err(anyhow::anyhow!("Authentication failed: {}", err)), 102 | }?; 103 | 104 | if mode == &AuthenticationMode::Token { 105 | let client = ctx.new_client(Some(identity_token.secret()))?; 106 | let user = client.get_self().send().await?; 107 | Ok(client 108 | .create_api_user_token() 109 | .user_id(&user.info.id) 110 | .body_map(|body| body.expires_at(Utc::now().add(Duration::days(365)))) 111 | .send() 112 | .await? 113 | .key 114 | .to_string()) 115 | } else { 116 | Ok(identity_token.secret().to_string()) 117 | } 118 | } 119 | } 120 | 121 | #[cfg(feature = "local-dev")] 122 | 123 | impl ProviderRunner for LocalProviderRunner { 124 | async fn run(&self, ctx: &mut Context, mode: &AuthenticationMode) -> Result { 125 | let identity_token = ctx 126 | .client()? 127 | .local_login() 128 | .body_map(|body| { 129 | body.email(self.email.clone()) 130 | .external_id(self.external_id.clone()) 131 | }) 132 | .send() 133 | .await? 134 | .into_inner(); 135 | 136 | let mut bytes = identity_token.into_inner(); 137 | 138 | let mut data = vec![]; 139 | while let Some(chunk) = bytes.next().await { 140 | data.append(&mut chunk?.to_vec()); 141 | } 142 | 143 | let identity_token = match serde_json::from_slice::< 144 | StandardTokenResponse, 145 | >(&data) 146 | { 147 | Ok(token) => Ok(token.access_token().to_owned()), 148 | Err(err) => Err(anyhow::anyhow!("Authentication failed: {}", err)), 149 | }?; 150 | 151 | if mode == &AuthenticationMode::Token { 152 | let client = ctx.new_client(Some(identity_token.secret()))?; 153 | let user = client.get_self().send().await?; 154 | Ok(client 155 | .create_api_user_token() 156 | .user_id(&user.info.id) 157 | .body_map(|body| body.expires_at(Utc::now().add(Duration::days(365)))) 158 | .send() 159 | .await? 160 | .key 161 | .to_string()) 162 | } else { 163 | Ok(identity_token.secret().to_string()) 164 | } 165 | } 166 | } 167 | 168 | impl ProviderRunner for LoginProviderCommand { 169 | async fn run(&self, ctx: &mut Context, mode: &AuthenticationMode) -> Result { 170 | match self { 171 | LoginProviderCommand::GitHub => { 172 | OAuthProviderRunner(OAuthProviderName::Github) 173 | .run(ctx, mode) 174 | .await 175 | } 176 | LoginProviderCommand::Google => { 177 | OAuthProviderRunner(OAuthProviderName::Google) 178 | .run(ctx, mode) 179 | .await 180 | } 181 | #[cfg(feature = "local-dev")] 182 | LoginProviderCommand::Local { email, external_id } => { 183 | LocalProviderRunner { 184 | email: email.to_string(), 185 | external_id: external_id.to_string(), 186 | } 187 | .run(ctx, mode) 188 | .await 189 | } 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /rfd-processor/src/search/mod.rs: -------------------------------------------------------------------------------- 1 | // This Source Code Form is subject to the terms of the Mozilla Public 2 | // License, v. 2.0. If a copy of the MPL was not distributed with this 3 | // file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | 5 | use hmac::{Hmac, Mac}; 6 | use md5::Md5; 7 | use meilisearch_sdk::{ 8 | client::Client, 9 | errors::{Error as MeiliError, ErrorCode}, 10 | indexes::Index, 11 | settings::Settings, 12 | }; 13 | use parse_rfd::{parse, ParsedDoc, ParserError, Section}; 14 | use rfd_data::RfdNumber; 15 | use serde::{Deserialize, Serialize}; 16 | use std::{cmp::min, collections::HashMap}; 17 | use thiserror::Error; 18 | use tracing::instrument; 19 | 20 | #[derive(Debug, Error)] 21 | pub enum SearchError { 22 | #[error(transparent)] 23 | Client(#[from] MeiliError), 24 | #[error(transparent)] 25 | Parse(#[from] ParserError), 26 | } 27 | 28 | #[derive(Debug)] 29 | pub struct RfdSearchIndex { 30 | client: Client, 31 | index: String, 32 | } 33 | 34 | impl RfdSearchIndex { 35 | pub fn new( 36 | host: impl Into, 37 | api_key: impl Into, 38 | index: impl Into, 39 | ) -> Result { 40 | Ok(Self { 41 | client: Client::new(host, Some(api_key))?, 42 | index: index.into(), 43 | }) 44 | } 45 | 46 | /// Trigger updating the search index for the RFD. 47 | #[instrument(skip(self, content), fields(index = ?self.index), err(Debug))] 48 | pub async fn index_rfd( 49 | &self, 50 | rfd_number: &RfdNumber, 51 | content: &str, 52 | public: bool, 53 | ) -> Result<(), SearchError> { 54 | let index = self.client.index(&self.index); 55 | 56 | let lookup = self.find_rfd_ids(&index, rfd_number).await; 57 | 58 | // The index may not exist yet if this is the first RFD being indexed. In that case the err 59 | // returned from look up is noted and then discarded. 60 | match lookup { 61 | Ok(ids_to_delete) => { 62 | tracing::info!(?ids_to_delete, "Deleting documents for RFD"); 63 | index.delete_documents(&ids_to_delete).await?; 64 | } 65 | Err(SearchError::Client(MeiliError::Meilisearch(err))) 66 | if err.error_code == ErrorCode::IndexNotFound 67 | || err.error_code == ErrorCode::InvalidSearchFilter => 68 | { 69 | tracing::info!( 70 | ?err, 71 | "Failed to find index during deletion lookup. Creating index and filters" 72 | ); 73 | 74 | let settings = Settings::new().with_filterable_attributes(["rfd_number"]); 75 | index.set_settings(&settings).await?; 76 | } 77 | Err(err) => { 78 | return Err(err)?; 79 | } 80 | } 81 | 82 | let mut parsed = Self::parse_document(rfd_number, content)?; 83 | for doc in parsed.iter_mut() { 84 | doc.public = public; 85 | } 86 | 87 | tracing::info!(count = parsed.len(), "Parsed RFD into sections to index"); 88 | 89 | index.add_documents(&parsed, Some("objectID")).await?; 90 | 91 | Ok(()) 92 | } 93 | 94 | #[instrument(skip(self, index))] 95 | pub async fn find_rfd_ids( 96 | &self, 97 | index: &Index, 98 | rfd_number: &RfdNumber, 99 | ) -> Result, SearchError> { 100 | let mut query = index.search(); 101 | let filter = format!("rfd_number = {}", rfd_number); 102 | query.with_array_filter(vec![&filter]); 103 | 104 | tracing::trace!(?filter, "Search for existing RFDs"); 105 | 106 | let results = query.execute::().await?; 107 | 108 | Ok(results 109 | .hits 110 | .into_iter() 111 | .map(|search_result| search_result.result.object_id) 112 | .collect::>()) 113 | } 114 | 115 | #[instrument(skip(content), err(Debug))] 116 | pub fn parse_document( 117 | rfd_number: &RfdNumber, 118 | content: &str, 119 | ) -> Result, SearchError> { 120 | let ParsedDoc { title, sections } = parse(content)?; 121 | Ok(sections 122 | .into_iter() 123 | .map(|section| IndexDocument::new(section, rfd_number, &title)) 124 | .collect::>()) 125 | } 126 | } 127 | 128 | #[derive(Debug, Clone, Deserialize, Serialize)] 129 | pub struct RfdId { 130 | #[serde(rename = "objectID")] 131 | object_id: String, 132 | } 133 | 134 | type HmacMd5 = Hmac; 135 | 136 | #[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] 137 | pub struct IndexDocument { 138 | #[serde(rename = "objectID")] 139 | pub object_id: String, 140 | pub name: String, 141 | pub level: usize, 142 | pub anchor: String, 143 | pub content: String, 144 | pub rfd_number: i32, 145 | #[serde(flatten)] 146 | pub hierarchy: HashMap, 147 | #[serde(flatten)] 148 | pub hierarchy_radio: HashMap, 149 | pub public: bool, 150 | } 151 | 152 | impl IndexDocument { 153 | pub fn new(section: Section, rfd_number: &RfdNumber, title: &str) -> Self { 154 | let level = section.parents.len() + 1; 155 | 156 | let mut hierarchy_radio = HashMap::new(); 157 | if level == 1 { 158 | hierarchy_radio.insert("hierarchy_radio_lvl1".to_string(), section.name.clone()); 159 | } else { 160 | hierarchy_radio.insert( 161 | format!("hierarchy_radio_lvl{}", min(5, level)), 162 | section.parents[section.parents.len() - 1].clone(), 163 | ); 164 | } 165 | 166 | let mut hierarchy = HashMap::new(); 167 | hierarchy.insert( 168 | "hierarchy_lvl0".to_string(), 169 | format!("RFD {} {}", rfd_number, title), 170 | ); 171 | hierarchy.insert("hierarchy_lvl1".to_string(), section.name.to_string()); 172 | 173 | for (i, section_name) in section.parents.into_iter().enumerate() { 174 | hierarchy.insert(format!("hierarchy_lvl{}", i + 2), section_name); 175 | } 176 | 177 | // The hash here is only intended to enforce uniqueness amongst documents. md5 and the 178 | // statically defined key is being used to maintain backward compatibility with previous 179 | // implementations. None of the key, the ids, nor hash are required to be kept secret 180 | let mut mac = HmacMd5::new_from_slice("dsflkajsdf".as_bytes()) 181 | .expect("Statically defined key should always be valid"); 182 | mac.update(rfd_number.as_number_string().as_bytes()); 183 | mac.update(section.section_id.as_bytes()); 184 | let object_id = hex::encode(&mac.finalize().into_bytes()[..]); 185 | 186 | Self { 187 | object_id, 188 | name: section.name, 189 | level, 190 | anchor: section.section_id, 191 | content: section.content, 192 | rfd_number: rfd_number.into(), 193 | hierarchy, 194 | hierarchy_radio, 195 | public: false, 196 | } 197 | } 198 | } 199 | --------------------------------------------------------------------------------