├── .devcontainer └── devcontainer.json ├── .github └── workflows │ ├── azure_deploy.yml │ ├── clippy.yml │ ├── dockerfile_build.yml │ └── dockerfile_lint.yml ├── .gitignore ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── README.md └── src └── main.rs /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/rust 3 | { 4 | "name": "Rust", 5 | // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile 6 | "image": "mcr.microsoft.com/devcontainers/rust:0-1-bullseye", 7 | "features": { 8 | "ghcr.io/devcontainers/features/docker-in-docker:2": {} 9 | }, 10 | 11 | // Use 'mounts' to make the cargo cache persistent in a Docker Volume. 12 | // "mounts": [ 13 | // { 14 | // "source": "devcontainer-cargo-cache-${devcontainerId}", 15 | // "target": "/usr/local/cargo", 16 | // "type": "volume" 17 | // } 18 | // ] 19 | 20 | // Features to add to the dev container. More info: https://containers.dev/features. 21 | // "features": {}, 22 | 23 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 24 | // "forwardPorts": [], 25 | 26 | // Use 'postCreateCommand' to run commands after the container is created. 27 | // "postCreateCommand": "rustc --version", 28 | "customizations": { 29 | // Configure properties specific to VS Code. 30 | "vscode": { 31 | // Set *default* container specific settings.json values on container create. 32 | // Add the IDs of extensions you want installed when the container is created. 33 | "extensions": [ 34 | "GitHub.copilot", 35 | "MS-vsliveshare.vsliveshare", 36 | "rust-lang.rust-analyzer" 37 | ] 38 | } 39 | }, 40 | 41 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 42 | // "remoteUser": "root" 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/azure_deploy.yml: -------------------------------------------------------------------------------- 1 | name: Trigger auto deployment for demo-container 2 | 3 | env: 4 | AZURE_CONTAINER_APP_NAME: demo-rust-container 5 | AZURE_GROUP_NAME: demo-rust-container 6 | 7 | on: 8 | # Automatically trigger it when detected changes in repo. Remove comments to enable 9 | #push: 10 | # branches: 11 | # [ main ] 12 | 13 | # Allow mannually trigger 14 | workflow_dispatch: 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - name: Checkout to the branch 22 | uses: actions/checkout@v2 23 | 24 | - name: Set up Docker Buildx 25 | uses: docker/setup-buildx-action@v1 26 | 27 | - name: Log in to GitHub container registry 28 | uses: docker/login-action@v1.10.0 29 | with: 30 | registry: ghcr.io 31 | username: ${{ github.actor }} 32 | password: ${{ github.token }} 33 | 34 | - name: Lowercase the repo name and username 35 | run: echo "REPO=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} 36 | 37 | - name: Build and push container image to registry 38 | uses: docker/build-push-action@v2 39 | with: 40 | push: true 41 | tags: ghcr.io/${{ env.REPO }}:${{ github.sha }} 42 | file: ./Dockerfile 43 | 44 | deploy: 45 | runs-on: ubuntu-latest 46 | needs: build 47 | 48 | steps: 49 | - name: Azure Login 50 | uses: azure/login@v1 51 | with: 52 | creds: ${{ secrets.AZURE_CREDENTIALS }} 53 | 54 | - name: Lowercase the repo name and username 55 | run: echo "REPO=${GITHUB_REPOSITORY,,}" >>${GITHUB_ENV} 56 | 57 | - name: Deploy to containerapp 58 | uses: azure/CLI@v1 59 | with: 60 | inlineScript: | 61 | az config set extension.use_dynamic_install=yes_without_prompt 62 | az containerapp registry set -n ${{ env.AZURE_CONTAINER_APP_NAME }} -g ${{ env.AZURE_GROUP_NAME }} --server ghcr.io --username ${{ github.actor }} --password ${{ secrets.PAT }} 63 | az containerapp update -n ${{ env.AZURE_CONTAINER_APP_NAME }} -g ${{ env.AZURE_GROUP_NAME }} --image ghcr.io/${{ env.REPO }}:${{ github.sha }} 64 | -------------------------------------------------------------------------------- /.github/workflows/clippy.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request 3 | 4 | name: Clippy check 5 | jobs: 6 | clippy_check: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v1 10 | - run: rustup component add clippy 11 | - uses: actions-rs/clippy-check@v1 12 | with: 13 | token: ${{ secrets.GITHUB_TOKEN }} 14 | args: --no-deps 15 | -------------------------------------------------------------------------------- /.github/workflows/dockerfile_build.yml: -------------------------------------------------------------------------------- 1 | name: Dockerfile building for PRs 2 | 3 | on: 4 | pull_request: 5 | branches: [ "main" ] 6 | # Allow mannually trigger 7 | workflow_dispatch: 8 | 9 | jobs: 10 | 11 | build: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Build the container image 18 | run: docker build . 19 | -------------------------------------------------------------------------------- /.github/workflows/dockerfile_lint.yml: -------------------------------------------------------------------------------- 1 | # create a github action workflow to lint dockerfiles using hadolint 2 | # 3 | 4 | name: Dockerfile Lint 5 | 6 | on: 7 | pull_request: 8 | paths: 9 | - '**/Dockerfile' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | 14 | dockerfile_lint: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | # use hadolint to lint dockerfiles 19 | - uses: hadolint/hadolint-action@v3.1.0 20 | with: 21 | dockerfile: Dockerfile 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | /target/ 4 | 5 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 6 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 7 | Cargo.lock 8 | 9 | # These are backup files generated by rustfmt 10 | **/*.rs.bk 11 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rust-tokenizers-api" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [dependencies] 9 | actix-web = "4" 10 | env_logger = "0.10.0" 11 | serde = { version = "1.0", features = ["derive"] } 12 | tokenizers = "0.13.2" 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:1.67.0-buster as builder 2 | 3 | WORKDIR /usr/src/app 4 | 5 | COPY . . 6 | 7 | RUN cargo build --release 8 | 9 | # Now copy it into our base image. 10 | FROM gcr.io/distroless/cc-debian10 11 | 12 | COPY --from=builder /usr/src/app/target/release/rust-tokenizers-api /usr/local/bin/rust-tokenizers-api 13 | CMD ["rust-tokenizers-api"] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Alfredo Deza 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Create a powerful Tokenizer API using distroless 2 | 3 | By using distroless, you ensure a low weight for your container. This full example will get you everything you need to run an API in a container that you can then use to deploy to a cloud provider like Azure. 4 | 5 | This repository gives you a good starting point with a Dockerfile, GitHub Actions workflow, and Rust code. 6 | 7 | ## Generate a PAT 8 | 9 | The access token will need to be added as an Action secret. [Create one](https://github.com/settings/tokens/new?description=Azure+Rust+Container+Apps+access&scopes=write:packages) with enough permissions to write to packages. 10 | 11 | Copy the generated token and add it as a [Github repository secret](/../../settings/secrets/actions/new) with the name `PAT`. (_If that link doesn't work, make sure you're reading this on your own copy of the repo, not the original template._) 12 | 13 | 14 | ## Create an Azure Service Principal 15 | 16 | You'll need the following: 17 | 18 | 1. An Azure subscription ID [find it here](https://portal.azure.com/#view/Microsoft_Azure_Billing/SubscriptionsBlade) or [follow this guide](https://docs.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id) 19 | 1. A Service Principal with the following details the AppID, password, and tenant information. Create one with: `az ad sp create-for-rbac -n "REST API Service Principal"` and assign the IAM role for the subscription. Alternatively set the proper role access using the following command (use a real subscription id and replace it): 20 | 21 | ``` 22 | az ad sp create-for-rbac --name "CICD" --role contributor --scopes /subscriptions/$AZURE_SUBSCRIPTION_ID --sdk-auth 23 | ``` 24 | 25 | Capture the output and add it as a [Github repository secret](/../../settings/secrets/actions/new) with the name `AZURE_CREDENTIALS`. (_If that link doesn't work, make sure you're reading this on your own copy of the repo, not the original template._) 26 | 27 | 28 | ## Azure Container Apps 29 | 30 | Make sure you have one instance already created, and then capture the name and resource group. These will be used in the workflow file. 31 | 32 | ## No need to change defaults 33 | 34 | Unlike other language runtimes like Python, you don't need to change the default container service. Rust will be more than happy to use a single CPU! 35 | 36 | ## Gotchas 37 | 38 | There are a few things that might get you into a failed state when deploying: 39 | 40 | * Not using authentication for accessing the remote registry (ghcr.io in this case). Authentication is always required 41 | * Not using a PAT (Personal Access Token) or using a PAT that doesn't have write permissions for "packages". 42 | * Different port than 80 in the container. By default Azure Container Apps use 80. Update to match the container. 43 | 44 | If running into trouble, check logs in the portal or use the following with the Azure CLI: 45 | 46 | ``` 47 | az containerapp logs show --name $CONTAINER_APP_NAME --resource-group $RESOURCE_GROUP_NAME --follow 48 | ``` 49 | 50 | Update both variables to match your environment 51 | 52 | **NOTE** Settings for Packages in your repo may need updating. Go to [Action Settings](/../../settings/actions) and scroll down to _"Workflow Permissions"_ and make sure it shows _"Read and write permissions"_ as selected, otherwise you'll see a `403 Forbidden` 53 | 54 | ## API Best Practices 55 | 56 | Although there are a few best practices for using the FastAPI framework, there are many different other suggestions to build solid HTTP APIs that can be applicable anywhere. 57 | 58 | ### Use HTTP Error codes 59 | The HTTP specification has several error codes available. Make use of the appropriate error code to match the condition that caused it. For example the `401` HTTP code can be used when access is unauthorized. You shouldn't use a single error code as a catch-all error. 60 | 61 | Here are some common scenarios associated with HTTP error codes: 62 | 63 | - `400 Bad request`: Use this to indicate a schema problem. For example if the server expected a string but got an integer 64 | - `401 Unauthorized`: When authentication is required and it wasn't present or satisfied 65 | - `404 Not found`: When the resource doesn't exist 66 | 67 | Note that it is a good practice to use `404 Not Found` to protect from requests that try to find if a resource exists without being authenticated. A good example of this is a service that doesn't want to expose usernames unless you are authenticated. 68 | 69 | 70 | ### Accept request types sparingly 71 | 72 | | GET | POST | PUT | HEAD| 73 | |---|---|---|---| 74 | | Read Only | Write Only | Update existing | Does it exist? | 75 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{get, middleware::Logger, post, web, App, HttpResponse, HttpServer, Responder}; 2 | use serde::Deserialize; 3 | use std::thread; 4 | use tokenizers::tokenizer::Tokenizer; 5 | 6 | //create a struct that will be used to deserialize the JSON payload 7 | #[derive(Deserialize)] 8 | struct Text { 9 | text: String, 10 | } 11 | 12 | //create a struct that will be used to serialize the JSON response 13 | #[derive(serde::Serialize)] 14 | struct TokenizedText { 15 | tokens: Vec, 16 | } 17 | 18 | async fn tokenize_text(pretrained_model: String, text: String) -> Vec { 19 | // create a thread to load the tokenizer because this is a blocking call that makes actix panic 20 | let handle = thread::spawn(move || { 21 | // create the tokenizer 22 | return Tokenizer::from_pretrained(pretrained_model, None); 23 | }); 24 | 25 | // shrey update this 26 | // updated 27 | let tokenizer = handle.join().expect("Failed to join thread"); 28 | 29 | // encode the text using the tokenizer 30 | let encoded = tokenizer 31 | .expect("could not create the tokenizer") 32 | .encode(text.clone(), false) 33 | .expect("could not read the text"); 34 | 35 | // get the tokens from the encoding by unwrapping the result 36 | let tokens = encoded.get_tokens(); 37 | let tokenized_values = Vec::from(tokens); 38 | 39 | return tokenized_values; 40 | } 41 | 42 | #[post("tokenizers/{pretrained_model}")] 43 | async fn tokenize( 44 | pretrained_model: web::Path, 45 | text: web::Json, 46 | ) -> impl Responder { 47 | let pretrained_model = pretrained_model.into_inner(); 48 | let tokenized_values = tokenize_text(pretrained_model, text.text.clone()).await; 49 | 50 | // return the tokenized values 51 | HttpResponse::Ok().json(TokenizedText { 52 | tokens: tokenized_values, 53 | }) 54 | } 55 | 56 | 57 | #[get("/")] 58 | async fn index() -> impl Responder { 59 | HttpResponse::Ok().body("

Summarization Service from Duke University

") 60 | } 61 | 62 | #[actix_web::main] 63 | async fn main() -> std::io::Result<()> { 64 | env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); 65 | 66 | HttpServer::new(|| { 67 | App::new() 68 | .wrap(Logger::default()) 69 | .wrap(Logger::new("%a %{User-Agent}i")) 70 | .service(tokenize) 71 | .service(index) 72 | }) 73 | .bind(("0.0.0.0", 8000))? 74 | .run() 75 | .await 76 | } 77 | --------------------------------------------------------------------------------