├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── _new_instance_form.yml │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── build.yml │ ├── deployment-management-bot.yml │ ├── instance-configuration-validation-bot.yml │ ├── paper.yml │ ├── resources │ ├── base_legistar_scraper.py │ └── custom_scraper.py │ ├── scripts │ ├── parse_form.py │ ├── requirements.txt │ └── validate_form.py │ └── test.yml ├── .gitignore ├── .zenodo.json ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── cookiecutter.json ├── hooks ├── post_gen_project.py └── pre_gen_project.py ├── local_extensions.py ├── paper ├── assets │ ├── cdp_core_infrastructure.png │ └── event-page-screenshot.png ├── paper.bib └── paper.md ├── setup.cfg └── {{ cookiecutter.hosting_github_repo_name }} ├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── build-main.yml │ ├── check-pr.yml │ ├── deploy-infra.yml │ ├── deploy-web.yml │ ├── event-gather-pipeline.yml │ ├── event-index-pipeline.yml │ ├── process-special-event.yml │ └── run-script.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Justfile ├── LICENSE ├── README.md ├── SETUP └── README.md ├── admin-docs ├── adding-analytics.md ├── customizing-automated-pipelines.md ├── manual-event-gather.md ├── resources │ ├── backfill-event-gather.png │ ├── cookiecutter-interrupt.png │ ├── example-custom-event.py │ ├── special-event-gather.png │ ├── update-and-git-status.png │ ├── vs-code-status.png │ └── workflows.png ├── running-extra-scripts.md └── updating-to-new-cookiecutter-releases.md ├── cookiecutter.yaml ├── infra └── requirements.txt ├── python ├── cdp_{{ cookiecutter.python_municipality_slug }}_backend │ ├── __init__.py │ └── scraper.py ├── event-gather-config.json ├── event-index-config.json ├── setup.cfg └── setup.py └── web ├── .gitignore ├── package.json ├── public ├── cdp-og-seo.png ├── cdp-twitter-seo.png ├── favicon.ico ├── index.html ├── manifest.json └── robots.txt └── src └── index.jsx /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.yml] 14 | indent_size = 2 15 | 16 | [*.bat] 17 | indent_style = tab 18 | end_of_line = crlf 19 | 20 | [LICENSE] 21 | insert_final_newline = false 22 | 23 | [Makefile] 24 | indent_style = tab 25 | 26 | [*.{diff,patch}] 27 | trim_trailing_whitespace = false 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/_new_instance_form.yml: -------------------------------------------------------------------------------- 1 | name: CDP Instance Configuration 2 | description: Configure and deploy a new CDP instance. 3 | title: "[Instance]: " 4 | labels: 5 | - "new instance" 6 | assignees: 7 | - evamaxfield 8 | body: 9 | - type: markdown 10 | attributes: 11 | value: | 12 | Please fill out the following information to start the process for deploying a new CDP instance. 13 | 14 | Please refer to our documentation on the [cost of maintaining a CDP instance](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment#cost) and verify that you are able to pay the estimated monthly cost of a new CDP instance. 15 | - type: markdown 16 | attributes: 17 | value: '## Instance Configuration Basics' 18 | - type: input 19 | id: municipality_name 20 | attributes: 21 | label: Municipality Name 22 | description: The name of the municipality (town, city, county, etc.) that this CDP Instance will store data for. 23 | placeholder: ex. "Seattle", "King County" 24 | validations: 25 | required: true 26 | - type: input 27 | id: timezone 28 | attributes: 29 | label: Municipality Timezone 30 | description: The timezone of your municipality. See [the TZ database name column](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to find the appropriate timezone string. 31 | placeholder: ex. "America/Los_Angeles" 32 | validations: 33 | required: true 34 | - type: input 35 | id: governing_body_type 36 | attributes: 37 | label: Governing Body Type 38 | description: What type of governing body this instance is for. Must be one of, 'city council', 'county council', 'other', or 'school board'. 39 | placeholder: ex. "city council", "county council" 40 | validations: 41 | required: true 42 | - type: input 43 | id: maintainer_github_name 44 | attributes: 45 | label: Maintainer GitHub Name 46 | description: Who will act as the primary maintainer of the new CDP Instance. 47 | placeholder: ex. "evamaxfield" 48 | validations: 49 | required: true 50 | - type: markdown 51 | attributes: 52 | value: | 53 | ## Recommended Legistar Options 54 | 55 | These fields are entirely optional however if your municipality utilizes Legistar, they are highly recommended as our tools may be entirely able to automate the creation and management of your scraper. 56 | 57 | To find out if your municipality uses Legistar for its legislation management software follow the steps outlined in our [Legistar Documentation](https://councildataproject.org/cdp-scrapers/finding_legistar_id.html). If Legistar is used, this instance configuration process could potentially be automated for you and you can fill in your Legistar Id below. If not, you will need to write a [custom event scraper](https://councildataproject.org/cdp-scrapers/index.html#creating-a-custom-scraper) to complete the process. 58 | 59 | If you plan on using Legistar, all fields in this section are required. I.e. the municipality's Legistar Client Id is required and can be found by following our recommendations in [documentation](https://councildataproject.org/cdp-scrapers/legistar_scraper.html). 60 | 61 | **Note:** If you already have a scraper written and published in `cdp-scrapers`, 62 | you can provide the municipality slug instead of these Legistar options. 63 | - type: input 64 | id: legistar_client_id 65 | attributes: 66 | label: Legistar Client Id 67 | description: If planning on using Legistar, the municipalities Legistar Client Id as described in above documentation. 68 | placeholder: ex. "seattle" 69 | - type: markdown 70 | attributes: 71 | value: '## Optional Infrastructure Options' 72 | - type: input 73 | id: municipality_slug 74 | attributes: 75 | label: Municipality Slug 76 | description: The name of the municipality cleaned for use in infrastructure and certain parts of repository naming. Default - municipality name lowercased and spaces replaced with '-'. This is useful if you want to add more specificity to the generated repository name, i.e. "seattle-wa" instead of simply "seattle". 77 | placeholder: ex. "seattle", "king-county" 78 | - type: input 79 | id: firestore_region 80 | attributes: 81 | label: Firestore Region 82 | description: The desired region to host the firestore instance. ([Firestore docs](https://firebase.google.com/docs/firestore/locations)) Default - 'us-central'. 83 | placeholder: ex. "us-west1", "us-central" 84 | - type: input 85 | id: event_gather_timedelta_lookback_days 86 | attributes: 87 | label: Event Gather Timedelta Lookback Days 88 | description: The number of days to look back from the current date every time the event scraper runs. Default - 2. 89 | placeholder: ex. 2, 4, 10 90 | - type: input 91 | id: event_gather_cron 92 | attributes: 93 | label: Event Gather CRON 94 | description: The event gather CRON configuration. ([GitHub Actions CRON Details](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule)) Default - randomly twice per day, twelve hours apart 95 | placeholder: ex. "26 0,6,12,18 * * *", "17 3,15 * * *" 96 | - type: checkboxes 97 | id: terms 98 | attributes: 99 | label: Code of Conduct 100 | description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/CouncilDataProject/councildataproject.github.io/blob/main/CODE_OF_CONDUCT.md) 101 | options: 102 | - label: I agree to follow this project's Code of Conduct 103 | required: true 104 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Create a report to help us improve cookiecutter-cdp-deployment 4 | labels: bug 5 | --- 6 | 7 | 13 | 14 | ### Describe the Bug 15 | 16 | _A clear and concise description of the bug._ 17 | 18 | ### Expected Behavior 19 | 20 | _What did you expect to happen instead?_ 21 | 22 | ### Reproduction 23 | 24 | _Steps to reproduce the behavior and/or a minimal example that exhibits the behavior._ 25 | 26 | ### Environment 27 | 28 | _Any additional information about your environment._ 29 | 30 | - OS Version: _[e.g. macOS 11.3.1]_ 31 | - Cookiecutter Version: _[e.g. 0.5.0]_ 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest a feature for cookiecutter-cdp-deployment 4 | labels: enhancement 5 | --- 6 | 7 | 13 | 14 | ### Feature Description 15 | 16 | _A clear and concise description of the feature you're requesting._ 17 | 18 | ### Use Case 19 | 20 | _Please provide a use case to help us understand your request in context._ 21 | 22 | ### Solution 23 | 24 | _Please describe your ideal solution._ 25 | 26 | ### Alternatives 27 | 28 | _Please describe any alternatives you've considered, even if you've dismissed them._ 29 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 15 | 16 | ### Link to Relevant Issue 17 | 18 | This pull request resolves # 19 | 20 | ### Description of Changes 21 | 22 | _Include a description of the proposed changes._ 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | commit-message: 8 | prefix: "ci(dependabot):" 9 | 10 | - package-ecosystem: "pip" 11 | directory: "/" 12 | schedule: 13 | interval: "daily" 14 | allow: 15 | - dependency-name: "cdp-backend" 16 | commit-message: 17 | prefix: "ci(dependabot):" 18 | 19 | - package-ecosystem: "npm" 20 | directory: "/" 21 | schedule: 22 | interval: "daily" 23 | allow: 24 | - dependency-name: "@councildataproject/cdp-frontend" 25 | commit-message: 26 | prefix: "ci(dependabot):" 27 | 28 | - package-ecosystem: "github-actions" 29 | directory: "/{{ cookiecutter.hosting_github_repo_name }}/" 30 | schedule: 31 | interval: "daily" 32 | commit-message: 33 | prefix: "ci(dependabot):" 34 | 35 | - package-ecosystem: "pip" 36 | directory: "/{{ cookiecutter.hosting_github_repo_name }}/" 37 | schedule: 38 | interval: "daily" 39 | allow: 40 | - dependency-name: "cdp-backend" 41 | commit-message: 42 | prefix: "ci(dependabot):" 43 | 44 | - package-ecosystem: "npm" 45 | directory: "/{{ cookiecutter.hosting_github_repo_name }}/" 46 | schedule: 47 | interval: "daily" 48 | allow: 49 | - dependency-name: "@councildataproject/cdp-frontend" 50 | commit-message: 51 | prefix: "ci(dependabot):" 52 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build Example Repo 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | # 9 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 10 | # Run every Monday at 23:26:00 UTC (Monday at 15:26:00 PST) 11 | # We offset from the hour and half hour to go easy on the servers :) 12 | - cron: '26 23 * * 1' 13 | 14 | jobs: 15 | build-repo: 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | # Setup languages 22 | - name: Set up Python 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: '3.11' 26 | - name: Setup Node 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: '16.x' 30 | 31 | # Run cookiecutter 32 | - name: Install Cookiecutter 33 | run: | 34 | pip install cookiecutter 35 | - name: Generate Repo 36 | run: | 37 | cookiecutter . --no-input 38 | rm -Rf example/.github/workflows/ 39 | 40 | # Check Python 41 | - name: Install Python Dependencies 42 | run: | 43 | cd example/python/ 44 | pip install .[test] 45 | - name: Lint and Format Python 46 | run: | 47 | cd example/python/ 48 | flake8 cdp_example_backend --count --verbose --show-source --statistics 49 | black --check cdp_example_backend 50 | 51 | # Check Web 52 | - name: Install Web App Dependencies 53 | run: | 54 | cd example/web/ 55 | npm i 56 | - name: Build Web App 57 | run: | 58 | cd example/web/ 59 | npm run build 60 | 61 | # Publish the generated repo 62 | - name: Publish Docs 63 | uses: JamesIves/github-pages-deploy-action@v4 64 | with: 65 | folder: example/ 66 | -------------------------------------------------------------------------------- /.github/workflows/deployment-management-bot.yml: -------------------------------------------------------------------------------- 1 | name: Instance Deployment Bot 2 | 3 | on: 4 | issue_comment: 5 | types: 6 | - created 7 | 8 | jobs: 9 | deploy-instance: 10 | runs-on: ubuntu-latest 11 | if: | 12 | contains(github.event.comment.html_url, '/issues/') && 13 | contains(github.event.comment.body, '/cdp-deploy') 14 | 15 | steps: 16 | ######################################################################### 17 | # Check initiator is a member of CDP 18 | 19 | - name: Get CDP Organization Members 20 | id: cdp-members 21 | env: 22 | GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} 23 | run: | 24 | members="$(gh api -X GET 'orgs/CouncilDataProject/members' -F per_page=100 --paginate --cache 1h --jq '[.[].login] | join("---")')" 25 | echo "::set-output name=members::$members" 26 | 27 | - name: Generate Safe Username Check 28 | id: safe-username 29 | run: | 30 | username=${{ github.event.comment.user.login }} 31 | username="---$username---" 32 | echo "::set-output name=username::$username" 33 | 34 | - name: Check Job Initiator - Message 35 | if: | 36 | !contains( 37 | steps.cdp-members.outputs.members, 38 | steps.safe-username.outputs.username 39 | ) 40 | uses: peter-evans/create-or-update-comment@v3 41 | with: 42 | issue-number: ${{ github.event.issue.number }} 43 | body: | 44 | ## Deployment Status 45 | 46 | ❌ ❌ **Rejected** ❌ ❌ 47 | 48 | User (${{ github.event.comment.user.login }}) attempted to deploy without permissions. 49 | 50 | _Only users which are members of the CouncilDataProject organization can deploy new instances with this bot._ 51 | 52 | **Stopping Deployment Procedure** 53 | 54 | - name: Check Job Initiator - Exit 55 | if: | 56 | !contains( 57 | steps.cdp-members.outputs.members, 58 | steps.safe-username.outputs.username 59 | ) 60 | run: | 61 | exit 1 62 | 63 | ######################################################################### 64 | # Workflow Setup 65 | 66 | - uses: actions/checkout@v4 67 | - name: Set up Python 68 | uses: actions/setup-python@v5 69 | with: 70 | python-version: '3.11' 71 | - name: Install Bot Scripts Dependencies 72 | run: | 73 | pip install --upgrade pip 74 | pip install -r .github/workflows/scripts/requirements.txt 75 | pip install cookiecutter 76 | 77 | # Run Validation Bot to get Configuration Options 78 | - name: Dump Issue Body to File 79 | run: | 80 | echo "${{ github.event.issue.body }}" > issue-body.md 81 | - name: Validate Form and Generate Configuration Files 82 | run: | 83 | python .github/workflows/scripts/validate_form.py issue-body.md 84 | 85 | # Store Cookiecutter Options 86 | - name: Store Cookiecutter Options 87 | id: cookiecutter-options 88 | run: | 89 | set -f 90 | content=$(cat planned-cookiecutter.json) 91 | content="${content//'%'/'%25'}" 92 | content="${content//$'\n'/'%0A'}" 93 | content="${content//$'\r'/'%0D'}" 94 | echo ::set-output name=options::$content 95 | 96 | ######################################################################### 97 | # Run fast checks 98 | 99 | - name: Read generation-options JSON 100 | id: generation-options 101 | run: | 102 | content="$(cat generation-options.json)" 103 | echo "::set-output name=content::$content" 104 | 105 | - name: Error Early - Message 106 | uses: peter-evans/create-or-update-comment@v3 107 | if: | 108 | fromJSON(steps.generation-options.outputs.content).scraper_options == null || 109 | fromJSON(steps.generation-options.outputs.content).maintainer_name == null || 110 | fromJSON(steps.generation-options.outputs.content).repository_path == null 111 | with: 112 | issue-number: ${{ github.event.issue.number }} 113 | body: | 114 | ## Deployment Status 115 | 116 | :warning: :warning: **Configuration Error** :warning: :warning: 117 | 118 | Not all configuration options are present or some options have errors. 119 | 120 | #### Configuration Options 121 | 122 | ```json 123 | ${{ steps.generation-options.outputs.content }} 124 | ``` 125 | 126 | **Stopping Deployment Procedure** 127 | 128 | - name: Error Early - Exit 129 | if: | 130 | fromJSON(steps.generation-options.outputs.content).scraper_options == null || 131 | fromJSON(steps.generation-options.outputs.content).maintainer_name == null || 132 | fromJSON(steps.generation-options.outputs.content).repository_path == null 133 | run: | 134 | exit 1 135 | 136 | ######################################################################### 137 | # Proceed with deployment 138 | 139 | - name: Get Prior Infrastructure Slug 140 | id: get-infra-slug-comment 141 | uses: peter-evans/find-comment@v2 142 | with: 143 | issue-number: ${{ github.event.issue.number }} 144 | comment-author: 'github-actions[bot]' 145 | body-includes: 'Generated Infrastructure Slug' 146 | 147 | - name: Parse Prior Infrastructure Slug 148 | id: infrastructure-slug 149 | run: | 150 | slug=$(python -c 'content = """${{ steps.get-infra-slug-comment.outputs.comment-body }}"""; print(content[content.index("`") + 1:content.index("`", content.index("`") + 1)]);') 151 | echo "::set-output name=slug::$slug" 152 | 153 | - name: Update Infrastructure Slug in Cookiecutter Options 154 | run: | 155 | replacement=$(jq '.infrastructure_slug = "${{ steps.infrastructure-slug.outputs.slug }}"' planned-cookiecutter.json) 156 | echo "$replacement" > planned-cookiecutter.json 157 | cat planned-cookiecutter.json 158 | 159 | - name: Run Cookiecutter 160 | run: | 161 | mv planned-cookiecutter.json cookiecutter.json 162 | cookiecutter . --no-input 163 | 164 | - name: Get Municipality Slugs 165 | id: municipality-slug 166 | run: | 167 | slug=$(jq -r '.municipality_slug' cookiecutter.json) 168 | python_slug=$(jq -r '.python_municipality_slug' cookiecutter.json) 169 | echo "::set-output name=slug::$slug" 170 | echo "::set-output name=python_slug::$python_slug" 171 | 172 | - name: Init Git 173 | run: | 174 | cd ${{ steps.municipality-slug.outputs.slug }} 175 | git config --global user.email "councildataproject@gmail.com" 176 | git config --global user.name "CouncilDataProjectServiceAccount" 177 | git init 178 | git checkout -b main 179 | git add -A 180 | git commit -m "Initial commit" 181 | 182 | # Replace scraper with provided 183 | - name: Add cdp-scrapers as dependency 184 | run: sed -i '10 i \ "cdp-scrapers[${{ steps.municipality-slug.outputs.python_slug }}]",' ${{ steps.municipality-slug.outputs.slug }}/python/setup.py 185 | 186 | # Get scraper path and update scraper function 187 | # The sed after param2 extract is to replace `/` with `\/` to escape the char 188 | - name: Get Scraper Path 189 | id: selected-scraper 190 | run: | 191 | scraper_options=$(jq -r '.scraper_options' generation-options.json) 192 | scraper_choice=$(echo $scraper_options | cut -f1 -d %) 193 | param1=$(echo $scraper_options | cut -f2 -d %) 194 | param2=$(echo $scraper_options | cut -f3 -d %) 195 | param2=$(sed 's/\//\\\//g' <<< $param2) 196 | echo "::set-output name=choice::$scraper_choice" 197 | echo "::set-output name=param1::$param1" 198 | echo "::set-output name=param2::$param2" 199 | 200 | # Use base legistar for scraper 201 | - name: Add Base Legistar Scraper 202 | if: | 203 | contains(steps.selected-scraper.outputs.choice, 'USE_BASE_LEGISTAR') 204 | run: | 205 | sed -i \ 206 | 's/REPLACE_LEGISTAR_CLIENT/${{ steps.selected-scraper.outputs.param1 }}/g' \ 207 | .github/workflows/resources/base_legistar_scraper.py 208 | sed -i \ 209 | 's/REPLACE_IANA_CLIENT_TZ/${{ steps.selected-scraper.outputs.param2 }}/g' \ 210 | .github/workflows/resources/base_legistar_scraper.py 211 | mv \ 212 | .github/workflows/resources/base_legistar_scraper.py \ 213 | ${{ steps.municipality-slug.outputs.slug }}/python/cdp_${{ steps.municipality-slug.outputs.python_slug }}_backend/scraper.py 214 | 215 | # Use custom scraper 216 | - name: Add Custom Scraper 217 | if: | 218 | contains(steps.selected-scraper.outputs.choice, 'USE_FOUND_SCRAPER') 219 | run: | 220 | sed -i \ 221 | 's/REPLACE_CUSTOM_SCRAPER/${{ steps.selected-scraper.outputs.param1 }}/g' \ 222 | .github/workflows/resources/custom_scraper.py 223 | mv \ 224 | .github/workflows/resources/custom_scraper.py \ 225 | ${{ steps.municipality-slug.outputs.slug }}/python/cdp_${{ steps.municipality-slug.outputs.python_slug }}_backend/scraper.py 226 | 227 | # Commit scraper changes 228 | - name: Commit Scraper Dep and Usage 229 | run: | 230 | cd ${{ steps.municipality-slug.outputs.slug }} 231 | git add -A 232 | git commit -m "Update scraper dependency and implementation" 233 | 234 | - name: Setup Git SSH 235 | run: | 236 | eval "$(ssh-agent -s)" 237 | mkdir ~/.ssh/ 238 | echo "${{ secrets.SA_SSH_KEY }}" > ~/.ssh/id_cdp_sa 239 | chmod 600 ~/.ssh/id_cdp_sa 240 | ssh-add ~/.ssh/id_cdp_sa 241 | echo "Host github.com 242 | HostName github.com 243 | IdentityFile ~/.ssh/id_cdp_sa" > ~/.ssh/config 244 | 245 | - name: Create New Instance Repo and Push 246 | run: | 247 | cd ${{ fromJSON(steps.cookiecutter-options.outputs.options ).hosting_github_repo_name }} 248 | gh repo create \ 249 | ${{ fromJSON(steps.generation-options.outputs.content).repository_path }} \ 250 | --public \ 251 | --description "CDP Instance for ${{ fromJSON(steps.cookiecutter-options.outputs.options).municipality }}" \ 252 | --homepage "${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_web_app_address }}" \ 253 | --disable-wiki 254 | sleep 10s 255 | git remote add origin git@github.com:${{ fromJSON(steps.generation-options.outputs.content).repository_path }}.git 256 | sleep 10s 257 | git push -u origin main 258 | env: 259 | GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} 260 | 261 | - name: Generate Safe Maintainer Name Check 262 | id: safe-maintainer 263 | run: | 264 | username=$(jq -r '.maintainer_name' generation-options.json) 265 | username="---$username---" 266 | echo "::set-output name=username::$username" 267 | 268 | # Add external collaborator to the repo if the maintainer isn't in CDP org 269 | - name: Add External Collaborator 270 | if: | 271 | !contains( 272 | steps.cdp-members.outputs.members, 273 | steps.safe-maintainer.outputs.username 274 | ) 275 | run: | 276 | gh api \ 277 | repos/CouncilDataProject/${{ steps.municipality-slug.outputs.slug }}/collaborators/$(jq -r '.maintainer_name' generation-options.json) \ 278 | -X PUT \ 279 | -f permission='maintain' 280 | env: 281 | GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }} 282 | 283 | - name: Log Success 284 | uses: peter-evans/create-or-update-comment@v3 285 | with: 286 | issue-number: ${{ github.event.issue.number }} 287 | body: | 288 | ## Deployment Status 289 | 290 | :tada: :tada: **Repository Created** :tada: :tada: 291 | 292 | A new CouncilDataProject Instance Repository was created ([${{ fromJSON(steps.generation-options.outputs.content).repository_path }}](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }})), external collaborator added (@${{ fromJSON(steps.generation-options.outputs.content).maintainer_name }}), and cookiecutter files generated and pushed to repository. 293 | 294 | The instance is setting itself up right now and the process will take around 10 minutes to complete. Once completed, a CDP maintainer will comment on this issue with your instance's website link. See [the instance's GitHub Action job history](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/actions) for more details on the deployment setup progress. 295 | 296 | Your CDP instance will be populated with data within 6 hours of website creation. 297 | 298 | At any point in the future if you would like to destroy this instance, please just add a comment to this thread and a maintainer will help you. 299 | 300 | --- 301 | 302 | #### Steps for Internal CDP Team 303 | 304 | ##### Final Setup 305 | 306 | * [ ] Copy the key generated from the prior `just init` process 307 | * [ ] Use the generated key as the [repository secret](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/secrets/actions) for `GOOGLE_CREDENTIALS` 308 | * [ ] Rerun the [failed jobs](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/actions), then: 309 | * [ ] Enable [GitHub Pages](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings/pages) 310 | * [ ] Comment on this issue with "Deployment Status - Complete" and the instance URL 311 | 312 | ##### Deletion Steps (Future Reference) 313 | 314 | * [ ] Delete the [instance repository](${{ fromJSON(steps.cookiecutter-options.outputs.options).hosting_github_url }}/settings) 315 | * [ ] Run `just login` and login to the CDP gcloud 316 | * [ ] Run `just destroy project=${{ steps.infrastructure-slug.outputs.slug }}` 317 | 318 | More details on the `just` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure). -------------------------------------------------------------------------------- /.github/workflows/instance-configuration-validation-bot.yml: -------------------------------------------------------------------------------- 1 | name: Instance Configuration Validation Bot 2 | 3 | on: 4 | issues: 5 | types: 6 | - opened 7 | - edited 8 | 9 | jobs: 10 | validate-form: 11 | if: ${{ contains(github.event.issue.labels.*.name, 'new instance') }} 12 | runs-on: ubuntu-latest 13 | permissions: 14 | issues: write 15 | 16 | steps: 17 | ######################################################################### 18 | # Initial Hello and Documentation 19 | 20 | - name: Find Validation Results Comment - Pre Run 21 | uses: peter-evans/find-comment@v2 22 | id: find-validation-results-comment-pre 23 | with: 24 | issue-number: ${{ github.event.issue.number }} 25 | comment-author: 'github-actions[bot]' 26 | body-includes: 'Form Validation Results' 27 | 28 | - name: Set Initial Comment State 29 | uses: peter-evans/create-or-update-comment@v3 30 | if: ${{ steps.find-validation-results-comment-pre.outputs.comment-id == 0 }} 31 | with: 32 | issue-number: ${{ github.event.issue.number }} 33 | body: | 34 | Hello! 👋 35 | Thanks for initiating the process to configure a new CDP instance. 36 | 37 | I am a bot that will validate the information provided in your form. If any check fails, please update your issue by opening the '⋯' dropdown in the top-right-corner of your GitHub Issue and selecting 'Edit'. I will automatically rerun the checks after you update the issue to validate the changes. 38 | 39 | A member from the CDP team will respond as soon as possible! 40 | 41 | #### Form Validation Results 42 | 43 | :hourglass_flowing_sand: Validating planned instance maintainer 44 | :hourglass_flowing_sand: Validating planned instance repository name 45 | :hourglass_flowing_sand: Determining event scraper strategy and optionally testing Legistar 46 | 47 | #### All Cookiecutter Parameters 48 | 49 | :hourglass_flowing_sand: Generating... 50 | 51 | _This comment was written by a bot!_ 52 | 53 | - name: Reset Initial Comment State 54 | uses: peter-evans/create-or-update-comment@v3 55 | if: ${{ steps.find-validation-results-comment-pre.outputs.comment-id != 0 }} 56 | with: 57 | comment-id: ${{ steps.find-validation-results-comment-pre.outputs.comment-id }} 58 | edit-mode: 'replace' 59 | body: | 60 | Hello! 👋 61 | Thanks for initiating the process to configure a new CDP instance. 62 | 63 | I am a bot that will validate the information provided in your form. If any check fails, please update your issue by opening the '⋯' dropdown in the top-right-corner of your GitHub Issue and selecting 'Edit'. I will automatically rerun the checks after you update the issue to validate the changes. 64 | 65 | A member from the CDP team will respond as soon as possible! 66 | 67 | #### Form Validation Results 68 | 69 | :hourglass_flowing_sand: Validating planned instance maintainer 70 | :hourglass_flowing_sand: Validating planned instance repository name 71 | :hourglass_flowing_sand: Determining event scraper strategy and optionally testing Legistar 72 | 73 | #### All Cookiecutter Parameters 74 | 75 | :hourglass_flowing_sand: Generating... 76 | 77 | _This comment was written by a bot!_ 78 | 79 | ######################################################################### 80 | # Workflow Setup 81 | 82 | - uses: actions/checkout@v4 83 | - name: Set up Python 84 | uses: actions/setup-python@v5 85 | with: 86 | python-version: '3.11' 87 | - name: Install Bot Scripts Dependencies 88 | run: | 89 | pip install --upgrade pip 90 | pip install -r .github/workflows/scripts/requirements.txt 91 | 92 | ######################################################################### 93 | # Parsing Form and Logging Details 94 | 95 | - name: Dump Issue Body to File 96 | run: | 97 | echo "${{ github.event.issue.body }}" > issue-body.md 98 | - name: Validate Form and Create Bot Response 99 | run: | 100 | python .github/workflows/scripts/validate_form.py issue-body.md 101 | - name: Set Response Content 102 | id: validation-message-response 103 | run: | 104 | body=$(cat form-validation-results.md) 105 | body="${body//'%'/'%25'}" 106 | body="${body//$'\n'/'%0A'}" 107 | body="${body//$'\r'/'%0D'}" 108 | echo ::set-output name=body::"$body" 109 | 110 | - name: Dump Cookiecutter Parameters 111 | id: dump-cookiecutter-parameters 112 | run: | 113 | set -f 114 | body=$(cat planned-cookiecutter.json) 115 | body="${body//'%'/'%25'}" 116 | body="${body//$'\n'/'%0A'}" 117 | body="${body//$'\r'/'%0D'}" 118 | echo ::set-output name=body::$body 119 | 120 | - name: Get Infrastructure Metadata 121 | id: infra-meta 122 | run: | 123 | slug=$(jq -r '.municipality_slug' planned-cookiecutter.json) 124 | infra_slug=$(jq -r '.infrastructure_slug' planned-cookiecutter.json) 125 | region=$(jq -r '.firestore_region' planned-cookiecutter.json) 126 | echo "::set-output name=slug::$slug" 127 | echo "::set-output name=infra_slug::$infra_slug" 128 | echo "::set-output name=region::$region" 129 | 130 | - name: Find Validation Results Comment - Post Run 131 | uses: peter-evans/find-comment@v2 132 | id: find-validation-results-comment-post 133 | with: 134 | issue-number: ${{ github.event.issue.number }} 135 | comment-author: 'github-actions[bot]' 136 | body-includes: 'Form Validation Results' 137 | 138 | - name: Post Validation Results 139 | uses: peter-evans/create-or-update-comment@v3 140 | with: 141 | comment-id: ${{ steps.find-validation-results-comment-post.outputs.comment-id }} 142 | edit-mode: 'replace' 143 | body: | 144 | Hello! 👋 145 | Thanks for initiating the process to configure a new CDP instance. 146 | 147 | I am a bot that will validate the information provided in your form. If any check fails, please update your issue by opening the '⋯' dropdown in the top-right-corner of your GitHub Issue and selecting 'Edit'. I will automatically rerun the checks after you update the issue to validate the changes. 148 | 149 | A member from the CDP team will respond as soon as possible! 150 | 151 | #### Form Validation Results 152 | 153 | ${{ steps.validation-message-response.outputs.body }} 154 | 155 | #### All Cookiecutter Parameters 156 | 157 | ```json 158 | ${{ steps.dump-cookiecutter-parameters.outputs.body }} 159 | ``` 160 | 161 | --- 162 | 163 | #### Steps for Internal CDP Team 164 | 165 | To proceed with the deployment process, please do the following: 166 | 167 | * [ ] Run `get_cdp_infrastructure_stack dev-infrastructure/` 168 | * [ ] Run `just login` in cdp-backend/dev-infrastructure and login to the CDP gcloud account 169 | * [ ] Run `just init ${{ steps.infra-meta.outputs.infra_slug }}` in cdp-backend/dev-infrastructure 170 | * [ ] Run `just setup ${{ steps.infra-meta.outputs.infra_slug }} ${{ steps.infra-meta.outputs.region }}` in cdp-backend/dev-infrastructure 171 | * [ ] Setup Firebase Storage [Link](https://console.firebase.google.com/u/0/project/${{ steps.infra-meta.outputs.infra_slug }}/storage) 172 | * [ ] Comment "/cdp-deploy" on this issue and follow the rest of the instructions 173 | 174 | More details on the `just` commands can be found in [cdp-backend](https://github.com/CouncilDataProject/cdp-backend/tree/main/dev-infrastructure). 175 | 176 | _This comment was written by a bot!_ 177 | 178 | - name: Find Infrastructure Slug Comment 179 | uses: peter-evans/find-comment@v2 180 | id: find-infra-slug-comment 181 | with: 182 | issue-number: ${{ github.event.issue.number }} 183 | comment-author: 'github-actions[bot]' 184 | body-includes: 'Generated Infrastructure Slug' 185 | 186 | - name: Set Initial Infra Slug Comment State 187 | uses: peter-evans/create-or-update-comment@v3 188 | if: ${{ steps.find-infra-slug-comment.outputs.comment-id == 0 }} 189 | with: 190 | issue-number: ${{ github.event.issue.number }} 191 | body: | 192 | ##### Generated Infrastructure Slug 193 | 194 | `${{ steps.infra-meta.outputs.infra_slug }}` 195 | 196 | _This comment was written by a bot!_ 197 | 198 | - name: Reset Infra Slug Comment State 199 | uses: peter-evans/create-or-update-comment@v3 200 | if: ${{ steps.find-infra-slug-comment.outputs.comment-id != 0 }} 201 | with: 202 | comment-id: ${{ steps.find-infra-slug-comment.outputs.comment-id }} 203 | edit-mode: 'replace' 204 | body: | 205 | ##### Generated Infrastructure Slug 206 | 207 | `${{ steps.infra-meta.outputs.infra_slug }}` 208 | 209 | _This comment was written by a bot!_ -------------------------------------------------------------------------------- /.github/workflows/paper.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | 6 | jobs: 7 | paper: 8 | runs-on: ubuntu-latest 9 | name: Paper Draft 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v4 13 | - name: Build Draft PDF 14 | uses: openjournals/openjournals-draft-action@master 15 | with: 16 | journal: joss 17 | paper-path: paper/paper.md 18 | - name: Upload 19 | uses: actions/upload-artifact@v3 20 | with: 21 | name: paper 22 | path: paper/paper.pdf 23 | -------------------------------------------------------------------------------- /.github/workflows/resources/base_legistar_scraper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import datetime 5 | from typing import List 6 | 7 | from cdp_backend.pipeline.ingestion_models import EventIngestionModel 8 | from cdp_scrapers.legistar_utils import LegistarScraper 9 | 10 | ############################################################################### 11 | 12 | 13 | def get_events( 14 | from_dt: datetime, 15 | to_dt: datetime, 16 | **kwargs, 17 | ) -> List[EventIngestionModel]: 18 | scraper = LegistarScraper( 19 | client="REPLACE_LEGISTAR_CLIENT", 20 | timezone="REPLACE_IANA_CLIENT_TZ", 21 | ) 22 | 23 | return scraper.get_events(begin=from_dt, end=to_dt) 24 | -------------------------------------------------------------------------------- /.github/workflows/resources/custom_scraper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import datetime 5 | from typing import List 6 | 7 | from cdp_backend.pipeline.ingestion_models import EventIngestionModel 8 | from cdp_scrapers.instances import REPLACE_CUSTOM_SCRAPER 9 | 10 | ############################################################################### 11 | 12 | 13 | def get_events( 14 | from_dt: datetime, 15 | to_dt: datetime, 16 | **kwargs, 17 | ) -> List[EventIngestionModel]: 18 | return REPLACE_CUSTOM_SCRAPER(from_dt=from_dt, to_dt=to_dt, **kwargs) 19 | -------------------------------------------------------------------------------- /.github/workflows/scripts/parse_form.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import json 6 | import logging 7 | import sys 8 | import traceback 9 | from typing import Dict, List, Optional 10 | from uuid import uuid4 11 | from random import randint 12 | 13 | from cdp_backend.utils.string_utils import clean_text 14 | 15 | ############################################################################### 16 | 17 | logging.basicConfig( 18 | level=logging.INFO, 19 | format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", 20 | ) 21 | log = logging.getLogger(__name__) 22 | 23 | 24 | ############################################################################### 25 | 26 | COUNCIL_DATA_PROJECT = "CouncilDataProject" 27 | 28 | FORM_VALUES = "form_values" 29 | COOKIECUTTER_OPTIONS = "cookiecutter_options" 30 | 31 | MUNICIPALITY_NAME = "municipality" 32 | GOVERNING_BODY_TYPE = "governing_body_type" 33 | MUNICIPALITY_SLUG = "municipality_slug" 34 | PYTHON_MUNICIPALITY_SLUG = "python_municipality_slug" 35 | TARGET_MAINTAINER = "maintainer_or_org_full_name" 36 | FIRESTORE_REGION = "firestore_region" 37 | 38 | LEGISTAR_CLIENT_ID = "legistar_client_id" 39 | IANA_CLIENT_TIMEZONE = "iana_timezone" 40 | EVENT_GATHER_TIMEDELTA = "event_gather_timedelta_lookback_days" 41 | EVENT_GATHER_CRON = "event_gather_cron" 42 | 43 | FORM_FIELD_TO_HEADER = { 44 | MUNICIPALITY_NAME: "Municipality Name", 45 | GOVERNING_BODY_TYPE: "Governing Body Type", 46 | MUNICIPALITY_SLUG: "Municipality Slug", 47 | TARGET_MAINTAINER: "Maintainer GitHub Name", 48 | FIRESTORE_REGION: "Firestore Region", 49 | LEGISTAR_CLIENT_ID: "Legistar Client Id", 50 | IANA_CLIENT_TIMEZONE: "Municipality Timezone", 51 | EVENT_GATHER_TIMEDELTA: "Event Gather Timedelta Lookback Days", 52 | EVENT_GATHER_CRON: "Event Gather CRON", 53 | } 54 | 55 | NO_RESPONSE = "_No response_" 56 | 57 | DEFAULT_FIRESTORE_REGION = "us-central" 58 | DEFAULT_EVENT_GATHER_TIMEDELTA = 2 59 | 60 | RANDOM_MINUTE = randint(0, 59) # inclusive 61 | RANDOM_HOUR = randint(0, 11) # inclusive 62 | OFFSET_HOUR = RANDOM_HOUR + 12 63 | DEFAULT_EVENT_GATHER_CRON = f"{RANDOM_MINUTE} {RANDOM_HOUR},{OFFSET_HOUR} * * *" 64 | 65 | ############################################################################### 66 | 67 | 68 | class Args(argparse.Namespace): 69 | def __init__(self) -> None: 70 | self.__parse() 71 | 72 | def __parse(self) -> None: 73 | p = argparse.ArgumentParser( 74 | prog="parse-form", 75 | description="Parse the form values and generate cookiecutter options.", 76 | ) 77 | p.add_argument( 78 | "issue_content_file", 79 | type=str, 80 | help="The path to the issue / form content file.", 81 | ) 82 | p.parse_args(namespace=self) 83 | 84 | 85 | def _get_field_value(lines: List[str], field_header: str) -> Optional[str]: 86 | # Get index of target then + 1 for the value 87 | header_index = lines.index(f"### {field_header}") 88 | value = lines[header_index + 1] 89 | return value if value != NO_RESPONSE else None 90 | 91 | 92 | def parse_form(issue_content_file: str) -> Dict[str, Dict[str, str]]: 93 | # Open the content file, read, strip, and clean 94 | with open(issue_content_file, "r") as open_f: 95 | lines = open_f.readlines() 96 | lines = [line.strip() for line in lines] 97 | lines = [line for line in lines if len(line) > 0] 98 | 99 | # Get all form values 100 | form_values: Dict[str, Optional[str]] = {} 101 | for field_name, form_header_string in FORM_FIELD_TO_HEADER.items(): 102 | form_values[field_name] = _get_field_value( 103 | lines=lines, 104 | field_header=form_header_string, 105 | ) 106 | log.info(form_values) 107 | 108 | # Get municipality slug 109 | if form_values[MUNICIPALITY_SLUG] is None: 110 | municipality_slug = ( 111 | clean_text(form_values[MUNICIPALITY_NAME]) 112 | .lower() 113 | .replace( 114 | " ", 115 | "-", 116 | ) 117 | ) 118 | else: 119 | municipality_slug = form_values[MUNICIPALITY_SLUG] 120 | 121 | # Get python municipality slug 122 | python_municipality_slug = municipality_slug.replace("-", "_") 123 | 124 | # Get default firestore region 125 | if form_values[FIRESTORE_REGION] is None: 126 | firestore_region = DEFAULT_FIRESTORE_REGION 127 | else: 128 | firestore_region = form_values[FIRESTORE_REGION] 129 | 130 | # Get event gather timedelta 131 | if form_values[EVENT_GATHER_TIMEDELTA] is None: 132 | event_gather_timedelta = DEFAULT_EVENT_GATHER_TIMEDELTA 133 | else: 134 | event_gather_timedelta = int(form_values[EVENT_GATHER_TIMEDELTA]) 135 | 136 | # Get event gather cron 137 | if form_values[EVENT_GATHER_CRON] is None: 138 | event_gather_cron = DEFAULT_EVENT_GATHER_CRON 139 | else: 140 | event_gather_cron = form_values[EVENT_GATHER_CRON] 141 | 142 | all_options = { 143 | FORM_VALUES: form_values, 144 | COOKIECUTTER_OPTIONS: { 145 | MUNICIPALITY_NAME: form_values[MUNICIPALITY_NAME], 146 | IANA_CLIENT_TIMEZONE: form_values[IANA_CLIENT_TIMEZONE], 147 | GOVERNING_BODY_TYPE: form_values[GOVERNING_BODY_TYPE], 148 | MUNICIPALITY_SLUG: municipality_slug, 149 | PYTHON_MUNICIPALITY_SLUG: python_municipality_slug, 150 | "infrastructure_slug": f"cdp-{municipality_slug}-{str(uuid4())[:8]}", 151 | TARGET_MAINTAINER: form_values[TARGET_MAINTAINER], 152 | "hosting_github_username_or_org": COUNCIL_DATA_PROJECT, 153 | "hosting_github_repo_name": municipality_slug, 154 | "hosting_github_url": ( 155 | f"https://github.com/{COUNCIL_DATA_PROJECT}/{municipality_slug}" 156 | ), 157 | "hosting_web_app_address": ( 158 | f"https://councildataproject.github.io/{municipality_slug}" 159 | ), 160 | FIRESTORE_REGION: firestore_region, 161 | EVENT_GATHER_TIMEDELTA: event_gather_timedelta, 162 | EVENT_GATHER_CRON: event_gather_cron, 163 | }, 164 | } 165 | 166 | # Dump to cookiecutter.json 167 | with open("planned-cookiecutter.json", "w") as open_f: 168 | open_f.write(json.dumps(all_options[COOKIECUTTER_OPTIONS], indent=4)) 169 | 170 | return all_options 171 | 172 | 173 | def main() -> None: 174 | try: 175 | args = Args() 176 | parse_form( 177 | issue_content_file=args.issue_content_file, 178 | ) 179 | except Exception as e: 180 | log.error("=============================================") 181 | log.error("\n\n" + traceback.format_exc()) 182 | log.error("=============================================") 183 | log.error("\n\n" + str(e) + "\n") 184 | log.error("=============================================") 185 | sys.exit(1) 186 | 187 | 188 | ############################################################################### 189 | # Allow caller to directly run this module (usually in development scenarios) 190 | 191 | if __name__ == "__main__": 192 | main() 193 | -------------------------------------------------------------------------------- /.github/workflows/scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | cdp-backend>=4.0.8 2 | cdp-scrapers[test]~=0.4 3 | cookiecutter~=1.7 4 | requests~=2.25 -------------------------------------------------------------------------------- /.github/workflows/scripts/validate_form.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | from datetime import datetime, timedelta 6 | import json 7 | import logging 8 | import sys 9 | import traceback 10 | 11 | from cdp_backend.infrastructure import GoverningBody 12 | from cdp_scrapers.legistar_utils import LegistarScraper 13 | from cdp_scrapers import instances 14 | import requests 15 | 16 | from parse_form import ( 17 | COOKIECUTTER_OPTIONS, 18 | FORM_VALUES, 19 | MUNICIPALITY_SLUG, 20 | GOVERNING_BODY_TYPE, 21 | MUNICIPALITY_NAME, 22 | LEGISTAR_CLIENT_ID, 23 | IANA_CLIENT_TIMEZONE, 24 | PYTHON_MUNICIPALITY_SLUG, 25 | TARGET_MAINTAINER, 26 | COUNCIL_DATA_PROJECT, 27 | parse_form, 28 | ) 29 | 30 | ############################################################################### 31 | 32 | logging.basicConfig( 33 | level=logging.INFO, 34 | format="[%(levelname)4s: %(module)s:%(lineno)4s %(asctime)s] %(message)s", 35 | ) 36 | log = logging.getLogger(__name__) 37 | 38 | ############################################################################### 39 | 40 | GITHUB_USERS_RESOURCE = "users" 41 | GITHUB_REPOSITORIES_RESOURCE = "repos" 42 | 43 | ############################################################################### 44 | 45 | 46 | class Args(argparse.Namespace): 47 | def __init__(self) -> None: 48 | self.__parse() 49 | 50 | def __parse(self) -> None: 51 | p = argparse.ArgumentParser( 52 | prog="validate-form", 53 | description=( 54 | "Validate the values provided in the CDP instance configuration form." 55 | ), 56 | ) 57 | p.add_argument( 58 | "issue_content_file", 59 | type=str, 60 | help="The path to the issue / form content file.", 61 | ) 62 | p.parse_args(namespace=self) 63 | 64 | 65 | def _check_github_resource_exists(resource: str, name: str) -> bool: 66 | # Request and get response 67 | response = requests.get(f"https://api.github.com/{resource}/{name}") 68 | content = response.json() 69 | 70 | # Return the boolean if the structure is consistent with successful query 71 | # or not name is only present if the resource exists 72 | # Otherwise the response looks like 73 | # {'message': 'Not Found', 'documentation_url': '...'} 74 | return "name" in content 75 | 76 | 77 | def validate_form(issue_content_file: str) -> None: 78 | # Parse and get cookiecutter options 79 | form_values_and_cookiecutter_options = parse_form( 80 | issue_content_file=issue_content_file, 81 | ) 82 | 83 | # Unpack form values 84 | form_values = form_values_and_cookiecutter_options[FORM_VALUES] 85 | 86 | # Unpack certain values 87 | municipality_slug = form_values_and_cookiecutter_options[COOKIECUTTER_OPTIONS][ 88 | MUNICIPALITY_SLUG 89 | ] 90 | python_municipality_slug = form_values_and_cookiecutter_options[ 91 | COOKIECUTTER_OPTIONS 92 | ][PYTHON_MUNICIPALITY_SLUG] 93 | 94 | # Governing body type in allowed 95 | governing_body_allowed_strings = [ 96 | getattr(GoverningBody, attr) for attr in dir(GoverningBody) if "__" not in attr 97 | ] 98 | governing_body_type_allowed = ( 99 | form_values[GOVERNING_BODY_TYPE] in governing_body_allowed_strings 100 | ) 101 | 102 | # Check planned maintainer exists 103 | planned_maintainer_exists = _check_github_resource_exists( 104 | resource=GITHUB_USERS_RESOURCE, 105 | name=form_values[TARGET_MAINTAINER], 106 | ) 107 | 108 | # Get municipality name 109 | repository_path = f"{COUNCIL_DATA_PROJECT}/{municipality_slug}" 110 | planned_repository_exists = _check_github_resource_exists( 111 | resource=GITHUB_REPOSITORIES_RESOURCE, name=repository_path 112 | ) 113 | 114 | # Test Legistar / existing scraper 115 | scraper_response = None 116 | scraper_options = None 117 | try: 118 | func_name = f"get_{python_municipality_slug}_events" 119 | getattr(instances, func_name) 120 | 121 | scraper_response = ( 122 | f"✅ An existing scraper for '{form_values[MUNICIPALITY_NAME]}' was found " 123 | f"in `cdp-scrapers` (`cdp_scrapers.instances.{func_name}`). " 124 | f"If this scraper was selected incorrectly, please update the " 125 | f"Municipality Slug field with more specificity " 126 | f"(i.e. 'seattle-wa' instead of 'seattle')." 127 | ) 128 | scraper_ready = True 129 | scraper_options = f"USE_FOUND_SCRAPER%{func_name}" 130 | except AttributeError: 131 | if ( 132 | form_values[LEGISTAR_CLIENT_ID] is not None 133 | and form_values[IANA_CLIENT_TIMEZONE] is not None 134 | ): 135 | log.info("Attempting Legistar data retrieval") 136 | 137 | # Init temp scraper and run 138 | scraper = LegistarScraper( 139 | client=form_values[LEGISTAR_CLIENT_ID], 140 | timezone=form_values[IANA_CLIENT_TIMEZONE], 141 | ) 142 | try: 143 | # Check that the provided client information 144 | # is even a Legistar municipality 145 | if not scraper.is_legistar_compatible: 146 | scraper_response = ( 147 | f"❌ No public Legistar instance found for " 148 | f"the provided client ({form_values[LEGISTAR_CLIENT_ID]}). " 149 | f"If your municipality uses Legistar but you received this " 150 | f"error, we recommended contacting your municipality clerk and " 151 | f"asking about public Legistar API access, " 152 | f"they may direct you to the IT department as well. " 153 | f"If they do not respond to your requests, you will need to " 154 | f"write a custom scraper to deploy your CDP instance." 155 | ) 156 | scraper_ready = False 157 | log.info("Legistar client available") 158 | 159 | # If everything runs correctly, log success and show event 160 | # model in comment 161 | for days_prior in [3, 7, 14, 28]: 162 | log.info(f"Attempting minimum CDP data for {days_prior} days") 163 | if scraper.check_for_cdp_min_ingestion(check_days=days_prior): 164 | log.info("Legistar client has minimum data") 165 | events = scraper.get_events( 166 | begin=datetime.utcnow() - timedelta(days=days_prior), 167 | ) 168 | single_event = events[0] 169 | event_as_json_str = single_event.to_json(indent=4) 170 | 171 | scraper_response = ( 172 | f"✅ The municipality's Legistar instance " 173 | f"contains the minimum required CDP event ingestion data.\n" 174 | f"Retrieved Data\n" 175 | f"
\n\n" # Extra new line for proper rendering 176 | f"```json\n" 177 | f"{event_as_json_str}\n" 178 | f"```\n" 179 | f"
" 180 | ) 181 | scraper_ready = True 182 | scraper_options = ( 183 | f"USE_BASE_LEGISTAR" 184 | f"%{form_values[LEGISTAR_CLIENT_ID]}" 185 | f"%{form_values[IANA_CLIENT_TIMEZONE]}" 186 | ) 187 | break 188 | 189 | if scraper_response is None: 190 | log.info( 191 | "Legistar client missing minimum data, " 192 | "attempting to simply pull data for logging." 193 | ) 194 | # Check if _any_ data was returned 195 | for days_prior in [3, 7, 14, 28]: 196 | log.info( 197 | f"Attempting to pull Legistar data for " 198 | f"previous {days_prior} days." 199 | ) 200 | events = scraper.get_events( 201 | begin=datetime.utcnow() - timedelta(days=days_prior), 202 | ) 203 | if len(events) > 0: 204 | log.info( 205 | f"Received Legistar data for " 206 | f"previous {days_prior} days." 207 | ) 208 | single_event = events[0] 209 | event_as_json_str = single_event.to_json(indent=4) 210 | scraper_response = ( 211 | f"❌ Your municipality uses Legistar but the minimum " 212 | f"required data for CDP event ingestion wasn't found. " 213 | f"A " 214 | f"[cdp-scrapers]" 215 | f"(https://github.com/" 216 | f"{COUNCIL_DATA_PROJECT}/cdp-scrapers) " 217 | f"maintainer will look into this issue however it is " 218 | f"likely that you (@{form_values[TARGET_MAINTAINER]}) " 219 | f"will need to write a custom scraper.\n" 220 | f"Retrieved Data\n" 221 | f"
\n\n" # Extra new line for proper rendering 222 | f"```json\n" 223 | f"{event_as_json_str}\n" 224 | f"```\n" 225 | f"
" 226 | ) 227 | scraper_ready = False 228 | break 229 | 230 | # Catch no video path available 231 | # User will need to write a custom Legistar scraper 232 | except NotImplementedError: 233 | scraper_response = ( 234 | f":warning: Your municipality uses Legistar but is " 235 | f"missing the video URLs for event recordings. " 236 | f"We recommended writing a custom Legistar Scraper that inherits " 237 | f"from our own [LegistarScraper]" 238 | f"(https://councildataproject.org/cdp-scrapers/" 239 | f"cdp_scrapers.html#cdp_scrapers.legistar_utils.LegistarScraper) " 240 | f"to resolve the issue. " 241 | f"Please see the " 242 | f"[cdp-scrapers]" 243 | f"(https://github.com/" 244 | f"{COUNCIL_DATA_PROJECT}/cdp-scrapers) repository " 245 | f"for more details. And please refer to the " 246 | f"[SeattleScraper]" 247 | f"(https://github.com/{COUNCIL_DATA_PROJECT}/cdp-scrapers" 248 | f"/blob/main/cdp_scrapers/instances/seattle.py) " 249 | f"for an example of a scraper that inherits from our " 250 | f"base `LegistarScraper` to resolve this issue." 251 | ) 252 | scraper_ready = False 253 | 254 | except Exception as e: 255 | scraper_response = ( 256 | f"❌ Something went wrong during Legistar client data validation. " 257 | f"A [cdp-scrapers]" 258 | f"(https://github.com/" 259 | f"{COUNCIL_DATA_PROJECT}/cdp-scrapers) maintainer " 260 | f"will look into the logs for this bug. Sorry about this!" 261 | ) 262 | log.error(e) 263 | log.error(traceback.format_exc()) 264 | scraper_ready = False 265 | 266 | # Handle bad / mis-parametrized legistar info 267 | if scraper_response is None: 268 | scraper_response = ( 269 | f"❌ **You didn't provide Legistar Client " 270 | f"information and no existing scraper was found in `cdp-scrapers`**. " 271 | f"Please either provide Legistar Client information and / or add a " 272 | f"custom scraper to " 273 | f"[cdp-scrapers]" 274 | f"(https://github.com/" 275 | f"{COUNCIL_DATA_PROJECT}/cdp-scrapers). " 276 | f"Please refer to our " 277 | f"[documentation for writing custom scrapers](TODO) " 278 | f"for more information. " 279 | f"Note, either a successful basic Legistar scraper run or the " 280 | f"addition of a custom scraper to `cdp-scrapers` is required before " 281 | f"moving on in the deployment process." 282 | ) 283 | scraper_ready = False 284 | 285 | # Construct message content 286 | if governing_body_type_allowed: 287 | governing_body_response = "✅ Governing body type is an accepted value." 288 | else: 289 | governing_body_response = ( 290 | f"❌ The provided governing body type is not an allowed value " 291 | f"({form_values[GOVERNING_BODY_TYPE]}). " 292 | f"Allowed values are: {governing_body_allowed_strings}." 293 | ) 294 | maintainer_name = None 295 | if planned_maintainer_exists: 296 | maintainer_name = form_values[TARGET_MAINTAINER] 297 | maintainer_response = ( 298 | f"✅ @{maintainer_name} " f"has been marked as the instance maintainer." 299 | ) 300 | maintainer_ready = True 301 | else: 302 | maintainer_response = ( 303 | f"❌ The planned instance maintainer: " 304 | f"'{form_values[TARGET_MAINTAINER]}', does not exist." 305 | ) 306 | maintainer_ready = False 307 | 308 | if planned_repository_exists: 309 | repository_response = ( 310 | f"❌ The planned repository already exists. " 311 | f"See: [{repository_path}](https://github.com/{repository_path})" 312 | ) 313 | repository_ready = False 314 | repository_path = None 315 | else: 316 | repository_response = f"✅ **{repository_path}** is available." 317 | repository_ready = True 318 | 319 | # Construct "ready" 320 | if all( 321 | [ 322 | scraper_ready, 323 | maintainer_ready, 324 | repository_ready, 325 | governing_body_type_allowed, 326 | ] 327 | ): 328 | ready_response = "#### ✅ All checks successful :tada:" 329 | else: 330 | ready_response = "#### ❌ Some checks failing" 331 | 332 | # Join all together 333 | comment_response = "\n".join( 334 | [ 335 | governing_body_response, 336 | maintainer_response, 337 | repository_response, 338 | scraper_response, 339 | ready_response, 340 | ] 341 | ) 342 | 343 | # Dump to file 344 | with open("form-validation-results.md", "w") as open_f: 345 | open_f.write(comment_response) 346 | 347 | # Save shorthand repo generation options to file 348 | # If any check failed, the value will be None / null 349 | with open("generation-options.json", "w") as open_f: 350 | json.dump( 351 | { 352 | "scraper_options": scraper_options, 353 | "maintainer_name": maintainer_name, 354 | "repository_path": repository_path, 355 | }, 356 | open_f, 357 | ) 358 | 359 | 360 | def main() -> None: 361 | try: 362 | args = Args() 363 | validate_form( 364 | issue_content_file=args.issue_content_file, 365 | ) 366 | except Exception as e: 367 | log.error("=============================================") 368 | log.error("\n\n" + traceback.format_exc()) 369 | log.error("=============================================") 370 | log.error("\n\n" + str(e) + "\n") 371 | log.error("=============================================") 372 | sys.exit(1) 373 | 374 | 375 | ############################################################################### 376 | # Allow caller to directly run this module (usually in development scenarios) 377 | 378 | if __name__ == "__main__": 379 | main() 380 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Repo Construction 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | test-repo: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | # Setup languages 16 | - name: Set up Python 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: '3.11' 20 | - name: Setup Node 21 | uses: actions/setup-node@v4 22 | with: 23 | node-version: '16.x' 24 | 25 | # Run cookiecutter 26 | - name: Install Cookiecutter 27 | run: | 28 | pip install cookiecutter 29 | - name: Generate Repo 30 | run: | 31 | cookiecutter . --no-input 32 | rm -Rf example/.github/workflows/ 33 | 34 | # Check Python 35 | - name: Install Python Dependencies 36 | run: | 37 | cd example/python/ 38 | pip install .[test] 39 | - name: Lint and Format Python 40 | run: | 41 | cd example/python/ 42 | flake8 cdp_example_backend --count --verbose --show-source --statistics 43 | black --check cdp_example_backend 44 | 45 | # Check Web 46 | - name: Install Web App Dependencies 47 | run: | 48 | cd example/web/ 49 | npm i 50 | - name: Build Web App 51 | run: | 52 | cd example/web/ 53 | npm run build 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # OSX useful to ignore 7 | *.DS_Store 8 | .AppleDouble 9 | .LSOverride 10 | 11 | # Thumbnails 12 | ._* 13 | 14 | # Files that might appear in the root of a volume 15 | .DocumentRevisions-V100 16 | .fseventsd 17 | .Spotlight-V100 18 | .TemporaryItems 19 | .Trashes 20 | .VolumeIcon.icns 21 | .com.apple.timemachine.donotpresent 22 | 23 | # Directories potentially created on remote AFP share 24 | .AppleDB 25 | .AppleDesktop 26 | Network Trash Folder 27 | Temporary Items 28 | .apdisk 29 | 30 | # C extensions 31 | *.so 32 | 33 | # Distribution / packaging 34 | .Python 35 | env/ 36 | build/ 37 | develop-eggs/ 38 | dist/ 39 | downloads/ 40 | eggs/ 41 | .eggs/ 42 | lib/ 43 | lib64/ 44 | parts/ 45 | sdist/ 46 | var/ 47 | *.egg-info/ 48 | .installed.cfg 49 | *.egg 50 | 51 | # PyInstaller 52 | # Usually these files are written by a python script from a template 53 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 54 | *.manifest 55 | *.spec 56 | 57 | # Installer logs 58 | pip-log.txt 59 | pip-delete-this-directory.txt 60 | 61 | # Unit test / coverage reports 62 | htmlcov/ 63 | .tox/ 64 | .coverage 65 | .coverage.* 66 | .cache 67 | nosetests.xml 68 | coverage.xml 69 | *,cover 70 | .hypothesis/ 71 | .pytest_cache/ 72 | 73 | # Translations 74 | *.mo 75 | *.pot 76 | 77 | # Django stuff: 78 | *.log 79 | 80 | # Sphinx documentation 81 | docs/_build/ 82 | 83 | # IntelliJ Idea family of suites 84 | .idea 85 | *.iml 86 | ## File-based project format: 87 | *.ipr 88 | *.iws 89 | ## mpeltonen/sbt-idea plugin 90 | .idea_modules/ 91 | 92 | # PyBuilder 93 | target/ 94 | 95 | # Cookiecutter 96 | output/ 97 | python_boilerplate/ 98 | 99 | # VSCode 100 | .vscode 101 | 102 | # Vim swap files 103 | *.swp 104 | *.swo 105 | 106 | # Project specific 107 | failing-issue.md 108 | successful-issue.md 109 | form-validation-results.md 110 | generation-options.json 111 | planned-cookiecutter.json 112 | example/ -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "creators": [ 3 | { 4 | "name": "Brown, Eva Maxfield", 5 | "affiliation": "University of Washington Information School, University of Washington, Seattle", 6 | "orcid": "0000-0003-2564-0373" 7 | }, 8 | { 9 | "name": "Huynh, To", 10 | "affiliation": "University of Washington, Seattle", 11 | "orcid": "0000-0002-9664-3662" 12 | }, 13 | { 14 | "name": "Na, Isaac", 15 | "affiliation": "Washington University, St. Louis", 16 | "orcid": "0000-0002-0182-1615" 17 | }, 18 | { 19 | "name": "Ledbetter, Brian", 20 | "affiliation": "University of Washington Information School, University of Washington, Seattle" 21 | }, 22 | { 23 | "name": "Ticehurst, Hawk", 24 | "affiliation": "University of Washington Information School, University of Washington, Seattle" 25 | }, 26 | { 27 | "name": "Liu, Sarah" 28 | }, 29 | { 30 | "name": "Gilles, Emily" 31 | }, 32 | { 33 | "name": "Greene, Katlyn M. F." 34 | }, 35 | { 36 | "name": "Cho, Sung" 37 | }, 38 | { 39 | "name": "Ragoler, Shak" 40 | }, 41 | { 42 | "name": "Weber, Nicholas", 43 | "affiliation": "University of Washington Information School, University of Washington, Seattle", 44 | "orcid": "0000-0002-6008-3763" 45 | } 46 | ], 47 | "keywords": [ 48 | "Python", 49 | "JavaScript", 50 | "open-government", 51 | "open-data", 52 | "open-infrastructure", 53 | "municipal-governance", 54 | "data-archival", 55 | "civic-technology", 56 | "natural-language-processing" 57 | ], 58 | "license": "MIT", 59 | "upload_type": "software" 60 | } 61 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # This CITATION.cff file was generated with cffinit. 2 | # Visit https://bit.ly/cffinit to generate yours today! 3 | 4 | cff-version: 1.2.0 5 | title: >- 6 | Council Data Project: Software for Municipal Data 7 | Collection, Analysis, and Publication 8 | message: Please cite this software using these metadata. 9 | type: software 10 | authors: 11 | - given-names: Eva Maxfield 12 | family-names: Brown 13 | email: jmxbrown@uw.edu 14 | affiliation: >- 15 | University of Washington Information School, 16 | University of Washington, Seattle 17 | orcid: 'https://orcid.org/0000-0003-2564-0373' 18 | - given-names: To 19 | family-names: Huynh 20 | affiliation: 'University of Washington, Seattle' 21 | orcid: 'https://orcid.org/0000-0002-9664-3662' 22 | - given-names: Isaac 23 | family-names: Na 24 | affiliation: 'Washington University, St. Louis' 25 | orcid: 'https://orcid.org/0000-0002-0182-1615' 26 | - given-names: Brian 27 | family-names: Ledbetter 28 | affiliation: >- 29 | University of Washington Information School, 30 | University of Washington, Seattle 31 | - given-names: Hawk 32 | family-names: Ticehurst 33 | affiliation: >- 34 | University of Washington Information School, 35 | University of Washington, Seattle 36 | - given-names: Sarah 37 | family-names: Liu 38 | - given-names: Emily 39 | family-names: Gilles 40 | - given-names: Katlyn M. F. 41 | family-names: Greene 42 | - given-names: Sung 43 | family-names: Cho 44 | - given-names: Shak 45 | family-names: Ragoler 46 | - given-names: Nicholas 47 | family-names: Weber 48 | email: nmweber@uw.edu 49 | affiliation: >- 50 | University of Washington Information School, 51 | University of Washington, Seattle 52 | orcid: 'https://orcid.org/0000-0002-6008-3763' 53 | identifiers: 54 | - type: doi 55 | value: 10.21105/joss.03904 56 | - type: url 57 | value: 'https://doi.org/10.21105/joss.03904' 58 | repository-code: >- 59 | https://github.com/CouncilDataProject/cookiecutter-cdp-deployment 60 | url: 'https://councildataproject.org' 61 | abstract: >- 62 | Cities, counties, and states throughout the USA are 63 | bound by law to archive recordings of public 64 | meetings. Most local governments comply with these 65 | laws by posting documents, audio, or video 66 | recordings online. As there is no set standard for 67 | municipal data archives however, parsing and 68 | processing such data is typically time consuming 69 | and highly dependent on each municipality. Council 70 | Data Project (CDP) is a set of open-source tools 71 | that improve the accessibility of local government 72 | data by systematically collecting, transforming, 73 | and re-publishing this data to the web. The data 74 | re-published by CDP is packaged and presented 75 | within a searchable web application that vastly 76 | simplifies the process of finding specific 77 | information within the archived data. We envision 78 | this project being used by a variety of groups 79 | including civic technologists hoping to promote 80 | government transparency, researchers focused on 81 | public policy, natural language processing, machine 82 | learning, or information retrieval and discovery, 83 | and many others. 84 | keywords: 85 | - public interest technology 86 | - open government 87 | - open data 88 | - data archival 89 | - municipal governance 90 | - Python 91 | - JavaScript 92 | - natural language processing 93 | license: MIT 94 | commit: 59c30846b40c289b1a8b6ab10b0b46b5bdfc3c59 95 | version: 3.0.0 96 | date-released: '2021-12-02' 97 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting any of the maintainers of this project and 59 | we will attempt to resolve the issues with respect and dignity. 60 | 61 | Project maintainers who do not follow or enforce the Code of Conduct in good 62 | faith may face temporary or permanent repercussions as determined by other 63 | members of the project's leadership. 64 | 65 | ## Attribution 66 | 67 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 68 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 69 | 70 | [homepage]: https://www.contributor-covenant.org 71 | 72 | For answers to common questions about this code of conduct, see 73 | https://www.contributor-covenant.org/faq 74 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome, and they are greatly appreciated! Every little bit 4 | helps, and credit will always be given. 5 | 6 | ## Cookiecutter Contribution vs Application Contribution 7 | 8 | Please note that this repository is only the cookiecutter and not the entire 9 | CDP tooling and infrastructure ecosystem. This repository ties all of our 10 | tooling together into a single repository that is easy to deploy and maintain. 11 | Contributions to this repository should largely be documentation, 12 | devops, bugfixes, or similar. 13 | 14 | If you experience a bug or incorrect documentation while using the cookiecutter 15 | please do send us a Pull Request! If you want to add or fix an auto-deployment 16 | bot, or add more GitHub Actions to the produced repository, all such contributions 17 | welcome and appreciated. 18 | 19 | Examples of these types of contributions include: 20 | 21 | - adding more instance admin documentation to the generated repository 22 | - updating the `cdp-backend` and `cdp-frontend` version pins 23 | - upgrading or fixing and auto-deployment GitHub Action 24 | - adding new GitHub Actions to the generated repository 25 | 26 | For contributions to the major pipelines and infrastructure that are used by all 27 | CDP deployments, please see: 28 | [cdp-backend](https://github.com/councildataproject/cdp-backend) 29 | 30 | For contributions to the web application which is used by all CDP deployments, please 31 | see: [cdp-frontend](https://github.com/councildataproject/cdp-frontend) 32 | 33 | For contributions to the existing event scrapers used by some CDP deployments, please 34 | see: [cdp-scrapers](https://github.com/councildataproject/cdp-scrapers) 35 | 36 | ## Get Started! 37 | 38 | Ready to contribute? Here's how to set up `cookiecutter-cdp-deployment` for local development. 39 | 40 | 1. Fork the `cookiecutter-cdp-deployment` repo on GitHub. 41 | 42 | 2. Clone your fork locally: 43 | 44 | ```bash 45 | git clone git@github.com:{your_name_here}/cookiecutter-cdp-deployment.git 46 | ``` 47 | 48 | 3. Install `cookiecutter`. (It is also recommended to work in a virtualenv or anaconda environment): 49 | 50 | ```bash 51 | cd cookiecutter-cdp-deployment/ 52 | pip install cookiecutter 53 | ``` 54 | 55 | 4. Create a branch for local development: 56 | 57 | ```bash 58 | git checkout -b {your_development_type}/short-description 59 | ``` 60 | 61 | Ex: feature/read-tiff-files or bugfix/handle-file-not-found
62 | Now you can make your changes locally. 63 | 64 | 5. When you're done making changes, check that the cookiecutter still generates 65 | properly: 66 | 67 | ```bash 68 | cookiecutter . --no-input 69 | ``` 70 | 71 | 6. Commit your changes and push your branch to GitHub: 72 | 73 | ```bash 74 | git add . 75 | git commit -m "Resolves gh-###. Your detailed description of your changes." 76 | git push origin {your_development_type}/short-description 77 | ``` 78 | 79 | 7. Submit a pull request through the GitHub website. 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Council Data Project 2 | 3 | Mozilla Public License, version 2.0 4 | 5 | 1. Definitions 6 | 7 | 1.1. “Contributor” 8 | 9 | means each individual or legal entity that creates, contributes to the 10 | creation of, or owns Covered Software. 11 | 12 | 1.2. “Contributor Version” 13 | 14 | means the combination of the Contributions of others (if any) used by a 15 | Contributor and that particular Contributor’s Contribution. 16 | 17 | 1.3. “Contribution” 18 | 19 | means Covered Software of a particular Contributor. 20 | 21 | 1.4. “Covered Software” 22 | 23 | means Source Code Form to which the initial Contributor has attached the 24 | notice in Exhibit A, the Executable Form of such Source Code Form, and 25 | Modifications of such Source Code Form, in each case including portions 26 | thereof. 27 | 28 | 1.5. “Incompatible With Secondary Licenses” 29 | means 30 | 31 | a. that the initial Contributor has attached the notice described in 32 | Exhibit B to the Covered Software; or 33 | 34 | b. that the Covered Software was made available under the terms of version 35 | 1.1 or earlier of the License, but not also under the terms of a 36 | Secondary License. 37 | 38 | 1.6. “Executable Form” 39 | 40 | means any form of the work other than Source Code Form. 41 | 42 | 1.7. “Larger Work” 43 | 44 | means a work that combines Covered Software with other material, in a separate 45 | file or files, that is not Covered Software. 46 | 47 | 1.8. “License” 48 | 49 | means this document. 50 | 51 | 1.9. “Licensable” 52 | 53 | means having the right to grant, to the maximum extent possible, whether at the 54 | time of the initial grant or subsequently, any and all of the rights conveyed by 55 | this License. 56 | 57 | 1.10. “Modifications” 58 | 59 | means any of the following: 60 | 61 | a. any file in Source Code Form that results from an addition to, deletion 62 | from, or modification of the contents of Covered Software; or 63 | 64 | b. any new file in Source Code Form that contains any Covered Software. 65 | 66 | 1.11. “Patent Claims” of a Contributor 67 | 68 | means any patent claim(s), including without limitation, method, process, 69 | and apparatus claims, in any patent Licensable by such Contributor that 70 | would be infringed, but for the grant of the License, by the making, 71 | using, selling, offering for sale, having made, import, or transfer of 72 | either its Contributions or its Contributor Version. 73 | 74 | 1.12. “Secondary License” 75 | 76 | means either the GNU General Public License, Version 2.0, the GNU Lesser 77 | General Public License, Version 2.1, the GNU Affero General Public 78 | License, Version 3.0, or any later versions of those licenses. 79 | 80 | 1.13. “Source Code Form” 81 | 82 | means the form of the work preferred for making modifications. 83 | 84 | 1.14. “You” (or “Your”) 85 | 86 | means an individual or a legal entity exercising rights under this 87 | License. For legal entities, “You” includes any entity that controls, is 88 | controlled by, or is under common control with You. For purposes of this 89 | definition, “control” means (a) the power, direct or indirect, to cause 90 | the direction or management of such entity, whether by contract or 91 | otherwise, or (b) ownership of more than fifty percent (50%) of the 92 | outstanding shares or beneficial ownership of such entity. 93 | 94 | 95 | 2. License Grants and Conditions 96 | 97 | 2.1. Grants 98 | 99 | Each Contributor hereby grants You a world-wide, royalty-free, 100 | non-exclusive license: 101 | 102 | a. under intellectual property rights (other than patent or trademark) 103 | Licensable by such Contributor to use, reproduce, make available, 104 | modify, display, perform, distribute, and otherwise exploit its 105 | Contributions, either on an unmodified basis, with Modifications, or as 106 | part of a Larger Work; and 107 | 108 | b. under Patent Claims of such Contributor to make, use, sell, offer for 109 | sale, have made, import, and otherwise transfer either its Contributions 110 | or its Contributor Version. 111 | 112 | 2.2. Effective Date 113 | 114 | The licenses granted in Section 2.1 with respect to any Contribution become 115 | effective for each Contribution on the date the Contributor first distributes 116 | such Contribution. 117 | 118 | 2.3. Limitations on Grant Scope 119 | 120 | The licenses granted in this Section 2 are the only rights granted under this 121 | License. No additional rights or licenses will be implied from the distribution 122 | or licensing of Covered Software under this License. Notwithstanding Section 123 | 2.1(b) above, no patent license is granted by a Contributor: 124 | 125 | a. for any code that a Contributor has removed from Covered Software; or 126 | 127 | b. for infringements caused by: (i) Your and any other third party’s 128 | modifications of Covered Software, or (ii) the combination of its 129 | Contributions with other software (except as part of its Contributor 130 | Version); or 131 | 132 | c. under Patent Claims infringed by Covered Software in the absence of its 133 | Contributions. 134 | 135 | This License does not grant any rights in the trademarks, service marks, or 136 | logos of any Contributor (except as may be necessary to comply with the 137 | notice requirements in Section 3.4). 138 | 139 | 2.4. Subsequent Licenses 140 | 141 | No Contributor makes additional grants as a result of Your choice to 142 | distribute the Covered Software under a subsequent version of this License 143 | (see Section 10.2) or under the terms of a Secondary License (if permitted 144 | under the terms of Section 3.3). 145 | 146 | 2.5. Representation 147 | 148 | Each Contributor represents that the Contributor believes its Contributions 149 | are its original creation(s) or it has sufficient rights to grant the 150 | rights to its Contributions conveyed by this License. 151 | 152 | 2.6. Fair Use 153 | 154 | This License is not intended to limit any rights You have under applicable 155 | copyright doctrines of fair use, fair dealing, or other equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under the 169 | terms of this License. You must inform recipients that the Source Code Form 170 | of the Covered Software is governed by the terms of this License, and how 171 | they can obtain a copy of this License. You may not attempt to alter or 172 | restrict the recipients’ rights in the Source Code Form. 173 | 174 | 3.2. Distribution of Executable Form 175 | 176 | If You distribute Covered Software in Executable Form then: 177 | 178 | a. such Covered Software must also be made available in Source Code Form, 179 | as described in Section 3.1, and You must inform recipients of the 180 | Executable Form how they can obtain a copy of such Source Code Form by 181 | reasonable means in a timely manner, at a charge no more than the cost 182 | of distribution to the recipient; and 183 | 184 | b. You may distribute such Executable Form under the terms of this License, 185 | or sublicense it under different terms, provided that the license for 186 | the Executable Form does not attempt to limit or alter the recipients’ 187 | rights in the Source Code Form under this License. 188 | 189 | 3.3. Distribution of a Larger Work 190 | 191 | You may create and distribute a Larger Work under terms of Your choice, 192 | provided that You also comply with the requirements of this License for the 193 | Covered Software. If the Larger Work is a combination of Covered Software 194 | with a work governed by one or more Secondary Licenses, and the Covered 195 | Software is not Incompatible With Secondary Licenses, this License permits 196 | You to additionally distribute such Covered Software under the terms of 197 | such Secondary License(s), so that the recipient of the Larger Work may, at 198 | their option, further distribute the Covered Software under the terms of 199 | either this License or such Secondary License(s). 200 | 201 | 3.4. Notices 202 | 203 | You may not remove or alter the substance of any license notices (including 204 | copyright notices, patent notices, disclaimers of warranty, or limitations 205 | of liability) contained within the Source Code Form of the Covered 206 | Software, except that You may alter any license notices to the extent 207 | required to remedy known factual inaccuracies. 208 | 209 | 3.5. Application of Additional Terms 210 | 211 | You may choose to offer, and to charge a fee for, warranty, support, 212 | indemnity or liability obligations to one or more recipients of Covered 213 | Software. However, You may do so only on Your own behalf, and not on behalf 214 | of any Contributor. You must make it absolutely clear that any such 215 | warranty, support, indemnity, or liability obligation is offered by You 216 | alone, and You hereby agree to indemnify every Contributor for any 217 | liability incurred by such Contributor as a result of warranty, support, 218 | indemnity or liability terms You offer. You may include additional 219 | disclaimers of warranty and limitations of liability specific to any 220 | jurisdiction. 221 | 222 | 4. Inability to Comply Due to Statute or Regulation 223 | 224 | If it is impossible for You to comply with any of the terms of this License 225 | with respect to some or all of the Covered Software due to statute, judicial 226 | order, or regulation then You must: (a) comply with the terms of this License 227 | to the maximum extent possible; and (b) describe the limitations and the code 228 | they affect. Such description must be placed in a text file included with all 229 | distributions of the Covered Software under this License. Except to the 230 | extent prohibited by statute or regulation, such description must be 231 | sufficiently detailed for a recipient of ordinary skill to be able to 232 | understand it. 233 | 234 | 5. Termination 235 | 236 | 5.1. The rights granted under this License will terminate automatically if You 237 | fail to comply with any of its terms. However, if You become compliant, 238 | then the rights granted under this License from a particular Contributor 239 | are reinstated (a) provisionally, unless and until such Contributor 240 | explicitly and finally terminates Your grants, and (b) on an ongoing basis, 241 | if such Contributor fails to notify You of the non-compliance by some 242 | reasonable means prior to 60 days after You have come back into compliance. 243 | Moreover, Your grants from a particular Contributor are reinstated on an 244 | ongoing basis if such Contributor notifies You of the non-compliance by 245 | some reasonable means, this is the first time You have received notice of 246 | non-compliance with this License from such Contributor, and You become 247 | compliant prior to 30 days after Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, counter-claims, 251 | and cross-claims) alleging that a Contributor Version directly or 252 | indirectly infringes any patent, then the rights granted to You by any and 253 | all Contributors for the Covered Software under Section 2.1 of this License 254 | shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 257 | license agreements (excluding distributors and resellers) which have been 258 | validly granted by You or Your distributors under this License prior to 259 | termination shall survive termination. 260 | 261 | 6. Disclaimer of Warranty 262 | 263 | Covered Software is provided under this License on an “as is” basis, without 264 | warranty of any kind, either expressed, implied, or statutory, including, 265 | without limitation, warranties that the Covered Software is free of defects, 266 | merchantable, fit for a particular purpose or non-infringing. The entire 267 | risk as to the quality and performance of the Covered Software is with You. 268 | Should any Covered Software prove defective in any respect, You (not any 269 | Contributor) assume the cost of any necessary servicing, repair, or 270 | correction. This disclaimer of warranty constitutes an essential part of this 271 | License. No use of any Covered Software is authorized under this License 272 | except under this disclaimer. 273 | 274 | 7. Limitation of Liability 275 | 276 | Under no circumstances and under no legal theory, whether tort (including 277 | negligence), contract, or otherwise, shall any Contributor, or anyone who 278 | distributes Covered Software as permitted above, be liable to You for any 279 | direct, indirect, special, incidental, or consequential damages of any 280 | character including, without limitation, damages for lost profits, loss of 281 | goodwill, work stoppage, computer failure or malfunction, or any and all 282 | other commercial damages or losses, even if such party shall have been 283 | informed of the possibility of such damages. This limitation of liability 284 | shall not apply to liability for death or personal injury resulting from such 285 | party’s negligence to the extent applicable law prohibits such limitation. 286 | Some jurisdictions do not allow the exclusion or limitation of incidental or 287 | consequential damages, so this exclusion and limitation may not apply to You. 288 | 289 | 8. Litigation 290 | 291 | Any litigation relating to this License may be brought only in the courts of 292 | a jurisdiction where the defendant maintains its principal place of business 293 | and such litigation shall be governed by laws of that jurisdiction, without 294 | reference to its conflict-of-law provisions. Nothing in this Section shall 295 | prevent a party’s ability to bring cross-claims or counter-claims. 296 | 297 | 9. Miscellaneous 298 | 299 | This License represents the complete agreement concerning the subject matter 300 | hereof. If any provision of this License is held to be unenforceable, such 301 | provision shall be reformed only to the extent necessary to make it 302 | enforceable. Any law or regulation which provides that the language of a 303 | contract shall be construed against the drafter shall not be used to construe 304 | this License against a Contributor. 305 | 306 | 307 | 10. Versions of the License 308 | 309 | 10.1. New Versions 310 | 311 | Mozilla Foundation is the license steward. Except as provided in Section 312 | 10.3, no one other than the license steward has the right to modify or 313 | publish new versions of this License. Each version will be given a 314 | distinguishing version number. 315 | 316 | 10.2. Effect of New Versions 317 | 318 | You may distribute the Covered Software under the terms of the version of 319 | the License under which You originally received the Covered Software, or 320 | under the terms of any subsequent version published by the license 321 | steward. 322 | 323 | 10.3. Modified Versions 324 | 325 | If you create software not governed by this License, and you want to 326 | create a new license for such software, you may create and use a modified 327 | version of this License if you rename the license and remove any 328 | references to the name of the license steward (except to note that such 329 | modified license differs from this License). 330 | 331 | 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses 332 | If You choose to distribute Source Code Form that is Incompatible With 333 | Secondary Licenses under the terms of this version of the License, the 334 | notice described in Exhibit B of this License must be attached. 335 | 336 | Exhibit A - Source Code Form License Notice 337 | 338 | This Source Code Form is subject to the 339 | terms of the Mozilla Public License, v. 340 | 2.0. If a copy of the MPL was not 341 | distributed with this file, You can 342 | obtain one at 343 | http://mozilla.org/MPL/2.0/. 344 | 345 | If it is not possible or desirable to put the notice in a particular file, then 346 | You may include the notice in a location (such as a LICENSE file in a relevant 347 | directory) where a recipient would be likely to look for such a notice. 348 | 349 | You may add additional accurate notices of copyright ownership. 350 | 351 | Exhibit B - “Incompatible With Secondary Licenses” Notice 352 | 353 | This Source Code Form is “Incompatible 354 | With Secondary Licenses”, as defined by 355 | the Mozilla Public License, v. 2.0. 356 | 357 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cookiecutter-cdp-deployment 2 | 3 | [![Cookiecutter Check Status](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment/workflows/Build%20Example%20Repo/badge.svg)](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment/tree/example-build) 4 | [![DOI](https://joss.theoj.org/papers/10.21105/joss.03904/status.svg)](https://doi.org/10.21105/joss.03904) 5 | 6 | Cookiecutter template for creating new Council Data Project deployments. 7 | 8 | --- 9 | 10 | ## Council Data Project 11 | 12 | Council Data Project is an open-source project dedicated to providing journalists, 13 | activists, researchers, and all members of each community we serve with the tools 14 | they need to stay informed and hold their Council Members accountable. 15 | 16 | For more information about Council Data Project, please visit 17 | [our website](https://councildataproject.org/). 18 | 19 | ## About 20 | 21 | This repository is a "cookiecutter template" for an entirely new 22 | Council Data Project (CDP) Instance. By following the steps defined in 23 | the [Usage](#usage) section, our tools will create and manage all the database, 24 | file storage, and processing infrastructure needed to serve the CDP web application. 25 | 26 | While our tools will setup and manage all processing and storage infrastructure, you 27 | (or your team) must provide and maintain the custom Python code to gather event 28 | information and handle billing for the costs of the deployment. 29 | 30 | For more information about costs and billing, see [Cost](#cost). 31 | 32 | ### CDP Instance Features 33 | 34 | - Plain text search of past events and meeting items
35 | _(search for "missing middle housing" or "bike lanes")_ 36 | - Filter and sort event and meeting item search results
37 | _(filter by date range, committee, etc.)_ 38 | - Automatic timestamped-transcript generation
39 | _(jump right to a specific public comment or debate)_ 40 | - Meeting item and amendment tracking
41 | _(check for amendment passage, upcoming meetings, etc.)_ 42 | - Share event at timepoint
43 | _(jump right to the point in the meeting you want to share)_ 44 | - Full event minutes details
45 | _(view all documents and presentations related to each event)_ 46 | 47 | See the current [Seattle CDP Instance](https://councildataproject.org/seattle/#/) for a live example. 48 | 49 | _Note: Some features are dependent on how much data is provided during event gather._ 50 | _More information see our [ingestion models documentation](https://councildataproject.org/cdp-backend/ingestion_models.html)._ 51 | 52 | ## Usage 53 | 54 | Regardless of your deployment strategy, you may find reading the 55 | [Things to Know](#things-to-know) section helpful prior to deployment. 56 | 57 | _Note: while this cookiecutter will help you setup a repository and CDP infrastructure,_ 58 | _you will still need to write your own custom data ingestion function._ 59 | _Writing a basic data ingestion function ranges from taking a couple of hours_ 60 | _to a couple of days depending on how much data you want to provide to our system._ 61 | 62 | ### Deploying Under the councildataproject.org Domain 63 | 64 | If you want your deployment under the councildataproject.org domain (i.e. https://councildataproject.org/seattle), 65 | you will need to fill out the 66 | ["New Instance Deployment" Issue Form](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment/issues/new/choose). 67 | 68 | The Council Data Project team will help you along in the process on the 69 | issue from there. 70 | 71 | ### Deploying Under Your Own Domain 72 | 73 | If you want to host your deployment under a different domain 74 | (i.e. Your-Org-Name.github.io/your-municipality), 75 | you will need to install `cookiecutter` and use this template. 76 | 77 | [**Follow along with the video walkthrough**](https://youtu.be/xdRhh-ocSfc) 78 | 79 | Before you begin, please note that you will need to install or have available the following: 80 | 81 | - [gcloud](https://cloud.google.com/sdk/docs/install) 82 | - [gsutil](https://cloud.google.com/storage/docs/gsutil_install) 83 | - [Python 3.10+](https://www.python.org/downloads/) (Any Python version greater than or equal to 3.10) 84 | 85 | Once all tools are installed, the rest of the infrastructure setup process 86 | should take an hour or two. 87 | 88 | In a terminal with Python 3.10+ installed: 89 | 90 | ```bash 91 | pip install cookiecutter 92 | cookiecutter gh:CouncilDataProject/cookiecutter-cdp-deployment 93 | ``` 94 | 95 | Follow the prompts in your terminal and fill in the details for the instance deployment. 96 | At the end of the process a new directory will have been created with all required 97 | files and further instructions to set up your new deployment. 98 | 99 | For more details and examples on each parameter of this cookiecutter template, 100 | see [Cookiecutter Parameters](#cookiecutter-parameters). 101 | 102 | Follow the steps in the "Initial Repository Setup" section of the 103 | `README.md` file within the generated `SETUP` directory. 104 | 105 | For more details on what is created from using this cookiecutter template, 106 | see [Cookiecutter Repo Generation](#cookiecutter-repo-generation). 107 | 108 | The short summary of setup tasks remaining are: 109 | 110 | - The creation of a new GitHub repository for the instance. 111 | - Logging in or creating an account for Google Cloud. 112 | - Initialize the basic infrastructure. 113 | - Assign a billing account to the created Google Cloud project. 114 | - Generate credentials for the Google Project for use in automated scripts. 115 | - Attach credentials as secrets to the GitHub repository. 116 | - Push the cookiecutter generated files to the GitHub repository. 117 | - Setup web hosting through GitHub Pages. 118 | - Enable open access for data stored by Google Cloud and Firebase. 119 | - Write a data ingestion function for your municipality (it may be useful to 120 | build off of [cdp-scrapers](https://github.com/CouncilDataProject/cdp-scrapers)). 121 | 122 | You can also see an example generated repository and the full steps listed 123 | [here](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment/tree/example-build/SETUP). 124 | 125 | ### Cookiecutter Parameters 126 | 127 | | Parameter | Description | Example 1 | Example 2 | 128 | | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------- | 129 | | municipality | The name of the municipality (town, city, county, etc.) that this CDP Instance will store data for. | Seattle | King County | 130 | | iana_timezone | The [IANA Timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) of the municipality that this CDP instance is for. | America/Los_Angeles | America/Chicago | 131 | | governing_body_type | What type of governing body this instance is for. | city council | county council | 132 | | municipality_slug | The name of the municipality cleaned for use in the web application and parts of repository naming. | seattle | king-county | 133 | | python_municipality_slug | The name of the municipality cleaned for use in specifically Python parts of the application. | seattle | king_county | 134 | | infrastructure_slug | The name of the municipality cleaned for use in specifically application infrastructure. Must be globally unique to GCP. | cdp-seattle-abasjkqy | cdp-king-county-uiqmsbaw | 135 | | maintainer_or_org_full_name | The full name of the primary maintainer or organization that will be managing this instance deployment. | Eva Maxfield Brown | Council Data Project | 136 | | hosting_github_username_or_org | The GitHub username or organization that will host this instance's repository. (Used in the web application's domain name) | evamaxfield | CouncilDataProject | 137 | | hosting_github_repo_name | A specific name to give to the repository. (Used in the web application's full address) | cdp-seattle | king-county | 138 | | hosting_github_url | From the provided information, the expected URL of the GitHub repository. | https://github.com/evamaxfield/cdp-seattle | https://github.com/CouncilDataProject/king-county | 139 | | hosting_web_app_address | From the provided information, the expected URL of the web application. | https://evamaxfield.github.io/cdp-seattle | https://councildataproject.org/king-county | 140 | | firestore_region | The desired region to host the firestore instance. ([Firestore docs](https://firebase.google.com/docs/firestore/locations)) | us-west1 | europe-central2 | 141 | | event_gather_timedelta_lookback_days | The number of days to look back from the current date every time the event scraper runs. | 2 | 6 | 142 | | event_gather_cron | The event gather CRON configuration. ([GitHub Actions CRON Details](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule)) | 26 0,6,12,18 * * * | 17 3,9,15,21 * * * | 143 | | event_gather_runner_timeout_minutes | Minutes to wait before creating a CML runner attempt will fail. | 15 | 16 | 144 | | event_gather_runner_max_attempts | Number of times to attempt to create a CML runner. | 4 | 36 | 145 | | event_gather_runner_retry_wait_seconds | Number of seconds to wait between CML runner create attempts. | 600 | 600 | 146 | 147 | ### Things to Know 148 | 149 | Much of Council Data Project processing and resource management can be handled for free 150 | and purely on GitHub. However we do rely on a select few resources outside of GitHub 151 | to manage all services and applications. 152 | 153 | The only service that will require a billing account to manage payment for resources 154 | used, is [Google Cloud](#google-cloud). Google Cloud will manage all databases, 155 | file storage, and heavy-compute such as [speech-to-text](#speech-to-text) for transcription. 156 | You can see more about the average monthly cost of running a 157 | CDP Instance in [Cost](#cost). 158 | 159 | For more details see [Cookiecutter Repo Generation](#cookiecutter-repo-generation). 160 | _After creating the repo, the following steps will have instructions and links specific_ 161 | _to your deployment in the generated repository's README._ 162 | 163 | ### Cookiecutter Repo Generation 164 | 165 | `Cookiecutter` is a Python package to generate templated projects. This repository is 166 | a template for `cookiecutter` to generate a CDP deployment repository which contains 167 | following: 168 | 169 | - A directory structure for your project 170 | - A directory for your web application to build and deploy from 171 | - A directory for infrastructure management 172 | - A directory for your Python event gather function and it's requirements 173 | - Continuous integration 174 | - Preconfigured for your web application to fully deploy 175 | - Preconfigured to deploy all required CDP infrastructure 176 | - Preconfigured to run CDP pipelines using GitHub Actions 177 | 178 | To generate a new repository from this template, in a terminal with Python 3.5+ 179 | installed, run: 180 | 181 | ```bash 182 | pip install cookiecutter 183 | cookiecutter gh:CouncilDataProject/cookiecutter-cdp-deployment 184 | ``` 185 | 186 | _Note: This will only create the basic repository._ 187 | _You will still need to set up a Google Cloud account._ 188 | 189 | ### Google Cloud 190 | 191 | All of your deployments data and some data processing will be done using 192 | Google Cloud Platform (GCP). 193 | 194 | - Your deployment's provided and generated data (meeting dates, committee names, councilmember details, etc) will live in [Firestore](https://cloud.google.com/firestore). 195 | - Your deployment's generated files (audio clips, transcripts, etc.) will live in [Filestore](https://cloud.google.com/filestore). 196 | - The audio from the provided video will be processed using [Whisper](https://github.com/openai/whisper) 197 | on [Google Compute Engine](https://cloud.google.com/compute/). 198 | - We additionally use [Faster-Whisper](https://github.com/guillaumekln/faster-whisper). 199 | 200 | ## Cost 201 | 202 | CDP was created and maintained by a group of people working on it in their free time. 203 | We didn't want to pay extreme amounts of money so why should you? 204 | 205 | To that end, we try to make CDP as low cost as possible. Many of the current features 206 | are entirely free as long as the repo is open source: 207 | 208 | Free Resources and Infrastructure: 209 | 210 | - Event Processing (GitHub Actions) 211 | - Event and Legislation Indexing (GitHub Actions) 212 | - Web Hosting (GitHub Pages) 213 | 214 | The backend resources and processing are the only real costs and depend on usage. 215 | The more users that use your web application, the more the database and 216 | file storage cost. The CDP-Seattle monthly averages below are for the most utilized 217 | months of its existence so take these as close to upper-bounds. 218 | 219 | Billed Resources and Infrastructure: 220 | 221 | - [Cloud Firestore Pricing](https://firebase.google.com/pricing/) 222 | _CDP-Seattle monthly average: ~$40.00_ 223 | - [Google Storage Pricing](https://cloud.google.com/storage/pricing#price-tables) 224 | _CDP-Seattle monthly average: ~$1.00_ 225 | - [Google Compute Engine Pricing](https://cloud.google.com/compute/all-pricing) (Using n1-standard-4 in us-central) 226 | _CDP-Seattle monthly average: ~$20.00_ 227 | 228 | **Total Average Monthly Cost**: $61.00 229 | 230 | This is the ongoing cost of storing new meetings as they occur once your instance is deployed. 231 | You may have an additonal upfront cost if you are seeding your database with older videos and 232 | using speech-to-text to transcribe them. 233 | 234 | ### Future Processing Features 235 | 236 | As we add more features to CDP that require additional processing or resources we 237 | will continue to try to minimize their costs wherever possible. 238 | Further, if a feature is optional, we will create a flag that maintainers can set 239 | to include or exclude the additional processing or resource usage. 240 | See [Upgrades and New Features](#upgrades-and-new-features) for more information. 241 | 242 | ## Upgrades and New Features 243 | 244 | In general, all upgrades, bugfixes, new features, and more will be delivered to your 245 | CDP repository via [Dependabot](https://github.com/dependabot). 246 | 247 | After releasing a new version of `cdp-backend` or `cdp-frontend`, GitHub and Dependabot 248 | will automatically create a pull request to your instance repository which updates 249 | the version requirements of the pipelines, infrastructure, and/or web application. 250 | 251 | These pull requests will contain the release notes for the each version that it upgrades 252 | through, i.e. if it upgrades from 3.0.7 to 3.0.9, it will contain the release notes 253 | for both 3.0.8 and 3.0.9. This should help you as a maintainer understand 254 | what each upgrade is fixing or adding. 255 | 256 | An example of such an automated pull request can be seen 257 | [here](https://github.com/CouncilDataProject/seattle/pull/5). 258 | 259 | Finally, in the case that an upgrade requires some additional work for the maintainer, 260 | i.e. "regenerate the latest cookiecutter," or "run this script" -- we will explicitly 261 | say so in our release notes. Those additional tasks are usually quite simple we just 262 | haven't fully automated them yet. 263 | 264 | An example of why we may ask for the maintainer to run a script after merging, 265 | would be to backfill the data needed for a new feature. For example, if we update our 266 | data model to allow for some new feature, data moving forward may be fine but data 267 | from the past will be missing values and it may be optional but recommended to run the 268 | backfill script to have the new feature available for all historical data. 269 | 270 | ## Citation 271 | 272 | If you have found CDP software, data, or ideas useful in your own work, 273 | please consider citing us: 274 | 275 | Brown et al., (2021). Council Data Project: Software for Municipal Data Collection, Analysis, and Publication. Journal of Open Source Software, 6(68), 3904, https://doi.org/10.21105/joss.03904 276 | 277 | ```bibtex 278 | @article{Brown2021, 279 | doi = {10.21105/joss.03904}, 280 | url = {https://doi.org/10.21105/joss.03904}, 281 | year = {2021}, 282 | publisher = {The Open Journal}, 283 | volume = {6}, 284 | number = {68}, 285 | pages = {3904}, 286 | author = {Eva Maxfield Brown and To Huynh and Isaac Na and Brian Ledbetter and Hawk Ticehurst and Sarah Liu and Emily Gilles and Katlyn M. f. Greene and Sung Cho and Shak Ragoler and Nicholas Weber}, 287 | title = {{Council Data Project: Software for Municipal Data Collection, Analysis, and Publication}}, 288 | journal = {Journal of Open Source Software} 289 | } 290 | ``` 291 | 292 | ## License 293 | 294 | [MIT](./LICENSE) 295 | -------------------------------------------------------------------------------- /cookiecutter.json: -------------------------------------------------------------------------------- 1 | { 2 | "municipality": "Example", 3 | "iana_timezone": "America/Los_Angeles", 4 | "governing_body_type": [ 5 | "city council", 6 | "county council", 7 | "school board", 8 | "other" 9 | ], 10 | "municipality_slug": "{{ cookiecutter.municipality.lower().replace(' ', '-').replace('_', '-') }}", 11 | "python_municipality_slug": "{{ cookiecutter.municipality_slug.replace('-', '_') }}", 12 | "infrastructure_slug": "cdp-{{ cookiecutter.municipality_slug }}-{{ random_ascii_string(8).lower() }}", 13 | "maintainer_or_org_full_name": "Council Data Project Contributors", 14 | "hosting_github_username_or_org": "CouncilDataProject", 15 | "hosting_github_repo_name": "{{ cookiecutter.municipality_slug }}", 16 | "hosting_github_url": "{{ '/'.join(['https://github.com', cookiecutter.hosting_github_username_or_org, cookiecutter.hosting_github_repo_name]) }}", 17 | "hosting_web_app_address": "https://{{ cookiecutter.hosting_github_username_or_org}}.github.io/{{ cookiecutter.hosting_github_repo_name }}", 18 | "firestore_region": [ 19 | "us-central", 20 | "us-west1", 21 | "us-west2", 22 | "us-west3", 23 | "us-west4", 24 | "northamerica-northeast1", 25 | "us-east1", 26 | "us-east4", 27 | "southamerica-east1", 28 | "europe-west2", 29 | "europe-west3", 30 | "europe-central2", 31 | "europe-west6", 32 | "asia-south1", 33 | "asia-southeast1", 34 | "asia-southeast2", 35 | "asia-east2", 36 | "asia-east1", 37 | "asia-northeast1", 38 | "asia-northeast2", 39 | "asia-northeast3", 40 | "australia-southeast1" 41 | ], 42 | "speech_to_text_model_version": [ 43 | "medium", 44 | "large", 45 | "small" 46 | ], 47 | "event_gather_timedelta_lookback_days": 2, 48 | "event_gather_cron": "{{ random_integer(0, 59) }} {{ random_integer(0, 23) }} * * *", 49 | "enable_clipping": ["true", "false"], 50 | "_extensions": [ 51 | "cookiecutter.extensions.RandomStringExtension", 52 | "local_extensions.RandomIntegerExtension" 53 | ], 54 | "event_gather_runner_timeout_minutes": 15, 55 | "event_gather_runner_max_attempts": 4, 56 | "event_gather_runner_retry_wait_seconds": 600 57 | } 58 | -------------------------------------------------------------------------------- /hooks/post_gen_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pathlib import Path 5 | 6 | 7 | class AnsiColors: 8 | HEADER = "\033[95m" 9 | OKBLUE = "\033[94m" 10 | OKCYAN = "\033[96m" 11 | OKGREEN = "\033[92m" 12 | WARNING = "\033[93m" 13 | FAIL = "\033[91m" 14 | ENDC = "\033[0m" 15 | BOLD = "\033[1m" 16 | UNDERLINE = "\033[4m" 17 | 18 | 19 | deployment_dir = Path(".").resolve() 20 | 21 | print() 22 | print( 23 | f"🎊 Success! Generated CDP Instance repo at {AnsiColors.OKGREEN}{deployment_dir}{AnsiColors.ENDC}." 24 | ) 25 | print() 26 | print( 27 | "To finish CDP Instance initialization, follow the instructions in SETUP/README.md." 28 | ) 29 | print() 30 | -------------------------------------------------------------------------------- /hooks/pre_gen_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | 6 | # Validate the specified Google Cloud Project name is <= 30 chars 7 | infrastructure_slug = '{{ cookiecutter.infrastructure_slug }}' 8 | 9 | if len(infrastructure_slug) > 30: 10 | print(f"ERROR: {infrastructure_slug} is not a valid Google Cloud Project name! (>30 characters)") 11 | sys.exit(1) 12 | -------------------------------------------------------------------------------- /local_extensions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import random 4 | 5 | from jinja2.ext import Extension 6 | 7 | ############################################################################### 8 | 9 | 10 | class RandomIntegerExtension(Extension): 11 | """ 12 | Enables the `random_integer` function for use in jinja injection. 13 | 14 | Example 15 | ------- 16 | Creating a CRON string which runs at a random time each day. 17 | 18 | { 19 | "cron": "{{ random_integer(0, 59) }} {{ random_integer(0, 23) }} * * *" 20 | } 21 | """ 22 | def __init__(self, environment): 23 | super().__init__(environment) 24 | 25 | def random_integer(min: int, max: int) -> int: 26 | return random.randint(min, max) 27 | 28 | environment.globals.update(random_integer=random_integer) -------------------------------------------------------------------------------- /paper/assets/cdp_core_infrastructure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/paper/assets/cdp_core_infrastructure.png -------------------------------------------------------------------------------- /paper/assets/event-page-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/paper/assets/event-page-screenshot.png -------------------------------------------------------------------------------- /paper/paper.bib: -------------------------------------------------------------------------------- 1 | @article{Trounstine2009, 2 | title = {{All Politics Is Local: The Reemergence of the Study of City Politics}}, 3 | volume = {7}, 4 | doi = {10.1017/S1537592709990892}, 5 | url = {https://doi.org/10.1017/S1537592709990892}, 6 | number = {3}, 7 | journal = {Perspectives on Politics}, 8 | publisher = {Cambridge University Press}, 9 | author = {Trounstine, Jessica}, 10 | year = {2009}, 11 | pages = {611–618} 12 | } 13 | 14 | @misc{cookiecutter, 15 | author = {Audrey Roy Greenfeld and Daniel Roy Greenfeld and Raphael Pierzina}, 16 | title = {{cookiecutter}}, 17 | abstract = {A command-line utility that creates projects from cookiecutters (project templates), e.g. creating a Python package project from a Python package project template.}, 18 | year = {2015}, 19 | publisher = {GitHub}, 20 | journal = {GitHub repository}, 21 | url = {https://github.com/cookiecutter/cookiecutter} 22 | } 23 | 24 | @article{Sparks2017, 25 | doi = {10.21105/joss.00411}, 26 | url = {https://doi.org/10.21105/joss.00411}, 27 | year = {2017}, 28 | publisher = {The Open Journal}, 29 | volume = {2}, 30 | number = {17}, 31 | pages = {411}, 32 | author = {Adam H. Sparks and Mark Padgham and Hugh Parsonage and Keith Pembleton}, 33 | title = {{bomrang: Fetch Australian Government Bureau of Meteorology Data in R}}, 34 | journal = {Journal of Open Source Software} 35 | } 36 | 37 | @misc{courtlistener, 38 | author = {{The Free Law Project}}, 39 | title = {{CourtListener}}, 40 | abstract = {CourtListener is a free legal research website containing millions of legal opinions from federal and state courts. With CourtListener, lawyers, journalists, academics, and the public can research an important case, stay up to date with new opinions as they are filed, or do deep analysis using our raw data.}, 41 | year = {2015}, 42 | publisher = {GitHub}, 43 | journal = {GitHub repository}, 44 | url = {https://github.com/freelawproject/courtlistener} 45 | } 46 | 47 | @misc{councilmatic, 48 | author = {Mjumbe Poe and Forest Gregg}, 49 | title = {{Councilmatic}}, 50 | abstract = {A subscription service for city council legislative information, started in Philadelphia.}, 51 | year = {2015}, 52 | publisher = {GitHub}, 53 | journal = {GitHub repository}, 54 | url = {https://github.com/codeforamerica/councilmatic} 55 | } 56 | 57 | @article{iconf2018, 58 | title = {Remediating Civic Tech}, 59 | author = {Weber, Nic and Brown, Eva}, 60 | journal = {iConference 2018 Proceedings}, 61 | year = {2018}, 62 | publisher = {iSchools} 63 | } 64 | 65 | @article{jacobi2017, 66 | title = {{Justice, interrupted: The effect of gender, ideology, and seniority at Supreme Court oral arguments}}, 67 | author = {Jacobi, Tonja and Schweers, Dylan}, 68 | journal = {Va. L. Rev.}, 69 | volume = {103}, 70 | pages = {1379}, 71 | year = {2017}, 72 | publisher = {HeinOnline}, 73 | url = {https://www.virginialawreview.org/wp-content/uploads/2020/12/JacobiSchweers_Online.pdf} 74 | } 75 | 76 | @article{einstein2021, 77 | title = {{Zoom Does Not Reduce Unequal Participation: Evidence from Public Meeting Minutes}}, 78 | author = {Einstein, Katherine Levine and Glick, David and Puig, Luisa Godinez and Palmer, Maxwell}, 79 | publisher = {Housing Politics Lab}, 80 | year = {2021}, 81 | url = {https://www.housingpolitics.com/research/online_meetings_participation.pdf} 82 | } 83 | -------------------------------------------------------------------------------- /paper/paper.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Council Data Project: Software for Municipal Data Collection, Analysis, and Publication" 3 | tags: 4 | - Python 5 | - JavaScript 6 | - open government 7 | - open data 8 | - open infrastructure 9 | - municipal governance 10 | - data archival 11 | - civic technology 12 | - natural language processing 13 | authors: 14 | - name: Eva Maxfield Brown 15 | orcid: 0000-0003-2564-0373 16 | affiliation: 1 17 | - name: To Huynh 18 | orcid: 0000-0002-9664-3662 19 | affiliation: 2 20 | - name: Isaac Na 21 | orcid: 0000-0002-0182-1615 22 | affiliation: 3 23 | - name: Brian Ledbetter 24 | affiliation: 1 25 | - name: Hawk Ticehurst 26 | affiliation: 1 27 | - name: Sarah Liu 28 | affiliation: 4 29 | - name: Emily Gilles 30 | affiliation: 4 31 | - name: Katlyn M. F. Greene 32 | affiliation: 4 33 | - name: Sung Cho 34 | affiliation: 4 35 | - name: Shak Ragoler 36 | affiliation: 4 37 | - name: Nicholas Weber 38 | orcid: 0000-0002-6008-3763 39 | affiliation: 1 40 | 41 | affiliations: 42 | - name: University of Washington Information School, University of Washington, Seattle 43 | index: 1 44 | - name: University of Washington, Seattle 45 | index: 2 46 | - name: Washington University, St. Louis 47 | index: 3 48 | - name: Independent Contributor 49 | index: 4 50 | 51 | date: 29 October 2021 52 | bibliography: paper.bib 53 | --- 54 | 55 | # Summary 56 | 57 | Cities, counties, and states throughout the USA are bound by law to archive recordings of public meetings. Most local governments comply with these laws by posting documents, audio, or video recordings online. As there is no set standard for municipal data archives however, parsing and processing such data is typically time consuming and highly dependent on each municipality. Council Data Project (CDP) is a set of open-source tools that improve the accessibility of local government data by systematically collecting, transforming, and re-publishing this data to the web. The data re-published by CDP is packaged and presented within a searchable web application that vastly simplifies the process of finding specific information within the archived data. We envision this project being used by a variety of groups including civic technologists hoping to promote government transparency, researchers focused on public policy, natural language processing, machine learning, or information retrieval and discovery, and many others. 58 | 59 | # Statement of Need 60 | 61 | Comparative research into municipal governance in the USA is often prohibitively difficult due to a broad federal system where states, counties, and cities divide legislative powers differently. This has contributed to the lack of large-scale quantitative studies of municipal government, and impeded necessary research into effective procedural elements of administrative and legislative processes [@Trounstine2009]. Council Data Project enables large-scale quantitative studies by generating standardized municipal governance corpora - including legislative voting records, timestamped transcripts, and full legislative matter attachments (related reports, presentations, amendments, etc.). 62 | 63 | ## Related Work 64 | 65 | Work in extracting and repackaging government data into machine-readable and experiment ready datasets has historically happened in fields with highly structured data, such as meteorology [@Sparks2017] and legal review and monitoring [@courtlistener]. Notably, there has been prior work in extracting and repackaging municipal government data with [Councilmatic](https://github.com/codeforamerica/councilmatic) [@councilmatic]. However, this work largely aims to make municipal data more accessible to a general public, and does not add any specific data processing to expand the research capabilities of the produced dataset. Recent advances in natural language processing have made it possible to conduct large-scale transcript-based studies on the effects of gender, ideology, and seniority in Supreme Court oral argument [@jacobi2017] and the effects that information communication technology has on civic participation [@einstein2021]. 66 | 67 | ## CDP Architecture 68 | 69 | Council Data Project consists of three primary tools: 70 | 71 | 1. [cookiecutter-cdp-deployment](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment): A Python [cookiecutter](https://cookiecutter.readthedocs.io/) [@cookiecutter] template to assist users in fully deploying a new CDP instance. A "CDP Instance" is a unique deployment of CDP software and tools. For example, there is an "instance" of CDP for the ["Seattle City Council"](https://councildataproject.org/seattle/#/) and an instance of CDP for the "King County Council". Each instance is comprised of its own repository, database, file storage bucket, processing pipelines, and web application. 72 | 73 | 2. [cdp-backend](https://github.com/CouncilDataProject/cdp-backend): A Python package containing CDP's database schema definition, a file format for transcripts generated by speech-to-text algorithms, an infrastructure specification, and processing pipelines. This package currently contains an event gather and processing workflow that will parse event details, generate a transcript for the event using either the provided closed caption file, or using Google Speech-to-Text from the provided event video, and finally, generate and store event metadata (voting records, thumbnails, minutes items, etc.) This package additionally provides a workflow for generating a TF-IDF based event index for weighted term search. The processing workflows and all utilities and schemas are separate from any one CDP instance so that all CDP instances can be easily upgraded whenever there is a new version of `cdp-backend` released. 74 | 75 | 3. [cdp-frontend](https://github.com/CouncilDataProject/cdp-frontend): A TypeScript and React-based component library and web application. The web application allows for simple data exploration and sharing, and as such, acts as a method to interactively explore the data produced by the backend pipelines. The web application and the component library are separate from any single CDP instance so that all CDP instances can be easily upgraded whenever there is a new version of `cdp-frontend` released. 76 | 77 | ## Cookiecutter and the Produced Repository 78 | 79 | `cookiecutter-cdp-deployment` will generate all necessary files for an entirely new CDP instance as well as additional setup documentation for the user to follow to fully complete the instance deployment process. 80 | 81 | Utilizing [GitHub Actions](https://github.com/features/actions) and [GitHub Pages](https://pages.github.com/), data processing and web hosting are entirely free as long as the user sets their instance's GitHub repository visibility to public. 82 | 83 | Deploying a CDP instance incurs some small primary costs by using: 84 | 85 | 1. [Google Speech-to-Text](https://cloud.google.com/speech-to-text/) for transcript generation. 86 | 2. [Firebase Cloud Firestore](https://firebase.google.com/docs/firestore/) for event metadata storage and access. 87 | 3. [Firebase Storage](https://firebase.google.com/docs/storage) for file storage and access. 88 | 89 | ![CDP Core Infrastructure and Pipelines. A CDP instance's event gather and processing pipeline can be triggered by providing the GitHub Action a custom datetime range to process, providing a pre-constructed JSON object (useful for special events like debates and such), or the pipeline will automatically run every 6 hours. Once initiated, the event gather pipeline will get the events for the provided or default date range, create a transcript and extra metadata objects (thumbnails, and more), then will finally archive the event to Firebase. A CDP instance's event indexing pipeline can be triggered by running the pipeline manually or will automatically run every two days.Once initiated, the event index pipeline will in parallel, generate and store unigrams, bigrams, and trigrams from all event transcripts and store them to Firebase for query. Finally, the web application is built from a manual trigger or once a week and simply runs a standard NPM build process then publishes the generated web application to GitHub Pages. Once built and published, the deployment website fetches data from Firebase for search and web access.\label{fig:core-infra}](./assets/cdp_core_infrastructure.png) 90 | 91 | CDP tools allow for decentralized control over the management and deployment of each CDP instance while producing a standardized open-access dataset for both research and for municipal transparency and accessibility. 92 | 93 | ## Data Access 94 | 95 | ### Web 96 | 97 | Once data is processed by a CDP instance, it is available through that instance's interactive web application. 98 | 99 | ![CDP Web Application. Screenshot of a single event's page. Navigation tabs for basic event details such as the minutes items, the entire transcript, and voting information. Additionally both the transcript search and the full transcript have links to jump to a specific sentence in the meeting. This example event page can be found on our Seattle City Council "staging" instance: http://councildataproject.org/seattle-staging/#/events/0ec08c565d45 \label{fig:event-page}](./assets/event-page-screenshot.png) 100 | 101 | ### Python 102 | 103 | For users who want programmatic access, each instance's repository README includes a programmatic quickstart guide and our database schema is automatically generated and stored in our `cdp-backend` [documentation](https://councildataproject.org/cdp-backend/database_schema.html). 104 | 105 | ```python 106 | from cdp_backend.database import models as db_models 107 | from cdp_backend.pipeline.transcript_model import Transcript 108 | import fireo 109 | from gcsfs import GCSFileSystem 110 | from google.auth.credentials import AnonymousCredentials 111 | from google.cloud.firestore import Client 112 | 113 | # Connect to the database 114 | fireo.connection(client=Client( 115 | project="cdp-test-deployment-435b5309", 116 | credentials=AnonymousCredentials() 117 | )) 118 | 119 | # Read from the database 120 | five_people = list(db_models.Person.collection.fetch(5)) 121 | 122 | # Connect to the file store 123 | fs = GCSFileSystem(project="cdp-test-deployment-435b5309", token="anon") 124 | 125 | # Read a transcript's details from the database 126 | transcript_model = list(db_models.Transcript.collection.fetch(1))[0] 127 | 128 | # Read the transcript directly from the file store 129 | with fs.open(transcript_model.file_ref.get().uri, "r") as open_resource: 130 | transcript = Transcript.from_json(open_resource.read()) 131 | 132 | # OR download and store the transcript locally with `get` 133 | fs.get(transcript_model.file_ref.get().uri, "local-transcript.json") 134 | # Then read the transcript from your local machine 135 | with open("local-transcript.json", "r") as open_resource: 136 | transcript = Transcript.from_json(open_resource.read()) 137 | ``` 138 | 139 | # Acknowledgements 140 | 141 | We wish to thank the many volunteers that have contributed code, design, conversation, and ideas to the project. We wish to thank DemocracyLab and Open Seattle for helping build a civic technology community. From DemocracyLab, we would specifically like to thank Mark Frischmuth for the continued support and helpful discussions. We wish to thank the University of Washington Information School for support. We wish to thank Code for Science and Society and the Digital Infrastructure Incubator for providing guidance on developing a sustainable open source project. 142 | 143 | # References 144 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = 3 | docs/ 4 | ignore = 5 | E203 6 | E402 7 | W291 8 | W503 9 | max-line-length = 88 10 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.yml] 14 | indent_size = 2 15 | 16 | [*.bat] 17 | indent_style = tab 18 | end_of_line = crlf 19 | 20 | [LICENSE] 21 | insert_final_newline = false 22 | 23 | [Makefile] 24 | indent_style = tab 25 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "pip" 4 | directory: "/python" 5 | schedule: 6 | interval: "daily" 7 | allow: 8 | - dependency-name: "cdp-backend" 9 | commit-message: 10 | prefix: "ci(dependabot):" 11 | 12 | - package-ecosystem: "pip" 13 | directory: "/infra" 14 | schedule: 15 | interval: "daily" 16 | allow: 17 | - dependency-name: "cdp-backend" 18 | commit-message: 19 | prefix: "ci(dependabot):" 20 | 21 | - package-ecosystem: "npm" 22 | directory: "/web" 23 | schedule: 24 | interval: "daily" 25 | allow: 26 | - dependency-name: "@councildataproject/cdp-frontend" 27 | commit-message: 28 | prefix: "ci(dependabot):" 29 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/build-main.yml: -------------------------------------------------------------------------------- 1 | name: Build Main 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | # 9 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 10 | # Run every Monday at 23:26:00 UTC (Monday at 15:26:00 PST) 11 | # We offset from the hour and half hour to go easy on the servers :) 12 | - cron: '26 23 * * 1' 13 | 14 | jobs: 15 | build-python: 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: actions/setup-python@v1 21 | with: 22 | python-version: '3.11' 23 | 24 | - name: Install Packages 25 | run: | 26 | sudo apt update 27 | sudo apt-get install ffmpeg --fix-missing 28 | 29 | - name: Install Python Dependencies 30 | run: | 31 | cd python/ 32 | pip install .[test] 33 | - name: Lint and Format Python 34 | run: | 35 | cd python/ 36 | flake8 cdp_{{ cookiecutter.python_municipality_slug }}_backend --count --verbose --show-source --statistics 37 | black --check cdp_{{ cookiecutter.python_municipality_slug }}_backend 38 | 39 | build-web: 40 | runs-on: ubuntu-latest 41 | 42 | steps: 43 | - uses: actions/checkout@v4 44 | - uses: actions/setup-node@v1 45 | with: 46 | node-version: '16.x' 47 | 48 | - name: Install Web App Dependencies 49 | run: | 50 | cd web/ 51 | npm i 52 | - name: Build Web App 53 | run: | 54 | cd web/ 55 | npm run build 56 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/check-pr.yml: -------------------------------------------------------------------------------- 1 | name: Check Pull Request 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build-python: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: actions/setup-python@v1 15 | with: 16 | python-version: '3.11' 17 | 18 | - name: Install Packages 19 | run: | 20 | sudo apt update 21 | sudo apt-get install ffmpeg --fix-missing 22 | 23 | - name: Install Python Dependencies 24 | run: | 25 | cd python/ 26 | pip install .[test] 27 | - name: Lint and Format Python 28 | run: | 29 | cd python/ 30 | flake8 cdp_{{ cookiecutter.python_municipality_slug }}_backend --count --verbose --show-source --statistics 31 | black --check cdp_{{ cookiecutter.python_municipality_slug }}_backend 32 | 33 | build-web: 34 | runs-on: ubuntu-latest 35 | 36 | steps: 37 | - uses: actions/checkout@v4 38 | - uses: actions/setup-node@v1 39 | with: 40 | node-version: '16.x' 41 | 42 | - name: Install Web App Dependencies 43 | run: | 44 | cd web/ 45 | npm i 46 | - name: Build Web App 47 | run: | 48 | cd web/ 49 | npm run build 50 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/deploy-infra.yml: -------------------------------------------------------------------------------- 1 | name: Infrastructure 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | # 9 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 10 | # Run every Monday at 23:26:00 UTC (Monday at 15:26:00 PST) 11 | # We offset from the hour and half hour to go easy on the servers :) 12 | - cron: '26 23 * * 1' 13 | 14 | jobs: 15 | deploy-infra: 16 | runs-on: ubuntu-latest 17 | permissions: 18 | contents: 'read' 19 | id-token: 'write' 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | # Install OS Deps 25 | - uses: actions/setup-python@v1 26 | with: 27 | python-version: '3.11' 28 | - uses: extractions/setup-just@v1 29 | env: 30 | GITHUB_TOKEN: {% raw %}${{ secrets.GITHUB_TOKEN }}{% endraw %} 31 | - name: Install firebase-cli 32 | run: | 33 | curl -sL https://firebase.tools | bash 34 | 35 | # Install cdp-backend 36 | - name: Install Dependencies 37 | run: | 38 | pip install -r infra/requirements.txt 39 | 40 | # Setup gcloud 41 | - uses: 'google-github-actions/auth@v2' 42 | with: 43 | credentials_json: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 44 | - name: Set up Cloud SDK 45 | uses: google-github-actions/setup-gcloud@v0 46 | 47 | # Generate / copy infrastructure files 48 | - name: Copy and Generate Infrastructure Files 49 | run: | 50 | get_cdp_infrastructure_stack infrastructure 51 | 52 | # Run infrastructure deploy 53 | - name: Run Infrastructure Deploy 54 | run: | 55 | cd infrastructure 56 | echo "$GOOGLE_CREDENTIALS" > google-creds.json 57 | export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/google-creds.json 58 | just deploy \ 59 | {{ cookiecutter.infrastructure_slug }} \ 60 | $(pwd)/../cookiecutter.yaml 61 | env: 62 | FIREBASE_TOKEN: {% raw %}${{ secrets.FIREBASE_TOKEN }}{% endraw %} 63 | GOOGLE_CREDENTIALS: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 64 | 65 | # Clipping Fill -- 66 | # {%- if cookiecutter.enable_clipping == "true" -%} 67 | 68 | # Enabled 69 | # - name: Run Clipping Deploy 70 | # run: | 71 | # cd infrastructure 72 | # echo "$GOOGLE_CREDENTIALS" > google-creds.json 73 | # export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/google-creds.json 74 | # just deploy-clipping \ 75 | # $(pwd)/google-creds.json \ 76 | # {{ cookiecutter.firestore_region }} 77 | # env: 78 | # FIREBASE_TOKEN: {% raw %}${{ secrets.FIREBASE_TOKEN }}{% endraw %} 79 | # GOOGLE_CREDENTIALS: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 80 | 81 | # {%- elif cookiecutter.enable_clipping == "false" -%} 82 | 83 | # Disabled 84 | # - name: Run Clipping Deploy 85 | # run: | 86 | # echo "Will not deploy clipping" 87 | 88 | # {% endif %} 89 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/deploy-web.yml: -------------------------------------------------------------------------------- 1 | name: Web App 2 | 3 | permissions: 4 | contents: write 5 | 6 | on: 7 | push: 8 | branches: 9 | - main 10 | schedule: 11 | # 12 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 13 | # Run every Monday at 23:26:00 UTC (Monday at 15:26:00 PST) 14 | # We offset from the hour and half hour to go easy on the servers :) 15 | - cron: '26 23 * * 1' 16 | 17 | jobs: 18 | deploy-web: 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | 24 | # Setup Node 25 | - name: Setup Node 26 | uses: actions/setup-node@v1 27 | with: 28 | node-version: '16.x' 29 | 30 | # Build Web 31 | - name: Install Web App Dependencies 32 | run: | 33 | cd web/ 34 | npm i 35 | - name: Build Web App 36 | run: | 37 | cd web/ 38 | npm run build 39 | 40 | # Deploy Web 41 | - name: Publish Docs 42 | uses: JamesIves/github-pages-deploy-action@v4 43 | with: 44 | folder: web/build/ -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/event-gather-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: Event Gather 2 | 3 | on: 4 | schedule: 5 | # 6 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 7 | - cron: '{{ cookiecutter.event_gather_cron }}' 8 | workflow_dispatch: 9 | inputs: 10 | from: 11 | description: "Optional ISO formatted string for datetime to begin event gather from." 12 | required: false 13 | to: 14 | description: "Optional ISO formatted string for datetime to end event gather at." 15 | required: false 16 | 17 | permissions: 18 | id-token: write 19 | contents: write 20 | pull-requests: write 21 | 22 | jobs: 23 | deploy-runner-on-gcp: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v4 27 | - name: Setup CML 28 | uses: iterative/setup-cml@v1 29 | - name: Create Runner 30 | uses: nick-fields/retry@v2 31 | env: 32 | REPO_TOKEN: {% raw %}${{ secrets.PERSONAL_ACCESS_TOKEN }}{% endraw %} 33 | GOOGLE_APPLICATION_CREDENTIALS_DATA: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 34 | with: 35 | timeout_minutes: {{ cookiecutter.event_gather_runner_timeout_minutes }} 36 | max_attempts: {{ cookiecutter.event_gather_runner_max_attempts }} 37 | retry_wait_seconds: {{ cookiecutter.event_gather_runner_retry_wait_seconds }} 38 | command: >- 39 | cml runner \ 40 | --single \ 41 | --labels=gcp-cdp-runner \ 42 | --cloud=gcp \ 43 | --cloud-region=us-west1-b \ 44 | --cloud-type=n1-standard-4 \ 45 | --cloud-gpu=nvidia-tesla-t4 \ 46 | --cloud-hdd-size=30 \ 47 | --idle-timeout=600 48 | 49 | process-events: 50 | needs: [deploy-runner-on-gcp] 51 | runs-on: [self-hosted, gcp-cdp-runner] 52 | container: 53 | image: ghcr.io/iterative/cml:0-dvc2-base1-gpu 54 | options: --gpus all 55 | 56 | steps: 57 | - uses: actions/checkout@v4 58 | - uses: actions/setup-python@v4 59 | with: 60 | python-version: '3.11' 61 | 62 | - name: Check GPU Drivers 63 | run: | 64 | nvidia-smi 65 | 66 | - name: Install Packages 67 | run: | 68 | sudo apt update 69 | sudo apt-get install -y --no-install-recommends \ 70 | libsndfile1 \ 71 | ffmpeg 72 | 73 | - name: Install Python Dependencies 74 | run: | 75 | cd python/ 76 | pip install --upgrade pip 77 | pip install . 78 | 79 | - name: Setup gcloud 80 | uses: google-github-actions/setup-gcloud@v0 81 | with: 82 | project_id: {{ cookiecutter.infrastructure_slug }} 83 | service_account_key: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 84 | export_default_credentials: true 85 | 86 | - name: Dump Credentials to JSON 87 | uses: jsdaniell/create-json@v1.2.2 88 | with: 89 | name: "google-creds.json" 90 | json: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 91 | dir: "python/" 92 | 93 | - name: Gather and Process New Events - CRON 94 | if: {% raw %}${{ github.event_name == 'schedule' }}{% endraw %} 95 | run: | 96 | cd python/ 97 | run_cdp_event_gather event-gather-config.json 98 | 99 | - name: Gather and Process Requested Events - Manual 100 | if: {% raw %}${{ github.event_name == 'workflow_dispatch' }}{% endraw %} 101 | run: | 102 | cd python/ 103 | CDP_FROM_USER={% raw %}${{ github.event.inputs.from }}{% endraw %} 104 | CDP_FROM_DEFAULT=$(date -Iseconds -d "2 days ago") 105 | CDP_FROM=${CDP_FROM_USER:-$CDP_FROM_DEFAULT} 106 | CDP_TO_USER={% raw %}${{ github.event.inputs.to }}{% endraw %} 107 | CDP_TO_DEFAULT=$(date -Iseconds) 108 | CDP_TO=${CDP_TO_USER:-$CDP_TO_DEFAULT} 109 | run_cdp_event_gather event-gather-config.json \ 110 | --from $CDP_FROM \ 111 | --to $CDP_TO 112 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/event-index-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: Event Index 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | schedule: 7 | # 8 | # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07 9 | # Run every Thursday at 3:26:00 UTC 10 | # (Thursday at 19:26:00 PST) 11 | # We offset from the hour and half hour to go easy on the servers :) 12 | - cron: '26 3 * * 4' 13 | 14 | # We doubly fan out 15 | # We first generate indexs for uni, bi, and trigrams with a matrix 16 | # Each index is split into chunks of 50,000 grams 17 | # Then we fan out by every chunk and upload 18 | {% raw %} 19 | jobs: 20 | generate-index-chunks: 21 | runs-on: ubuntu-latest 22 | strategy: 23 | matrix: 24 | n-gram: [1, 2, 3] 25 | fail-fast: false 26 | 27 | outputs: 28 | ngram-1-chunks: ${{ steps.output-index-chunks.outputs.ngram-1-chunks }} 29 | ngram-2-chunks: ${{ steps.output-index-chunks.outputs.ngram-2-chunks }} 30 | ngram-3-chunks: ${{ steps.output-index-chunks.outputs.ngram-3-chunks }} 31 | 32 | steps: 33 | # Setup Runner 34 | - uses: actions/checkout@v4 35 | - uses: actions/setup-python@v1 36 | with: 37 | python-version: '3.11' 38 | 39 | # Setup GCloud / Creds 40 | - name: Setup gcloud 41 | uses: google-github-actions/setup-gcloud@v0 42 | with: 43 | project_id: {% endraw %}{{ cookiecutter.infrastructure_slug }}{% raw %} 44 | service_account_key: ${{ secrets.GOOGLE_CREDENTIALS }} 45 | export_default_credentials: true 46 | 47 | - name: Dump Credentials to JSON 48 | uses: jsdaniell/create-json@v1.2.2 49 | with: 50 | name: "google-creds.json" 51 | json: ${{ secrets.GOOGLE_CREDENTIALS }} 52 | dir: "python/" 53 | 54 | # Installs 55 | - name: Install Python Dependencies 56 | run: | 57 | cd python/ 58 | pip install . 59 | 60 | # Index 61 | - name: Index Events ${{ matrix.n-gram }}-grams 62 | run: | 63 | cd python/ 64 | run_cdp_event_index_generation event-index-config.json \ 65 | --n_grams ${{ matrix.n-gram }} \ 66 | --store_remote \ 67 | --parallel 68 | 69 | # Store generated files to step output 70 | - name: Store Index Fileset to Outputs 71 | id: output-index-chunks 72 | run: | 73 | cd python/index/ 74 | output=$(python -c 'import os, json; print(json.dumps(os.listdir(".")))') 75 | echo "::set-output name=ngram-${{ matrix.n-gram }}-chunks::$output" 76 | 77 | combine-matrix-ngram-chunks: 78 | needs: generate-index-chunks 79 | runs-on: ubuntu-latest 80 | outputs: 81 | all-chunks: ${{ steps.combine-index-chunks.outputs.combined-chunks }} 82 | 83 | steps: 84 | # Setup Runner 85 | - uses: actions/checkout@v4 86 | - uses: actions/setup-python@v1 87 | with: 88 | python-version: '3.11' 89 | 90 | # Process 91 | - name: Combine Chunks 92 | id: 'combine-index-chunks' 93 | run: | 94 | echo 'print(${{ needs.generate-index-chunks.outputs.ngram-1-chunks }} + ${{ needs.generate-index-chunks.outputs.ngram-2-chunks }} + ${{ needs.generate-index-chunks.outputs.ngram-3-chunks }})' >> print-combined-chunks.py 95 | output=$(python print-combined-chunks.py) 96 | echo "::set-output name=combined-chunks::$output" 97 | 98 | upload-index-chunks: 99 | needs: combine-matrix-ngram-chunks 100 | runs-on: ubuntu-latest 101 | strategy: 102 | max-parallel: 6 103 | matrix: 104 | filename: ${{ fromJson(needs.combine-matrix-ngram-chunks.outputs.all-chunks) }} 105 | fail-fast: false 106 | 107 | steps: 108 | # Setup Runner 109 | - uses: actions/checkout@v4 110 | - uses: actions/setup-python@v1 111 | with: 112 | python-version: '3.11' 113 | 114 | # Setup GCloud / Creds 115 | - name: Setup gcloud 116 | uses: google-github-actions/setup-gcloud@v0 117 | with: 118 | project_id: {% endraw %}{{ cookiecutter.infrastructure_slug }}{% raw %} 119 | service_account_key: ${{ secrets.GOOGLE_CREDENTIALS }} 120 | export_default_credentials: true 121 | - name: Dump Credentials to JSON 122 | uses: jsdaniell/create-json@v1.2.2 123 | with: 124 | name: "google-creds.json" 125 | json: ${{ secrets.GOOGLE_CREDENTIALS }} 126 | dir: "python/" 127 | 128 | # Installs 129 | - name: Install Python Dependencies 130 | run: | 131 | cd python/ 132 | pip install . 133 | 134 | # Upload Index Chunk 135 | - name: Process Upload 136 | run: | 137 | cd python/ 138 | process_cdp_event_index_chunk event-index-config.json \ 139 | ${{ matrix.filename }} \ 140 | --parallel 141 | {% endraw %} -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/process-special-event.yml: -------------------------------------------------------------------------------- 1 | name: Process Special Event 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | event-details-json-string: 7 | description: "Event details (see EventIngestionModel in cdp-backend) in JSON string form. The string must be free of any newline or tab characters." 8 | required: true 9 | 10 | permissions: 11 | id-token: write 12 | contents: write 13 | pull-requests: write 14 | 15 | jobs: 16 | deploy-runner-on-gcp: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | - name: Setup CML 21 | uses: iterative/setup-cml@v1 22 | - name: Create Runner 23 | env: 24 | REPO_TOKEN: {% raw %}${{ secrets.PERSONAL_ACCESS_TOKEN }}{% endraw %} 25 | GOOGLE_APPLICATION_CREDENTIALS_DATA: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 26 | run: | 27 | cml runner \ 28 | --single \ 29 | --labels=gcp-cdp-runner \ 30 | --cloud=gcp \ 31 | --cloud-region=us-west1-b \ 32 | --cloud-type=n1-standard-4 \ 33 | --cloud-gpu=nvidia-tesla-t4 \ 34 | --cloud-hdd-size=30 \ 35 | --idle-timeout=600 36 | 37 | process-events: 38 | needs: [deploy-runner-on-gcp] 39 | runs-on: [self-hosted, gcp-cdp-runner] 40 | container: 41 | image: ghcr.io/iterative/cml:0-dvc2-base1-gpu 42 | options: --gpus all 43 | 44 | steps: 45 | - uses: actions/checkout@v4 46 | - uses: actions/setup-python@v4 47 | with: 48 | python-version: '3.11' 49 | 50 | - name: Check GPU Drivers 51 | run: | 52 | nvidia-smi 53 | 54 | - name: Install Packages 55 | run: | 56 | sudo apt update 57 | sudo apt-get install -y --no-install-recommends \ 58 | libsndfile1 \ 59 | ffmpeg 60 | 61 | - name: Install Python Dependencies 62 | run: | 63 | cd python/ 64 | pip install --upgrade pip 65 | pip install . 66 | 67 | - name: Setup gcloud 68 | uses: google-github-actions/setup-gcloud@v0 69 | with: 70 | project_id: {{ cookiecutter.infrastructure_slug }} 71 | service_account_key: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 72 | export_default_credentials: true 73 | 74 | - name: Dump Credentials to JSON 75 | uses: jsdaniell/create-json@v1.2.2 76 | with: 77 | name: "google-creds.json" 78 | json: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 79 | dir: "python/" 80 | 81 | - name: Dump Event Details to JSON 82 | run: | 83 | echo {% raw %}${{ github.event.inputs.event-details-json-string }}{% endraw %} > python/event-details.json 84 | 85 | - name: Process special events into event gather pipeline 86 | if: {% raw %}${{ github.event_name == 'workflow_dispatch' }}{% endraw %} 87 | run: | 88 | cd python/ 89 | {% raw %}process_special_event \ 90 | --event_details_file event-details.json \ 91 | --event_gather_config_file event-gather-config.json{% endraw %} 92 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.github/workflows/run-script.yml: -------------------------------------------------------------------------------- 1 | name: Run Command 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | workflow_dispatch: 8 | inputs: 9 | command-and-args: 10 | description: "A command and all arguments to passthrough to the runner." 11 | required: true 12 | 13 | jobs: 14 | run-command: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: actions/setup-python@v1 20 | with: 21 | python-version: '3.11' 22 | 23 | - name: Install Packages 24 | run: | 25 | sudo apt update 26 | sudo apt-get install ffmpeg --fix-missing 27 | 28 | - name: Install Python Dependencies 29 | run: | 30 | cd python/ 31 | pip install . 32 | 33 | - name: Setup gcloud 34 | uses: google-github-actions/setup-gcloud@v0 35 | with: 36 | project_id: {{ cookiecutter.infrastructure_slug }} 37 | service_account_key: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 38 | export_default_credentials: true 39 | 40 | - name: Dump Credentials to JSON 41 | uses: jsdaniell/create-json@v1.2.2 42 | with: 43 | name: "google-creds.json" 44 | json: {% raw %}${{ secrets.GOOGLE_CREDENTIALS }}{% endraw %} 45 | dir: "python/" 46 | 47 | - name: Run Command 48 | if: {% raw %}${{ github.event_name == 'workflow_dispatch' }}{% endraw %} 49 | run: | 50 | cd python/ 51 | {% raw %}${{ github.event.inputs.command-and-args }}{% endraw %} 52 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # OS generated files 29 | .DS_Store 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # PyBuilder 69 | target/ 70 | 71 | # Jupyter Notebook 72 | .ipynb_checkpoints 73 | 74 | # pyenv 75 | .python-version 76 | 77 | # celery beat schedule file 78 | celerybeat-schedule 79 | 80 | # Dask 81 | dask-worker-space 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # dotenv 87 | .env 88 | 89 | # virtualenv 90 | .venv 91 | venv/ 92 | ENV/ 93 | 94 | # Spyder project settings 95 | .spyderproject 96 | .spyproject 97 | 98 | # Rope project settings 99 | .ropeproject 100 | 101 | # mkdocs documentation 102 | /site 103 | 104 | # mypy 105 | .mypy_cache/ 106 | 107 | # VSCode 108 | .vscode 109 | 110 | # Secrets 111 | .keys/ -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting any of the maintainers of this project and 59 | we will attempt to resolve the issues with respect and dignity. 60 | 61 | Project maintainers who do not follow or enforce the Code of Conduct in good 62 | faith may face temporary or permanent repercussions as determined by other 63 | members of the project's leadership. 64 | 65 | ## Attribution 66 | 67 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 68 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 69 | 70 | [homepage]: https://www.contributor-covenant.org 71 | 72 | For answers to common questions about this code of conduct, see 73 | https://www.contributor-covenant.org/faq 74 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/Justfile: -------------------------------------------------------------------------------- 1 | # list all available commands 2 | default: 3 | just --list 4 | 5 | # update this repo using latest cookiecutter-py-package 6 | update-from-cookiecutter: 7 | pip install cookiecutter 8 | cookiecutter gh:CouncilDataProject/cookiecutter-cdp-deployment \ 9 | --config-file cookiecutter.yaml \ 10 | --no-input \ 11 | --overwrite-if-exists \ 12 | --output-dir .. -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Council Data Project, {{ cookiecutter.maintainer_or_org_full_name }} 2 | 3 | Mozilla Public License, version 2.0 4 | 5 | 1. Definitions 6 | 7 | 1.1. “Contributor” 8 | 9 | means each individual or legal entity that creates, contributes to the 10 | creation of, or owns Covered Software. 11 | 12 | 1.2. “Contributor Version” 13 | 14 | means the combination of the Contributions of others (if any) used by a 15 | Contributor and that particular Contributor’s Contribution. 16 | 17 | 1.3. “Contribution” 18 | 19 | means Covered Software of a particular Contributor. 20 | 21 | 1.4. “Covered Software” 22 | 23 | means Source Code Form to which the initial Contributor has attached the 24 | notice in Exhibit A, the Executable Form of such Source Code Form, and 25 | Modifications of such Source Code Form, in each case including portions 26 | thereof. 27 | 28 | 1.5. “Incompatible With Secondary Licenses” 29 | means 30 | 31 | a. that the initial Contributor has attached the notice described in 32 | Exhibit B to the Covered Software; or 33 | 34 | b. that the Covered Software was made available under the terms of version 35 | 1.1 or earlier of the License, but not also under the terms of a 36 | Secondary License. 37 | 38 | 1.6. “Executable Form” 39 | 40 | means any form of the work other than Source Code Form. 41 | 42 | 1.7. “Larger Work” 43 | 44 | means a work that combines Covered Software with other material, in a separate 45 | file or files, that is not Covered Software. 46 | 47 | 1.8. “License” 48 | 49 | means this document. 50 | 51 | 1.9. “Licensable” 52 | 53 | means having the right to grant, to the maximum extent possible, whether at the 54 | time of the initial grant or subsequently, any and all of the rights conveyed by 55 | this License. 56 | 57 | 1.10. “Modifications” 58 | 59 | means any of the following: 60 | 61 | a. any file in Source Code Form that results from an addition to, deletion 62 | from, or modification of the contents of Covered Software; or 63 | 64 | b. any new file in Source Code Form that contains any Covered Software. 65 | 66 | 1.11. “Patent Claims” of a Contributor 67 | 68 | means any patent claim(s), including without limitation, method, process, 69 | and apparatus claims, in any patent Licensable by such Contributor that 70 | would be infringed, but for the grant of the License, by the making, 71 | using, selling, offering for sale, having made, import, or transfer of 72 | either its Contributions or its Contributor Version. 73 | 74 | 1.12. “Secondary License” 75 | 76 | means either the GNU General Public License, Version 2.0, the GNU Lesser 77 | General Public License, Version 2.1, the GNU Affero General Public 78 | License, Version 3.0, or any later versions of those licenses. 79 | 80 | 1.13. “Source Code Form” 81 | 82 | means the form of the work preferred for making modifications. 83 | 84 | 1.14. “You” (or “Your”) 85 | 86 | means an individual or a legal entity exercising rights under this 87 | License. For legal entities, “You” includes any entity that controls, is 88 | controlled by, or is under common control with You. For purposes of this 89 | definition, “control” means (a) the power, direct or indirect, to cause 90 | the direction or management of such entity, whether by contract or 91 | otherwise, or (b) ownership of more than fifty percent (50%) of the 92 | outstanding shares or beneficial ownership of such entity. 93 | 94 | 95 | 2. License Grants and Conditions 96 | 97 | 2.1. Grants 98 | 99 | Each Contributor hereby grants You a world-wide, royalty-free, 100 | non-exclusive license: 101 | 102 | a. under intellectual property rights (other than patent or trademark) 103 | Licensable by such Contributor to use, reproduce, make available, 104 | modify, display, perform, distribute, and otherwise exploit its 105 | Contributions, either on an unmodified basis, with Modifications, or as 106 | part of a Larger Work; and 107 | 108 | b. under Patent Claims of such Contributor to make, use, sell, offer for 109 | sale, have made, import, and otherwise transfer either its Contributions 110 | or its Contributor Version. 111 | 112 | 2.2. Effective Date 113 | 114 | The licenses granted in Section 2.1 with respect to any Contribution become 115 | effective for each Contribution on the date the Contributor first distributes 116 | such Contribution. 117 | 118 | 2.3. Limitations on Grant Scope 119 | 120 | The licenses granted in this Section 2 are the only rights granted under this 121 | License. No additional rights or licenses will be implied from the distribution 122 | or licensing of Covered Software under this License. Notwithstanding Section 123 | 2.1(b) above, no patent license is granted by a Contributor: 124 | 125 | a. for any code that a Contributor has removed from Covered Software; or 126 | 127 | b. for infringements caused by: (i) Your and any other third party’s 128 | modifications of Covered Software, or (ii) the combination of its 129 | Contributions with other software (except as part of its Contributor 130 | Version); or 131 | 132 | c. under Patent Claims infringed by Covered Software in the absence of its 133 | Contributions. 134 | 135 | This License does not grant any rights in the trademarks, service marks, or 136 | logos of any Contributor (except as may be necessary to comply with the 137 | notice requirements in Section 3.4). 138 | 139 | 2.4. Subsequent Licenses 140 | 141 | No Contributor makes additional grants as a result of Your choice to 142 | distribute the Covered Software under a subsequent version of this License 143 | (see Section 10.2) or under the terms of a Secondary License (if permitted 144 | under the terms of Section 3.3). 145 | 146 | 2.5. Representation 147 | 148 | Each Contributor represents that the Contributor believes its Contributions 149 | are its original creation(s) or it has sufficient rights to grant the 150 | rights to its Contributions conveyed by this License. 151 | 152 | 2.6. Fair Use 153 | 154 | This License is not intended to limit any rights You have under applicable 155 | copyright doctrines of fair use, fair dealing, or other equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under the 169 | terms of this License. You must inform recipients that the Source Code Form 170 | of the Covered Software is governed by the terms of this License, and how 171 | they can obtain a copy of this License. You may not attempt to alter or 172 | restrict the recipients’ rights in the Source Code Form. 173 | 174 | 3.2. Distribution of Executable Form 175 | 176 | If You distribute Covered Software in Executable Form then: 177 | 178 | a. such Covered Software must also be made available in Source Code Form, 179 | as described in Section 3.1, and You must inform recipients of the 180 | Executable Form how they can obtain a copy of such Source Code Form by 181 | reasonable means in a timely manner, at a charge no more than the cost 182 | of distribution to the recipient; and 183 | 184 | b. You may distribute such Executable Form under the terms of this License, 185 | or sublicense it under different terms, provided that the license for 186 | the Executable Form does not attempt to limit or alter the recipients’ 187 | rights in the Source Code Form under this License. 188 | 189 | 3.3. Distribution of a Larger Work 190 | 191 | You may create and distribute a Larger Work under terms of Your choice, 192 | provided that You also comply with the requirements of this License for the 193 | Covered Software. If the Larger Work is a combination of Covered Software 194 | with a work governed by one or more Secondary Licenses, and the Covered 195 | Software is not Incompatible With Secondary Licenses, this License permits 196 | You to additionally distribute such Covered Software under the terms of 197 | such Secondary License(s), so that the recipient of the Larger Work may, at 198 | their option, further distribute the Covered Software under the terms of 199 | either this License or such Secondary License(s). 200 | 201 | 3.4. Notices 202 | 203 | You may not remove or alter the substance of any license notices (including 204 | copyright notices, patent notices, disclaimers of warranty, or limitations 205 | of liability) contained within the Source Code Form of the Covered 206 | Software, except that You may alter any license notices to the extent 207 | required to remedy known factual inaccuracies. 208 | 209 | 3.5. Application of Additional Terms 210 | 211 | You may choose to offer, and to charge a fee for, warranty, support, 212 | indemnity or liability obligations to one or more recipients of Covered 213 | Software. However, You may do so only on Your own behalf, and not on behalf 214 | of any Contributor. You must make it absolutely clear that any such 215 | warranty, support, indemnity, or liability obligation is offered by You 216 | alone, and You hereby agree to indemnify every Contributor for any 217 | liability incurred by such Contributor as a result of warranty, support, 218 | indemnity or liability terms You offer. You may include additional 219 | disclaimers of warranty and limitations of liability specific to any 220 | jurisdiction. 221 | 222 | 4. Inability to Comply Due to Statute or Regulation 223 | 224 | If it is impossible for You to comply with any of the terms of this License 225 | with respect to some or all of the Covered Software due to statute, judicial 226 | order, or regulation then You must: (a) comply with the terms of this License 227 | to the maximum extent possible; and (b) describe the limitations and the code 228 | they affect. Such description must be placed in a text file included with all 229 | distributions of the Covered Software under this License. Except to the 230 | extent prohibited by statute or regulation, such description must be 231 | sufficiently detailed for a recipient of ordinary skill to be able to 232 | understand it. 233 | 234 | 5. Termination 235 | 236 | 5.1. The rights granted under this License will terminate automatically if You 237 | fail to comply with any of its terms. However, if You become compliant, 238 | then the rights granted under this License from a particular Contributor 239 | are reinstated (a) provisionally, unless and until such Contributor 240 | explicitly and finally terminates Your grants, and (b) on an ongoing basis, 241 | if such Contributor fails to notify You of the non-compliance by some 242 | reasonable means prior to 60 days after You have come back into compliance. 243 | Moreover, Your grants from a particular Contributor are reinstated on an 244 | ongoing basis if such Contributor notifies You of the non-compliance by 245 | some reasonable means, this is the first time You have received notice of 246 | non-compliance with this License from such Contributor, and You become 247 | compliant prior to 30 days after Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, counter-claims, 251 | and cross-claims) alleging that a Contributor Version directly or 252 | indirectly infringes any patent, then the rights granted to You by any and 253 | all Contributors for the Covered Software under Section 2.1 of this License 254 | shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 257 | license agreements (excluding distributors and resellers) which have been 258 | validly granted by You or Your distributors under this License prior to 259 | termination shall survive termination. 260 | 261 | 6. Disclaimer of Warranty 262 | 263 | Covered Software is provided under this License on an “as is” basis, without 264 | warranty of any kind, either expressed, implied, or statutory, including, 265 | without limitation, warranties that the Covered Software is free of defects, 266 | merchantable, fit for a particular purpose or non-infringing. The entire 267 | risk as to the quality and performance of the Covered Software is with You. 268 | Should any Covered Software prove defective in any respect, You (not any 269 | Contributor) assume the cost of any necessary servicing, repair, or 270 | correction. This disclaimer of warranty constitutes an essential part of this 271 | License. No use of any Covered Software is authorized under this License 272 | except under this disclaimer. 273 | 274 | 7. Limitation of Liability 275 | 276 | Under no circumstances and under no legal theory, whether tort (including 277 | negligence), contract, or otherwise, shall any Contributor, or anyone who 278 | distributes Covered Software as permitted above, be liable to You for any 279 | direct, indirect, special, incidental, or consequential damages of any 280 | character including, without limitation, damages for lost profits, loss of 281 | goodwill, work stoppage, computer failure or malfunction, or any and all 282 | other commercial damages or losses, even if such party shall have been 283 | informed of the possibility of such damages. This limitation of liability 284 | shall not apply to liability for death or personal injury resulting from such 285 | party’s negligence to the extent applicable law prohibits such limitation. 286 | Some jurisdictions do not allow the exclusion or limitation of incidental or 287 | consequential damages, so this exclusion and limitation may not apply to You. 288 | 289 | 8. Litigation 290 | 291 | Any litigation relating to this License may be brought only in the courts of 292 | a jurisdiction where the defendant maintains its principal place of business 293 | and such litigation shall be governed by laws of that jurisdiction, without 294 | reference to its conflict-of-law provisions. Nothing in this Section shall 295 | prevent a party’s ability to bring cross-claims or counter-claims. 296 | 297 | 9. Miscellaneous 298 | 299 | This License represents the complete agreement concerning the subject matter 300 | hereof. If any provision of this License is held to be unenforceable, such 301 | provision shall be reformed only to the extent necessary to make it 302 | enforceable. Any law or regulation which provides that the language of a 303 | contract shall be construed against the drafter shall not be used to construe 304 | this License against a Contributor. 305 | 306 | 307 | 10. Versions of the License 308 | 309 | 10.1. New Versions 310 | 311 | Mozilla Foundation is the license steward. Except as provided in Section 312 | 10.3, no one other than the license steward has the right to modify or 313 | publish new versions of this License. Each version will be given a 314 | distinguishing version number. 315 | 316 | 10.2. Effect of New Versions 317 | 318 | You may distribute the Covered Software under the terms of the version of 319 | the License under which You originally received the Covered Software, or 320 | under the terms of any subsequent version published by the license 321 | steward. 322 | 323 | 10.3. Modified Versions 324 | 325 | If you create software not governed by this License, and you want to 326 | create a new license for such software, you may create and use a modified 327 | version of this License if you rename the license and remove any 328 | references to the name of the license steward (except to note that such 329 | modified license differs from this License). 330 | 331 | 10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses 332 | If You choose to distribute Source Code Form that is Incompatible With 333 | Secondary Licenses under the terms of this version of the License, the 334 | notice described in Exhibit B of this License must be attached. 335 | 336 | Exhibit A - Source Code Form License Notice 337 | 338 | This Source Code Form is subject to the 339 | terms of the Mozilla Public License, v. 340 | 2.0. If a copy of the MPL was not 341 | distributed with this file, You can 342 | obtain one at 343 | http://mozilla.org/MPL/2.0/. 344 | 345 | If it is not possible or desirable to put the notice in a particular file, then 346 | You may include the notice in a location (such as a LICENSE file in a relevant 347 | directory) where a recipient would be likely to look for such a notice. 348 | 349 | You may add additional accurate notices of copyright ownership. 350 | 351 | Exhibit B - “Incompatible With Secondary Licenses” Notice 352 | 353 | This Source Code Form is “Incompatible 354 | With Secondary Licenses”, as defined by 355 | the Mozilla Public License, v. 2.0. 356 | 357 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/README.md: -------------------------------------------------------------------------------- 1 | # CDP - {{ cookiecutter.municipality }} 2 | 3 | [![Infrastructure Deployment Status]({{ cookiecutter.hosting_github_url }}/workflows/Infrastructure/badge.svg)]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Infrastructure%22) 4 | [![Event Processing Pipeline]({{ cookiecutter.hosting_github_url }}/workflows/Event%20Gather/badge.svg)]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Event+Gather%22) 5 | [![Event Index Pipeline]({{ cookiecutter.hosting_github_url }}/workflows/Event%20Index/badge.svg)]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Event+Index%22) 6 | [![Web Deployment Status]({{ cookiecutter.hosting_github_url }}/workflows/Web%20App/badge.svg)]({{ cookiecutter.hosting_web_app_address }}) 7 | [![Repo Build Status]({{ cookiecutter.hosting_github_url }}/workflows/Build%20Main/badge.svg)]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Build+Main%22) 8 | 9 | --- 10 | 11 | ## Council Data Project 12 | 13 | Council Data Project is an open-source project dedicated to providing journalists, activists, researchers, and all members of each community we serve with the tools they need to stay informed and hold their Council Members accountable. 14 | 15 | For more information about Council Data Project, please visit [our website](https://councildataproject.org/). 16 | 17 | ## Instance Information 18 | 19 | This repo serves the municipality: **{{ cookiecutter.municipality }}** 20 | 21 | ### Python Access 22 | 23 | Install: 24 | 25 | `pip install cdp-backend` 26 | 27 | Quickstart: 28 | 29 | ```python 30 | from cdp_backend.database import models as db_models 31 | from cdp_backend.pipeline.transcript_model import Transcript 32 | import fireo 33 | from gcsfs import GCSFileSystem 34 | from google.auth.credentials import AnonymousCredentials 35 | from google.cloud.firestore import Client 36 | 37 | # Connect to the database 38 | fireo.connection(client=Client( 39 | project="{{ cookiecutter.infrastructure_slug }}", 40 | credentials=AnonymousCredentials() 41 | )) 42 | 43 | # Read from the database 44 | five_people = list(db_models.Person.collection.fetch(5)) 45 | 46 | # Connect to the file store 47 | fs = GCSFileSystem(project="{{ cookiecutter.infrastructure_slug }}", token="anon") 48 | 49 | # Read a transcript's details from the database 50 | transcript_model = list(db_models.Transcript.collection.fetch(1))[0] 51 | 52 | # Read the transcript directly from the file store 53 | with fs.open(transcript_model.file_ref.get().uri, "r") as open_resource: 54 | transcript = Transcript.from_json(open_resource.read()) 55 | 56 | # OR download and store the transcript locally with `get` 57 | fs.get(transcript_model.file_ref.get().uri, "local-transcript.json") 58 | # Then read the transcript from your local machine 59 | with open("local-transcript.json", "r") as open_resource: 60 | transcript = Transcript.from_json(open_resource.read()) 61 | ``` 62 | 63 | - See the [CDP Database Schema](https://councildataproject.org/cdp-backend/database_schema.html) 64 | for a Council Data Project database schema diagram. 65 | - See the [FireO documentation](https://octabyte.io/FireO/) 66 | to learn how to construct queries using CDP database models. 67 | - See the [GCSFS documentation](https://gcsfs.readthedocs.io/en/latest/index.html) 68 | to learn how to retrieve files from the file store. 69 | 70 | ## Contributing 71 | 72 | If you wish to contribute to CDP please note that the best method to do so is to contribute to the upstream libraries that compose the CDP Instances themselves. These are detailed below. 73 | 74 | - [cdp-backend](https://github.com/CouncilDataProject/cdp-backend): Contains all the database models, data processing pipelines, and infrastructure-as-code for CDP deployments. Contributions here will be available to all CDP Instances. Entirely written in Python. 75 | - [cdp-frontend](https://github.com/CouncilDataProject/cdp-frontend): Contains all of the components used by the web apps to be hosted on GitHub Pages. Contributions here will be available to all CDP Instances. Entirely written in TypeScript and React. 76 | - [cookiecutter-cdp-deployment](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment): The repo used to generate new CDP Instance deployments. Like this repo! 77 | - [councildataproject.org](https://github.com/CouncilDataProject/councildataproject.github.io): Our landing page! Contributions here should largely be text changes and admin updates. 78 | 79 | ## Instance Admin Documentation 80 | 81 | You can find documentation on how to customize, update, and maintain this CDP instance 82 | in the 83 | [admin-docs directory]({{ cookiecutter.hosting_github_url }}/tree/main/admin-docs). 84 | 85 | ## License 86 | 87 | CDP software is licensed under a [MIT License](./LICENSE). 88 | 89 | Content produced by this instance is available under a [Creative Commons Attribution 4.0 International License](https://creativecommons.org/licenses/by/4.0/). 90 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/SETUP/README.md: -------------------------------------------------------------------------------- 1 | # CDP Instance Setup 2 | 3 | This document outlines the steps necessary to finish initializing this CDP Instance. 4 | 5 | ## Before You Begin 6 | 7 | Install the command line tools that will help shorten the setup process 8 | 9 | 1. Install [gcloud](https://cloud.google.com/sdk/docs/install) 10 | 1. Install [gsutil](https://cloud.google.com/storage/docs/gsutil_install) 11 | 1. Install [firebase-tools](https://firebase.google.com/docs/cli/) 12 | 1. Install [just](https://github.com/casey/just) 13 | 14 | ## Initial Repository Setup 15 | 16 | There are additional tasks required after generating this repository. 17 | 18 | 1. Create the GitHub repository for this deployment to live in. 19 | 20 | [Create a new Repository](https://github.com/new) with the following parameters: 21 | 22 | - Set the repo name to: **{{ cookiecutter.hosting_github_repo_name }}** 23 | - Set the repo owner to: **{{ cookiecutter.hosting_github_username_or_org }}** 24 | - Set the repo visibility to: "Public" 25 | - Do not initialize with any of the extra options 26 | - Click "Create repository". 27 | 28 | 1. Install `cdp-backend`. 29 | 30 | This step should be ran while within the `SETUP` directory (`cd SETUP`). 31 | 32 | ```bash 33 | pip install ../python/ 34 | ``` 35 | 36 | 1. Get the infrastructure files. 37 | 38 | This step should be ran while within the `SETUP` directory (`cd SETUP`). 39 | 40 | ```bash 41 | get_cdp_infrastructure_stack . 42 | ``` 43 | 44 | 1. Login to Google Cloud. 45 | 46 | This step should be run while within the `SETUP` directory (`cd SETUP`). 47 | 48 | Run: 49 | 50 | ```bash 51 | just login 52 | ``` 53 | 54 | 1. Initialize the basic project infrastructure. 55 | 56 | This step should be run while within the `SETUP` directory (`cd SETUP`) 57 | 58 | Run: 59 | 60 | ```bash 61 | just init {{ cookiecutter.infrastructure_slug }} 62 | ``` 63 | 64 | This step will also generate a Google Service Account JSON file and store it 65 | in a directory called `.keys` in the root of this repository. 66 | 67 | 1. Set or update the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the 68 | path to the key that was just generated. 69 | 70 | ```bash 71 | export GOOGLE_APPLICATION_CREDENTIALS="INSERT/PATH/HERE" 72 | ``` 73 | 74 | 1. Create (or re-use) a 75 | [Google Cloud billing account](https://console.cloud.google.com/billing/linkedaccount?project={{ cookiecutter.infrastructure_slug }}) 76 | and attach it to the newly created project ({{ cookiecutter.infrastructure_slug }}). 77 | 78 | For more details on the cost of maintaining a CDP Instance, see our [estimated cost breakdown](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment#cost). 79 | 80 | 1. Generate a Firebase CI token. 81 | 82 | ```bash 83 | firebase login:ci 84 | ``` 85 | 86 | Save the created token for a following step! 87 | 88 | 1. Create a GitHub Personal Access Token. 89 | 90 | Create a new (classic) GitHub Personal Access Token by navigating to 91 | [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new). 92 | 93 | - Click the "Generate new token" dropdown. 94 | - Select "Generate new token (classic)". 95 | - Give the token a descriptive name / note. We recommend: `{{ cookiecutter.infrastructure_slug }}` 96 | - Set the expiration to "No expiration" 97 | - You can set a set expiration if you would like, you will simply have to update this token later. 98 | - Select the `repo` checkbox to give access this token access to the repo. 99 | - Click the "Generate token" button. 100 | 101 | Save the created token for a following step. 102 | 103 | For more documentation and assistance see 104 | [GitHub's Documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token#creating-a-personal-access-token-classic). 105 | 106 | 1. Attach the Google Service Account JSON as GitHub Repository Secret. 107 | 108 | 1. Create a [new secret]({{ cookiecutter.hosting_github_url }}/settings/secrets/actions/new) 109 | 110 | - Set the name to: **GOOGLE_CREDENTIALS** 111 | - Set the value to: the contents of the file `.keys/{{ cookiecutter.infrastructure_slug }}.json` 112 | - Click "Add secret" 113 | 114 | 2. Create a [new secret]({{ cookiecutter.hosting_github_url }}/settings/secrets/actions/new) 115 | 116 | - Set the name to: **FIREBASE_TOKEN** 117 | - Set the value to: the value of the Firebase CI token you created in a prior step. 118 | - Click "Add secret" 119 | 120 | 3. Create a [new secret]({{ cookiecutter.hosting_github_url }}/settings/secrets/actions/new) 121 | 122 | - Set the name to: **PERSONAL_ACCESS_TOKEN** 123 | - Set the value to: the value of the GitHub Personal Access Token you created in a prior step. 124 | - Click "Add secret" 125 | 126 | 1. Build the basic project infrastructure. 127 | 128 | This step should be run while within the `SETUP` directory (`cd SETUP`) 129 | 130 | ```bash 131 | just setup {{ cookiecutter.infrastructure_slug }} {{ cookiecutter.firestore_region }} 132 | ``` 133 | 134 | 1. Initialize Firebase Storage. 135 | 136 | [Firestore Storage Page](https://console.firebase.google.com/u/0/project/{{ cookiecutter.infrastructure_slug }}/storage) 137 | 138 | The default settings ("Start in Production Mode" and default region) for setting up 139 | storage are fine. 140 | 141 | 1. Initialize and push the local repository to GitHub. 142 | 143 | This step should be run while within the base directory of the repository (`cd ..`). 144 | 145 | To initialize the repo locally, run: 146 | 147 | ```bash 148 | git init 149 | git add -A 150 | git commit -m "Initial commit" 151 | git branch -M main 152 | ``` 153 | 154 | To setup a connection to our GitHub repo, run either: 155 | 156 | ```bash 157 | git remote add origin {{ cookiecutter.hosting_github_url }}.git 158 | ``` 159 | 160 | Or (with SSH): 161 | 162 | ```bash 163 | git remote add origin git@github.com:{{ cookiecutter.hosting_github_username_or_org }}/{{ cookiecutter.hosting_github_repo_name }}.git 164 | ``` 165 | 166 | Finally, to push this repo to GitHub, run: 167 | 168 | ```bash 169 | git push -u origin main 170 | ``` 171 | 172 | Now refresh your repository's dashboard to ensure that all files were pushed. 173 | 174 | 1. Once the 175 | ["Web App" GitHub Action Successfully Complete]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Web+App%22) 176 | configure GitHub Pages. 177 | 178 | Go to your repository's [GitHub Pages Configuration]({{ cookiecutter.hosting_github_url }}/settings/pages) 179 | 180 | - Set the source to: "gh-pages" 181 | - Set the folder to: `/ (root)` 182 | - Click "Save" 183 | 184 | 1. Once the ["Infrastructure" GitHub Action Successfully Completes]({{ cookiecutter.hosting_github_url }}/actions?query=workflow%3A%22Infrastructure%22) request a quota increase for `compute.googleapis.com/gpus_all_regions`. 185 | 186 | [Direct Link to Quota](https://console.cloud.google.com/iam-admin/quotas?project={{ cookiecutter.infrastructure_slug }}&pageState=(%22allQuotasTable%22:(%22f%22:%22%255B%257B_22k_22_3A_22Metric_22_2C_22t_22_3A10_2C_22v_22_3A_22_5C_22compute.googleapis.com%252Fgpus_all_regions_5C_22_22_2C_22s_22_3Atrue_2C_22i_22_3A_22metricName_22%257D%255D%22))) 187 | 188 | - Click the checkbox for the "GPUs (all regions)" 189 | - Click the "EDIT QUOTAS" button 190 | - In the "New limit" text field, enter a value of: `2`. 191 | - You can request more or less than `2` GPUs, however we have noticed that a 192 | request of `2` is generally automatically accepted. 193 | - In the "Request description" text field, enter a value of: speech-to-text 194 | model application and downstream text tasks 195 | - Click the "NEXT" button 196 | - Enter your name and phone number into the contact fields. 197 | - Click the "SUBMIT REQUEST" button 198 | 199 | If the above direct link doesn't work, follow the instructions from 200 | [Google Documentation](https://cloud.google.com/docs/quota#requesting_higher_quota). 201 | 202 | You will need to wait until the quota increase has been approved before running any 203 | event processing. From our experience, the quota is approved within 15 minutes. 204 | 205 | **If all steps complete successful your web application will be viewable at: {{ cookiecutter.hosting_web_app_address }}** 206 | 207 | ## Data Gathering Setup 208 | 209 | Once your repository, infrastructure, and web application have been set up, you will need to write an event data gathering function. 210 | 211 | Navigate and follow the instructions in the the file: `python/cdp_{{ cookiecutter.python_municipality_slug }}_backend/scraper.py`. 212 | 213 | As soon as you push your updates to your event gather function (`get_events`) to your GitHub repository, everything will be tested and configured for the next pipeline run. Events are gathered from this function every 6 hours from the default branch via a Github Action cron job. If you'd like to manually run event gathering, you can do so from within the Actions tab of your repo -> Event Gather -> Run workflow. 214 | 215 | It is expected that the Event Index workflow will fail to start, as your database will not yet be populated with events to index. 216 | 217 | There are some optional configurations for the data gathering pipeline which can be added to `python/event-gather-config.json`. No action is needed for a barebones pipeline run, but the optional parameters can be checked in the [CDP pipeline config documentation](https://councildataproject.org/cdp-backend/cdp_backend.pipeline.html#module-cdp_backend.pipeline.pipeline_config). Note that `google_credentials_file` and `get_events_function_path` should not be modified and will populate automatically if you have followed the steps above. 218 | 219 | Be sure to review the [CDP Ingestion Model documentation](https://councildataproject.github.io/cdp-backend/ingestion_models.html) for the object definition to return from your `get_events` function. 220 | 221 | Once your function is complete and pushed to the `main` branch, feel free to delete this setup directory. 222 | 223 | ## Other Documentation 224 | 225 | For more documentation on adding data to your new CDP instance and maintainer or customizing your instance 226 | please see the "admin-docs" directory. 227 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/adding-analytics.md: -------------------------------------------------------------------------------- 1 | # Adding Analytics 2 | 3 | Council Data Project is setup with [Plausible Analytics](https://plausible.io/about) 4 | by default. 5 | 6 | ## CDP Org Hosted Instances 7 | 8 | If your CDP instance is hosted under the "councildataproject.org" domain, 9 | you shouldn't have to change anything from the default cookiecutter settings 10 | as the `data-domain` value for the Plausible Analytics script in `web/public/index.html` 11 | should be set to `councildataproject.github.io` for you. 12 | 13 | If it isn't however, please update the `data-domain` value to `councildataproject.github.io`. 14 | 15 | Once done, the analytics for your CDP instance should be publicly available on our 16 | [Plausible Dashboard](https://plausible.io/councildataproject.github.io?page=%2F{{ cookiecutter.municipality_slug }}%2F**). 17 | 18 | ## Self Hosted Instances 19 | 20 | If you want to use a different analytics platform or service, 21 | simply replace the Plausible Analytics script in `web/public/index.html` with the service 22 | of your choosing and setup the rest as you normally would. 23 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/customizing-automated-pipelines.md: -------------------------------------------------------------------------------- 1 | # Customizing Automated Pipelines 2 | 3 | Council Data Pipelines all run for you automatically on set schedules 4 | or on push to the repository and generally you should never need to change the 5 | default configuration. 6 | 7 | However, you may want to for example, change the interval, or the time that pipelines 8 | run. 9 | 10 | To do so, navigate to the `.github` directory of your repository 11 | ([GitHub link]({{ cookiecutter.hosting_github_url }}/tree/main/.github)), 12 | and then to the `workflows` sub-directory 13 | ([GitHub link]({{ cookiecutter.hosting_github_url }}/tree/main/.github/workflows)). 14 | 15 | ![image of workflows sub-directory](./resources/workflows.png) 16 | 17 | All automated and manual workflows recide in this `.github/workflows` sub-directory. 18 | 19 | The following sections will detail what is "safe" to edit. 20 | If there isn't a section for the workflow you are wish to customize, 21 | it is likely because we haven't thought of a reason for needing to customize the 22 | pipeline. In general however, please refer to 23 | [GitHub Actions Documentation](https://docs.github.com/en/actions) for more 24 | information on how all of these workflows are constructed. 25 | 26 | ## Event Gather and Processing Pipeline 27 | 28 | This pipeline ([including the manual trigger](./manual-event-gather.md)) 29 | can be found under `event-gather-pipeline.yml` 30 | ([GitHub link]({{ cookiecutter.hosting_github_url }}/tree/main/.github/workflows/event-gather-pipeline.yml)). 31 | 32 | The common reasons for customizing this pipeline are: 33 | 34 | 1. to change the automated schedule 35 | 2. to add required extra OS level dependencies (such as language and tool installations) 36 | 37 | ### Customizing Schedule 38 | 39 | The pipeline schedule is handled by the 40 | [GitHub Action CRON string](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events). 41 | 42 | You may want to change the CRON string to something that more closely matches the 43 | time intervals when you know your municipality tends to post events. 44 | 45 | Be careful, don't add too many intervals as if you try to scrape or request 46 | from a resource too often, your pipeline may start to fail because you are 47 | requesting _too_ often. 48 | 49 | ### Adding Extra OS Dependencies 50 | 51 | The pipeline runs on an Ubuntu server and as such to install many OS level 52 | dependencies you can use `apt` to install more packages. In general, 53 | it is safe to add as many of these extra dependencies as you need and there is already 54 | a section where we add dependencies like this to the pipeline. 55 | 56 | See the "Install Packages" task of this workflow file and add any more 57 | packages you may need there. 58 | 59 | For more programming language support, look into 60 | [GitHub's existing "setup x" actions](https://github.com/actions) and add them 61 | to the pipeline just like the current "actions/setup-python" task. 62 | 63 | ## Event Indexing Pipeline 64 | 65 | This pipeline can be found under `event-index-pipeline.md` 66 | ([GitHub link]({{ cookiecutter.hosting_github_url }}/tree/main/.github/workflows/event-index-pipeline.yml)). 67 | 68 | The common reasons for customizing this pipeline are: 69 | 70 | 1. to increase or decrease the frequency a new index is created 71 | 72 | ### Changing Indexing Frequency 73 | 74 | This is controlled by the 75 | [GitHub Action CRON string](https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events). 76 | 77 | You may want to increase the frequency to generate a fresh index more often or decrease 78 | the frequency because events only happen once a week (or less) and in doing so you can 79 | reduce the cost of running the instance. 80 | 81 | The more frequent the index pipeline runs and the more events that it is indexing, 82 | the database cost increases simply due to how many times you are writing many thousands 83 | of documents to the database per pipeline run. 84 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/manual-event-gather.md: -------------------------------------------------------------------------------- 1 | # Manually Triggering Event Gather 2 | 3 | ### Background 4 | 5 | While event gather and processing is automated and runs on a schedule, 6 | you may want to manually trigger the event gather and processing pipeline: 7 | 8 | 1. To backfill data into the infrastructure 9 | - I.e. to gather the weeks or months from before the instance 10 | was created. 11 | 2. To reprocess data 12 | - I.e. a prior automated run failed for some reason and you wish to rerun 13 | that time period. 14 | - I.e. new and more data in available from the scraper and as such updated and 15 | new data can enter into the CDP infrastructure. 16 | 3. To add a custom or special event 17 | - I.e. a press conference, debate, forum, etc. 18 | 19 | ## Backfilling and Reprocessing 20 | 21 | To backfill or rerun the pipeline for a specific datetime range go to the 22 | [Event Gather GitHub Action Page]({{ cookiecutter.hosting_github_url }}/actions/workflows/event-gather-pipeline.yml). 23 | 24 | Once there, you can add the begin and end datetimes as parameters to the workflow run. 25 | 26 | ![screenshot of "Run workflow" for event gather pipeline](./resources/backfill-event-gather.png) 27 | 28 | See the Python 29 | [`datetime.fromisoformat` documentation](https://docs.python.org/3/library/datetime.html#datetime.datetime.fromisoformat) 30 | for examples of the allowed string patterns for these two parameters. 31 | 32 | **Note:** If you chose a datetime range that results in many of events, the pipeline may 33 | error as the pipeline can only run for six hours for one job. So if you wish to 34 | backfill many months of data, consider breaking up the whole time period into 35 | small enough datetime ranges that no single pipeline run lasts longer than six hours. 36 | 37 | ## Adding Custom Events 38 | 39 | To add custom or special events to the CDP infrastructure go to the 40 | [Process Special Event GitHub Action Page]({{ cookiecutter.hosting_github_url }}/actions/workflows/process-special-event.yml). 41 | 42 | Once there, you can add the full event details as a parameter to the workflow run. 43 | 44 | ![screenshot of "Run workflow" for special event pipeline](./resources/special-event-gather.png) 45 | 46 | The safest method to construct the event JSON string is using the 47 | `cdp-backend` Python API: 48 | 49 | 1. `pip install cdp-backend` 50 | 2. Create a Python file with your Event definition, at the end, 51 | print the JSON string of the event 52 | (including the wrapping `'` characters -- `repr` does this for you). 53 | 54 | ```python 55 | from datetime import datetime 56 | 57 | from cdp_backend.pipeline import ingestion_models 58 | 59 | # Define your event 60 | event = ingestion_models.EventIngestionModel( 61 | body=ingestion_models.Body(name="2021 Mayoral Debates"), 62 | sessions=[ 63 | ingestion_models.Session( 64 | video_uri="https://video.seattle.gov/media/council/brief_091321_2012171V.mp4", 65 | session_datetime=datetime(2021, 9, 13), 66 | session_index=0, 67 | ), 68 | ], 69 | ) 70 | 71 | # Print out the JSON string in it's full form 72 | print(repr(event.to_json())) 73 | ``` 74 | 75 | 3. Run your file 76 | 77 | ```bash 78 | python your-file.py 79 | ``` 80 | 81 | 4. Run the workflow 82 | 83 | Copy and past the output from your terminal into the "Run workflow" parameter and 84 | then click the "Run workflow" button itself. If the data provided is valid and 85 | publically available (i.e. videos are publically downloadable), the pipeline will 86 | run and your custom event will make it's way into CDP infrastructure. 87 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/backfill-event-gather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/backfill-event-gather.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/cookiecutter-interrupt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/cookiecutter-interrupt.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/example-custom-event.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from cdp_backend.pipeline import ingestion_models 4 | 5 | # Define your event 6 | event = ingestion_models.EventIngestionModel( 7 | body=ingestion_models.Body(name="2021 Mayoral Debates"), 8 | sessions=[ 9 | ingestion_models.Session( 10 | video_uri="https://video.seattle.gov/media/council/brief_091321_2012171V.mp4", 11 | session_datetime=datetime(2021, 9, 13), 12 | session_index=0, 13 | ), 14 | ], 15 | ) 16 | 17 | # Print out the JSON string in it's full form 18 | print(repr(event.to_json())) 19 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/special-event-gather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/special-event-gather.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/update-and-git-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/update-and-git-status.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/vs-code-status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/vs-code-status.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/workflows.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/admin-docs/resources/workflows.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/running-extra-scripts.md: -------------------------------------------------------------------------------- 1 | # Running Extra Scripts 2 | 3 | ### Background 4 | 5 | While we hope for it to be rare, when we ship changes to `cdp-backend` or `cdp-frontend` 6 | require changes to the data or file storage system that we cannot make during a normal 7 | pipeline run, or if you want to manage your data in some way, there is a GitHub Action 8 | that allows the passthrough of any command straight to the running GitHub Action 9 | worker. 10 | 11 | ## Backfilling Session Context Hash 12 | 13 | In our [v3.0.4 release of `cdp-backend`](https://github.com/CouncilDataProject/cdp-backend/releases/tag/v3.0.4) 14 | we added a property to the `Session` collection that stores the `session_context_hash`. 15 | This value is incredibly useful for programmatic work but isn't _needed_ for the 16 | web application. As such, if you don't want to run this script, you don't need to, 17 | but it is easy enough to run. 18 | 19 | Open up the [Run Command]({{ cookiecutter.hosting_github_url }}/actions/workflows/run-script.yml) 20 | GitHub Action and click the "Run workflow" button. 21 | 22 | Paste in the following: `add_content_hash_to_sessions --google_credentials_file google-creds.json` 23 | 24 | Click the "Run workflow" button and the backfill process should kick off. 25 | (If the run doesn't appear, try refreshing the page.) 26 | 27 | If something goes wrong, please create a 28 | [GitHub Issue in `cdp-backend`](https://github.com/CouncilDataProject/cdp-backend/issues) 29 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/admin-docs/updating-to-new-cookiecutter-releases.md: -------------------------------------------------------------------------------- 1 | # Updating to New Cookiecutter Releases 2 | 3 | As we have abstracted away the `cdp-backend` for pipeline, infrastructure, 4 | and, database functionality, and `cdp-frontend` for web-app functionality, 5 | and because every time the pipelines which utilize those packages run, the 6 | pipeline pulls in the latest (non-breaking) versions of each, you should 7 | rarely need to update anything yourself. 8 | 9 | However, in the instance that a new version of 10 | _this cookiecutter generated repository_ becomes available 11 | (as seen on our [releases page](https://github.com/CouncilDataProject/cookiecutter-cdp-deployment/releases)), 12 | you may want to pull in the changes as they might change default pipeline configuration, 13 | enable new features, add more documentation, and more. 14 | 15 | To do so, you should feel comfortable with the command line and git. 16 | 17 | ### Steps to Upgrade 18 | 19 | 1. Clone (or fetch and pull) your repository: 20 | 21 | - `git clone {{ cookiecutter.hosting_github_url }}.git` 22 | 23 | OR if you have previously cloned your repository and other changes have occurred: 24 | 25 | - `git checkout main` 26 | - `git fetch` 27 | - `git pull main` 28 | 29 | 2. Install `cookiecutter`: 30 | 31 | You will need Python 3.6+ installed for this. 32 | 33 | - `pip install cookiecutter` 34 | 35 | 3. Update your repo using the latest cookiecutter version: 36 | 37 | This will pull in the current cookiecutter updates as changes to the current 38 | repository. 39 | 40 | Ensure you are in the repository directory: 41 | 42 | - `cd {{ cookiecutter.municipality_slug }}` 43 | 44 | Then run: 45 | 46 | - `just update-from-cookiecutter` 47 | 48 | **Note: you may need to install [just](https://github.com/casey/just#packages).** 49 | 50 | 4. Select the desired changes to commit: 51 | 52 | In an editor or in the terminal you can now review the changes to commit. 53 | 54 | With git in a terminal: 55 | 56 | - `git status` 57 | - `git add {some-file}` 58 | - OR `git restore {some-file}` 59 | 60 | ![screenshot of just update and resulting git status](./resources/update-and-git-status.png) 61 | 62 | In VS Code: 63 | 64 | ![screenshot of source control pane in vs code](./resources/vs-code-status.png) 65 | 66 | 5. Commit and push: 67 | 68 | With the desired changes selected, commit and push to update your repository. 69 | It is recommended to include the cookiecutter version in your commit message. 70 | 71 | - `git commit -m "Update to cookiecutter v0.1.0"` 72 | - `git push origin main` 73 | 74 | After all steps are complete, you should see updates on your repository and 75 | any pipelines that run on `push` they will automatically trigger. 76 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/cookiecutter.yaml: -------------------------------------------------------------------------------- 1 | default_context: 2 | municipality: "{{ cookiecutter.municipality }}" 3 | iana_timezone: "{{ cookiecutter.iana_timezone }}" 4 | governing_body_type: "{{ cookiecutter.governing_body_type }}" 5 | municipality_slug: "{{ cookiecutter.municipality_slug }}" 6 | python_municipality_slug: "{{ cookiecutter.python_municipality_slug }}" 7 | infrastructure_slug: "{{ cookiecutter.infrastructure_slug }}" 8 | maintainer_or_org_full_name: "{{ cookiecutter.maintainer_or_org_full_name }}" 9 | hosting_github_username_or_org: "{{ cookiecutter.hosting_github_username_or_org }}" 10 | hosting_github_repo_name: "{{ cookiecutter.hosting_github_repo_name }}" 11 | hosting_github_url: "{{ cookiecutter.hosting_github_url }}" 12 | hosting_web_app_address: "{{ cookiecutter.hosting_web_app_address }}" 13 | firestore_region: "{{ cookiecutter.firestore_region }}" 14 | event_gather_timedelta_lookback_days: {{ cookiecutter.event_gather_timedelta_lookback_days }} 15 | event_gather_cron: "{{ cookiecutter.event_gather_cron }}" 16 | enable_clipping: "{{ cookiecutter.enable_clipping }}" 17 | speech_to_text_model_version: "{{ cookiecutter.speech_to_text_model_version }}" 18 | event_gather_runner_timeout_minutes: "{{ cookiecutter.event_gather_runner_timeout_minutes }}" 19 | event_gather_runner_max_attempts: "{{ cookiecutter.event_gather_runner_max_attempts }}" 20 | event_gather_runner_retry_wait_seconds: "{{ cookiecutter.event_gather_runner_retry_wait_seconds }}" 21 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/infra/requirements.txt: -------------------------------------------------------------------------------- 1 | cdp-backend==4.1.3 -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/cdp_{{ cookiecutter.python_municipality_slug }}_backend/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Top-level package for {{ cookiecutter.municipality }} 5 | CDP Instance backend. 6 | """ 7 | 8 | __author__ = "{{ cookiecutter.maintainer_or_org_full_name }}" 9 | __version__ = "1.0.0" 10 | 11 | 12 | def get_module_version() -> str: 13 | return __version__ 14 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/cdp_{{ cookiecutter.python_municipality_slug }}_backend/scraper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import datetime 5 | from typing import List 6 | 7 | from cdp_backend.pipeline.ingestion_models import EventIngestionModel 8 | 9 | ############################################################################### 10 | 11 | 12 | def get_events( 13 | from_dt: datetime, 14 | to_dt: datetime, 15 | **kwargs, 16 | ) -> List[EventIngestionModel]: 17 | """ 18 | Get all events for the provided timespan. 19 | 20 | Parameters 21 | ---------- 22 | from_dt: datetime 23 | Datetime to start event gather from. 24 | to_dt: datetime 25 | Datetime to end event gather at. 26 | 27 | Returns 28 | ------- 29 | events: List[EventIngestionModel] 30 | All events gathered that occured in the provided time range. 31 | 32 | Notes 33 | ----- 34 | As the implimenter of the get_events function, you can choose to ignore the from_dt 35 | and to_dt parameters. However, they are useful for manually kicking off pipelines 36 | from GitHub Actions UI. 37 | """ 38 | 39 | # Your implementation here 40 | return [] 41 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/event-gather-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "google_credentials_file": "google-creds.json", 3 | "get_events_function_path": "cdp_{{ cookiecutter.python_municipality_slug }}_backend.scraper.get_events", 4 | "gcs_bucket_name": null, 5 | "whisper_model_name": "{{ cookiecutter.speech_to_text_model_version }}", 6 | "whisper_model_confidence": null, 7 | "default_event_gather_from_days_timedelta": {{ cookiecutter.event_gather_timedelta_lookback_days }} 8 | } 9 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/event-index-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "google_credentials_file": "google-creds.json", 3 | "gcs_bucket_name": null, 4 | "datetime_weighting_days_decay": 30 5 | } 6 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [flake8] 5 | ignore = 6 | E203 7 | E402 8 | W291 9 | W503 10 | max-line-length = 88 -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """The setup script.""" 5 | 6 | from setuptools import find_packages, setup 7 | 8 | requirements = [ 9 | "cdp-backend[pipeline]==4.1.3", 10 | ] 11 | 12 | test_requirements = [ 13 | "black>=19.10b0", 14 | "flake8>=3.8.3", 15 | "flake8-debugger>=3.2.1", 16 | ] 17 | 18 | dev_requirements = [ 19 | *test_requirements, 20 | "wheel>=0.34.2", 21 | ] 22 | 23 | extra_requirements = { 24 | "test": test_requirements, 25 | "dev": dev_requirements, 26 | "all": [ 27 | *requirements, 28 | *dev_requirements, 29 | ], 30 | } 31 | 32 | setup( 33 | author="{{ cookiecutter.maintainer_or_org_full_name }}", 34 | classifiers=[ 35 | "Development Status :: 2 - Pre-Alpha", 36 | "Intended Audience :: Developers", 37 | "License :: OSI Approved :: MIT License", 38 | "Natural Language :: English", 39 | "Programming Language :: Python :: 3.10", 40 | "Programming Language :: Python :: 3.11", 41 | ], 42 | description="Package containing the gather functions for Example.", 43 | install_requires=requirements, 44 | license="MIT license", 45 | long_description_content_type="text/markdown", 46 | include_package_data=True, 47 | keywords="civic technology, open government", 48 | name="cdp-{{ cookiecutter.python_municipality_slug }}-backend", 49 | packages=find_packages(exclude=["tests", "*.tests", "*.tests.*"]), 50 | python_requires=">=3.10", 51 | tests_require=test_requirements, 52 | extras_require=extra_requirements, 53 | url="{{ cookiecutter.hosting_github_url }}", 54 | version="1.0.0", 55 | zip_safe=False, 56 | ) 57 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | /.pnp 4 | .pnp.js 5 | 6 | # Testing 7 | /coverage 8 | 9 | # Production 10 | /build 11 | 12 | # Misc 13 | .DS_Store 14 | .env.local 15 | .env.development.local 16 | .env.test.local 17 | .env.production.local 18 | 19 | npm-debug.log* 20 | yarn-debug.log* 21 | yarn-error.log* 22 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdp-{{ cookiecutter.municipality_slug }}", 3 | "version": "1.0.0", 4 | "homepage": "{{ cookiecutter.hosting_web_app_address }}", 5 | "license": "MIT", 6 | "scripts": { 7 | "start": "react-scripts start", 8 | "build": "react-scripts build", 9 | "clean": "rimraf build", 10 | "deploy": "gh-pages -d build" 11 | }, 12 | "dependencies": { 13 | "@councildataproject/cdp-frontend": "3.2.3", 14 | "react": "^16.13.1", 15 | "react-dom": "^16.13.1" 16 | }, 17 | "devDependencies": { 18 | "gh-pages": "^2.2.0", 19 | "react-scripts": "^4.0.3", 20 | "rimraf": "^3.0.2" 21 | }, 22 | "browserslist": { 23 | "production": [ 24 | ">0.2%", 25 | "not dead", 26 | "not op_mini all" 27 | ], 28 | "development": [ 29 | "last 1 chrome version", 30 | "last 1 firefox version", 31 | "last 1 safari version" 32 | ] 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/cdp-og-seo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/web/public/cdp-og-seo.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/cdp-twitter-seo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/web/public/cdp-twitter-seo.png -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CouncilDataProject/cookiecutter-cdp-deployment/84c13f27ea569441399b6b596878eb3fb29845d3/{{ cookiecutter.hosting_github_repo_name }}/web/public/favicon.ico -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Council Data Project | {{ cookiecutter.municipality }} – Searchable 7 | city council videos, transcripts, voting records, and legislation. 8 | 9 | 13 | 17 | 18 | 22 | 23 | 24 | 28 | 29 | 30 | 31 | 35 | 39 | 43 | 47 | 48 | 49 | 50 | 54 | 58 | 62 | 66 | 67 | 68 | 73 | 74 | 75 | 76 |
77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "CDP - {{ cookiecutter.municipality }}", 3 | "name": "Council Data Project - {{ cookiecutter.municipality }}", 4 | "icons": [{ 5 | "src": "favicon.ico", 6 | "sizes": "64x64 32x32 24x24 16x16", 7 | "type": "image/x-icon" 8 | }], 9 | "start_url": ".", 10 | "display": "standalone", 11 | "theme_color": "#000000", 12 | "background_color": "#ffffff" 13 | } 14 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | -------------------------------------------------------------------------------- /{{ cookiecutter.hosting_github_repo_name }}/web/src/index.jsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import ReactDOM from "react-dom"; 3 | import { App, AppConfigProvider } from "@councildataproject/cdp-frontend"; 4 | 5 | import "@councildataproject/cdp-frontend/dist/index.css"; 6 | 7 | const config = { 8 | firebaseConfig: { 9 | options: { 10 | projectId: "{{ cookiecutter.infrastructure_slug }}", 11 | }, 12 | settings: {}, 13 | }, 14 | municipality: { 15 | name: "{{ cookiecutter.municipality }}", 16 | timeZone: "{{ cookiecutter.iana_timezone }}", 17 | footerLinksSections: [], 18 | }, 19 | features: { 20 | // enableClipping: {{ cookiecutter.enable_clipping }}, 21 | }, 22 | } 23 | 24 | ReactDOM.render( 25 |
26 | 27 | 28 | 29 |
, 30 | document.getElementById("root") 31 | ); --------------------------------------------------------------------------------