├── make_ghpages ├── .gitignore ├── mod │ ├── __init__.py │ └── templates │ │ ├── base.html │ │ ├── main_index.html │ │ └── singlepage.html ├── requirements.txt ├── static │ ├── favicon.png │ └── css │ │ └── style.css ├── commit.sh └── make_pages.py ├── requirements.txt ├── setup.cfg ├── .pre-commit-config.yaml ├── README.md ├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── github-pages.yml └── .gitignore /make_ghpages/.gitignore: -------------------------------------------------------------------------------- 1 | out/ -------------------------------------------------------------------------------- /make_ghpages/mod/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pre-commit~=4.2 2 | -------------------------------------------------------------------------------- /make_ghpages/requirements.txt: -------------------------------------------------------------------------------- 1 | jinja2==3.1.6 2 | optimade[server]==1.1.10 3 | -------------------------------------------------------------------------------- /make_ghpages/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Materials-Consortia/providers-dashboard/master/make_ghpages/static/favicon.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | # Line to long. Handled by black. 4 | E501 5 | # Line break before binary operator. This is preferred formatting for black. 6 | W503 -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.6.0 4 | hooks: 5 | - id: trailing-whitespace 6 | exclude: README.md 7 | - id: check-yaml 8 | - id: check-json 9 | 10 | - repo: https://github.com/ambv/black 11 | rev: 24.4.2 12 | hooks: 13 | - id: black 14 | name: Blacken 15 | 16 | - repo: https://github.com/pycqa/flake8 17 | rev: '7.0.0' 18 | hooks: 19 | - id: flake8 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OPTIMADE dashboard of known providers 2 | 3 | This repository contains the code to generate the dashboard of known providers, 4 | extracted from [https://providers.optimade.org](https://providers.optimade.org). 5 | 6 | **In order to see the dashboard, go to [https://www.optimade.org/providers-dashboard/](https://www.optimade.org/providers-dashboard/)**. 7 | 8 | The dashboard page is automatically regenerated via scheduled GitHub Actions (individual pages mention the last time the information was fetched). 9 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | target-branch: master 8 | labels: 9 | - dependency_updates 10 | groups: 11 | python-dependencies: 12 | applies-to: version-updates 13 | dependency-type: production 14 | - package-ecosystem: github-actions 15 | directory: "/" 16 | schedule: 17 | interval: monthly 18 | target-branch: master 19 | labels: 20 | - CI 21 | groups: 22 | github-actions: 23 | applies-to: version-updates 24 | dependency-type: production 25 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Building GitHub pages (CI) 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | 10 | # Cancel running workflows when additional changes are pushed 11 | # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value 12 | concurrency: 13 | group: ${{ github.head_ref || github.run_id }} 14 | cancel-in-progress: true 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Python 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: 3.11 23 | 24 | - name: Install OPTIMADE 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install -r make_ghpages/requirements.txt 28 | 29 | - name: Make pages 30 | run: cd make_ghpages && python make_pages.py 31 | -------------------------------------------------------------------------------- /make_ghpages/commit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Exit with nonzero exit code if anything fails (-e) 4 | # Print all executed lines (-x) 5 | set -ex 6 | 7 | # Commit sha of master branch 8 | SHA=`git rev-parse --verify HEAD` 9 | TARGET_BRANCH="gh-pages" 10 | 11 | if [ "${GITHUB_ACTIONS}" != "true" ]; then 12 | echo "Skipping deploy; just doing a build." 13 | exit 0 14 | fi 15 | 16 | # Move html to temp dir 17 | mv make_ghpages/out ../page-build 18 | git checkout "${TARGET_BRANCH}" || git checkout --orphan "${TARGET_BRANCH}" 19 | rm -rf * || exit 0 20 | cp -r ../page-build/* . 21 | 22 | git config user.name "${COMMIT_AUTHOR}" 23 | git config user.email "${COMMIT_AUTHOR_EMAIL}" 24 | 25 | git add -A . 26 | # If there are no changes to the compiled out (e.g. this is a README update) then just bail. 27 | if git diff --cached --quiet; then 28 | echo "No changes to the output on this push; exiting." 29 | exit 0 30 | fi 31 | 32 | git commit -m "Deploy to GitHub Pages: ${SHA}" 33 | -------------------------------------------------------------------------------- /.github/workflows/github-pages.yml: -------------------------------------------------------------------------------- 1 | name: Building and deploying GitHub pages 2 | 3 | on: 4 | # schedule runs _only_ for the default branch (for a repository) and the base branch (for a PR). 5 | # See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule for more info. 6 | schedule: 7 | # See https://help.github.com/en/actions/automating-your-workflow-with-github-actions/events-that-trigger-workflows#scheduled-events-schedule 8 | # run every day at 05:20 UTC 9 | - cron: "20 5 * * *" 10 | 11 | # also update pages on pushes to master 12 | push: 13 | branches: 14 | - master 15 | 16 | jobs: 17 | daily_build: 18 | runs-on: ubuntu-latest 19 | env: 20 | COMMIT_AUTHOR: Daily Deploy Action 21 | COMMIT_AUTHOR_EMAIL: action@github.com 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - name: Set up Python 3.11 26 | uses: actions/setup-python@v5 27 | with: 28 | python-version: '3.11' 29 | 30 | - name: Install OPTIMADE 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install -r make_ghpages/requirements.txt 34 | 35 | - name: Make pages 36 | run: cd make_ghpages && python make_pages.py 37 | timeout-minutes: 120 38 | 39 | - name: Commit to gh-pages 40 | run: ./make_ghpages/commit.sh 41 | 42 | - name: Push changes 43 | uses: ad-m/github-push-action@v0.8.0 44 | with: 45 | branch: gh-pages 46 | force: true 47 | github_token: ${{ secrets.GITHUB_TOKEN }} 48 | -------------------------------------------------------------------------------- /make_ghpages/mod/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {{title}} 9 | 10 | 11 | 12 | {% block css %} 13 | 14 | 15 | {% endblock css %} 16 | 17 | 20 | 21 | 22 | 23 | {% block header %} 24 |
25 |

26 | Materials Consortia's OPTIMADE list of providers 27 |

28 |

[View on GitHub/List your provider]

29 |
30 | 31 | {% endblock header %} 32 |
33 | {% block body %} {% endblock body %} 34 |
35 | {% block footer %} 36 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Python template 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | *~ 8 | .DS_Store 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | .static_storage/ 60 | .media/ 61 | local_settings.py 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # VSCode project settings 105 | .vscode 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | 113 | # pytest 114 | .pytest_cache/ 115 | 116 | .DS_Store 117 | .idea/ 118 | 119 | Untitled.ipynb 120 | local_openapi.json 121 | local_index_openapi.json 122 | server.cfg 123 | -------------------------------------------------------------------------------- /make_ghpages/mod/templates/main_index.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block css %} 4 | 5 | 6 | {% endblock css %} 7 | 8 | {% block body %} 9 |
10 |
11 |
12 | 13 | Known providersNumber of known providers 14 | {{ providers | length }} providers 15 | 16 | 17 | 18 | Available providersProviders that have registered a base URL with the OPTIMADE federation 19 | {{ globalsummary.with_base_url }} providers 20 | 21 | 22 | 23 | Available sub-databasesNumber of sub-databases aggregated over all providers 24 | {{ globalsummary.num_sub_databases }} sub-databases 25 | 26 | 27 | Number of structuresNumber of structures served by all databases with aggregation enabled 28 | {{ "{:,}".format(globalsummary.num_structures) }} structures available 29 | 30 |
31 | 32 |
33 | 34 |

35 | Provider list (alphabetically ordered by provider ID) 36 |

37 | 38 |
39 | {% for provider in providers %} 40 | {% if provider.attributes.base_url %} 41 |
42 |

{{provider.id}}: {{ provider.attributes.name }}

43 | 44 | {% if provider.attributes.description %} 45 |

{{provider.attributes.description}}

46 | {% endif %} 47 | 48 |
49 | {% if provider.index_metadb.num_non_null_subdbs %} 50 | Available sub-databases 51 | {{ provider.index_metadb.num_non_null_subdbs }} 52 | {% endif %} 53 |
54 | 55 | 61 | 62 | {% if provider.summaryinfo %} 63 |

64 | {% for summaryinfoelem in provider.summaryinfo %} 65 | 66 | {{summaryinfoelem.text}} 67 | {{summaryinfoelem.count}} 68 | 69 | {% endfor %} 70 |

71 | {% endif %} 72 |
73 | {% endif %} 74 | {% endfor %} 75 |
76 | 77 |
78 | {% for provider in providers %} 79 | {% if not provider.attributes.base_url %} 80 |
81 |

{{provider.id}}: 82 | {{ provider.attributes.name }} 83 |

84 | 85 | {% if provider.attributes.description %} 86 |

{{provider.attributes.description}}

87 | {% endif %} 88 | 89 | 96 |
97 | {% endif %} 98 | {% endfor %} 99 | 100 |
101 | 102 |
103 | {% endblock body %} 104 | -------------------------------------------------------------------------------- /make_ghpages/static/css/style.css: -------------------------------------------------------------------------------- 1 | html { 2 | height: 100%; 3 | } 4 | 5 | body { 6 | min-height: 100%; 7 | font-family: 'Noto Sans', sans-serif; 8 | padding: 0px; 9 | margin: 0px; 10 | /*position: relative;*/ 11 | /*padding-bottom: 100px;*/ 12 | /* display: table; */ 13 | } 14 | 15 | /* * {-moz-box-sizing: border-box; box-sizing: border-box;} */ 16 | 17 | main { 18 | padding-left: 16px; 19 | margin-left: 16px; 20 | padding-top: 0px; 21 | margin-top: 0px; 22 | margin-right: 16px; 23 | padding-right: 0px; 24 | padding-bottom: 0px; 25 | margin-bottom: 0px; 26 | } 27 | 28 | p { 29 | margin: 0; 30 | padding: 0; 31 | } 32 | 33 | a { 34 | color: #00a; 35 | text-decoration: none; 36 | } 37 | 38 | a:visited { 39 | color: #00a; 40 | text-decoration: none; 41 | } 42 | 43 | a:hover { 44 | text-decoration: underline; 45 | } 46 | 47 | h1 { 48 | font-weight: bold; 49 | } 50 | 51 | span { 52 | } 53 | 54 | .keyword { 55 | display: inline; 56 | padding: .2em .6em .3em; 57 | font-size: 75%; 58 | color: #000; 59 | border-radius: .25em; 60 | background-color: #A2CBFF; 61 | } 62 | 63 | #entrytitle { 64 | color: #fff; 65 | } 66 | 67 | #entrytitle a { 68 | color: #fff; 69 | } 70 | 71 | .provide-header, .provider-header a { 72 | color: #007; 73 | } 74 | 75 | h2 { 76 | padding-top: 16px; 77 | margin: 0; 78 | font-size: 140%; 79 | color: #005; 80 | } 81 | 82 | .description { 83 | color: #333; 84 | } 85 | 86 | .details { 87 | margin-top: 12px; 88 | } 89 | 90 | .currentstate { 91 | color: #666; 92 | font-size: 90%; 93 | margin-bottom: 12px; 94 | } 95 | 96 | .summaryinfo { 97 | color: #000; 98 | font-size: 80%; 99 | margin-bottom: 12px; 100 | margin-top: 12px; 101 | } 102 | 103 | #entrylist .authors { 104 | color: #666; 105 | font-size: 90%; 106 | margin-bottom: 9px; 107 | } 108 | 109 | 110 | #description p { 111 | margin-left: 0px; 112 | } 113 | 114 | 115 | h3 { 116 | padding-top: 20px; 117 | padding-bottom: 5px; 118 | margin:0; 119 | font-size: 120%; 120 | color: #005; 121 | } 122 | 123 | h4 { 124 | padding-top: 20px; 125 | padding-bottom: 5px; 126 | margin:0; 127 | } 128 | 129 | ul { 130 | margin-block-start: 0.1em; 131 | margin-block-end: 0.1em; 132 | } 133 | 134 | 135 | table { 136 | background-color: transparent; 137 | display: table; 138 | border-color: grey; 139 | line-height: 1.5; 140 | border-spacing: 0; 141 | border-collapse: collapse; 142 | } 143 | 144 | th { 145 | vertical-align: bottom; 146 | border-bottom: 2px solid #ddd; 147 | padding-right: 16px; 148 | padding-left: 0px; 149 | padding-top: 8px; 150 | padding-bottom: 8px; 151 | text-align: left; 152 | } 153 | 154 | td { 155 | vertical-align: top; 156 | padding-right: 16px; 157 | padding-left: 0px; 158 | padding-top: 8px; 159 | padding-bottom: 8px; 160 | } 161 | 162 | .footer { 163 | display: table; 164 | width: calc(100% - 64px); 165 | /* Distance with other content */ 166 | margin-top: 16px; 167 | /*background-color: #a00;*/ 168 | /* internal margin */ 169 | padding-top: 0.5rem; 170 | padding-bottom: 0.5rem; 171 | padding-right: 32px; 172 | padding-left: 32px; 173 | color: #555; 174 | 175 | font-size: 90%; 176 | } 177 | 178 | 179 | .globalsummary-box { 180 | width:90%; 181 | padding:10px; 182 | margin: 20px; 183 | padding-left: 40px; 184 | padding-right: 40px; 185 | background-color: rgba(5,115,255,0.19); 186 | border:1px solid rgba(0,65,127,0.4); 187 | border-radius:4px; 188 | -webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05); 189 | box-shadow:inset 0 1px 1px rgba(0,0,0,.05); 190 | line-height: 1.5; 191 | } 192 | 193 | .submenu-entry { 194 | width:90%; 195 | min-height:140px; 196 | padding:10px; 197 | margin: 20px; 198 | padding-left: 40px; 199 | padding-right: 40px; 200 | background-color:rgba(245,245,245,.19); 201 | border:1px solid rgba(227,227,227,.31); 202 | border-radius:4px; 203 | -webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05); 204 | box-shadow:inset 0 1px 1px rgba(0,0,0,.05); 205 | } 206 | 207 | .classbox { 208 | display: inline-block; 209 | background-color: #777; 210 | padding: 0em .2em 0em; 211 | font-size: 75%; 212 | color: #fff; 213 | border-radius: .25em 214 | } 215 | 216 | /* Tooltip text */ 217 | .classbox .tooltiptext { 218 | visibility: hidden; 219 | background-color: black; 220 | color: #fff; 221 | text-align: center; 222 | padding: 3px; 223 | border-radius: 6px; 224 | 225 | /* Position the tooltip text - see examples below! */ 226 | position: absolute; 227 | z-index: 1; 228 | } 229 | 230 | /* Show the tooltip text when you mouse over the tooltip container */ 231 | .classbox:hover .tooltiptext { 232 | visibility: visible; 233 | } 234 | 235 | .entrypointraw { 236 | color: #777; 237 | } 238 | 239 | 240 | 241 | .badge { 242 | white-space: nowrap; 243 | display: inline-block; 244 | vertical-align: middle; 245 | /*vertical-align: baseline;*/ 246 | font-family: "DejaVu Sans", Verdana, Geneva, sans-serif; 247 | /*font-size: 90%;*/ 248 | } 249 | 250 | span.badge-left { 251 | border-radius: .25rem; 252 | border-top-right-radius: 0; 253 | border-bottom-right-radius: 0; 254 | color: #212529; 255 | background-color: #A2CBFF; 256 | /* color: #ffffff; */ 257 | text-shadow: 1px 1px 1px rgba(0,0,0,0.3); 258 | 259 | padding: .25em .4em; 260 | line-height: 0.8; 261 | text-align: center; 262 | white-space: nowrap; 263 | float: left; 264 | display: block; 265 | } 266 | 267 | span.badge-right { 268 | border-radius: .25rem; 269 | border-top-left-radius: 0; 270 | border-bottom-left-radius: 0; 271 | 272 | color: #fff; 273 | background-color: #343a40; 274 | 275 | padding: .25em .4em; 276 | line-height: 0.8; 277 | text-align: center; 278 | white-space: nowrap; 279 | float: left; 280 | display: block; 281 | } 282 | 283 | .badge-right.light-blue, .badge-left.light-blue { 284 | background-color: #A2CBFF; 285 | color: #212529; 286 | } 287 | 288 | .badge-right.light-red, .badge-left.light-red { 289 | background-color: rgb(255, 162, 162); 290 | color: rgb(43, 14, 14); 291 | } 292 | 293 | .badge-right.red, .badge-left.red { 294 | background-color: #e41a1c; 295 | color: #fff; 296 | } 297 | 298 | .badge-right.blue, .badge-left.blue { 299 | background-color: #377eb8; 300 | color: #fff; 301 | } 302 | 303 | .badge-right.green, .badge-left.green { 304 | background-color: #4daf4a; 305 | color: #fff; 306 | } 307 | 308 | .badge-right.purple, .badge-left.purple { 309 | background-color: #984ea3; 310 | color: #fff; 311 | } 312 | 313 | .badge-right.orange, .badge-left.orange { 314 | background-color: #ff7f00; 315 | color: #fff; 316 | } 317 | 318 | .badge-right.brown, .badge-left.brown { 319 | background-color: #a65628; 320 | color: #fff; 321 | } 322 | 323 | .badge-right.dark-gray, .badge-left.dark-gray { 324 | color: #fff; 325 | background-color: #343a40; 326 | } 327 | 328 | 329 | .badge a { 330 | text-decoration: none; 331 | padding: 0; 332 | border: 0; 333 | color: inherit; 334 | } 335 | 336 | .badge a:visited, .badge a:active { 337 | color: inherit; 338 | } 339 | 340 | .badge a:focus, .badge a:hover { 341 | color: rgba(255,255,255,0.5); 342 | mix-blend-mode: difference; 343 | text-decoration: none; 344 | /* background-color: rgb(192, 219, 255); */ 345 | } 346 | 347 | 348 | .svg-badge { 349 | vertical-align: middle; 350 | } 351 | 352 | .tooltip { 353 | position: relative; 354 | display: inline-block; 355 | border-bottom: 1px dotted black; 356 | } 357 | 358 | .tooltip .tooltiptext { 359 | visibility: hidden; 360 | /* width: 120px; */ 361 | background-color: rgb(255, 247, 175); 362 | color: #000; 363 | text-align: center; 364 | border-radius: 6px; 365 | padding: 5px; 366 | 367 | /* Position the tooltip */ 368 | position: absolute; 369 | z-index: 1; 370 | } 371 | 372 | .tooltip:hover .tooltiptext { 373 | visibility: visible; 374 | } 375 | 376 | ul.provider-info { 377 | list-style: none; 378 | margin-left: 0; 379 | padding-left: 0; 380 | padding-top: 5px; 381 | } 382 | 383 | ul.provider-info li { 384 | padding-left: 1em; 385 | text-indent: -1em; 386 | } 387 | 388 | ul.provider-info li:before { 389 | content: "→"; 390 | padding-right: 5px; 391 | } 392 | 393 | .errors { 394 | margin-top: 1em; 395 | margin-bottom: 2em; 396 | width: 85%; 397 | font-weight: bold; 398 | font-family: monospace; 399 | background-color: Gainsboro; 400 | color: DimGrey; 401 | overflow-x: auto; 402 | overflow-y: auto; 403 | white-space: nowrap; 404 | padding: 1em; 405 | border: 1px solid IndianRed; 406 | } 407 | 408 | .errors a { 409 | text-decoration: underline; 410 | color: IndianRed; 411 | } 412 | 413 | .errors p { 414 | color: DimGrey; 415 | padding-left: 3em; 416 | font-weight: normal; 417 | } 418 | 419 | .property-name { 420 | font-family: monospace; 421 | font-size: 1rem; 422 | color: DarkSlateBlue; 423 | background-color: #f0f0f0; 424 | border: 1px solid #d0d0d0; 425 | border-radius: 5px; 426 | padding: 0.2em 0.5em; 427 | margin: 0.2em; 428 | } 429 | 430 | .property-description { 431 | font-size: 0.9rem; 432 | padding-left: 25px; 433 | padding-top: 0.5em; 434 | } 435 | 436 | .entry-type-name { 437 | font-family: monospace; 438 | font-size: 1.2rem; 439 | font-weight: bold; 440 | background-color: #f0f0f0; 441 | color: DarkOrchid; 442 | border: 1px solid #d0d0d0; 443 | border-radius: 5px; 444 | padding: 0.2em 0.5em; 445 | margin: 0.2em; 446 | } 447 | -------------------------------------------------------------------------------- /make_ghpages/mod/templates/singlepage.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block css %} 4 | 5 | 6 | {% endblock css %} 7 | 8 | {% block body %} 9 |
10 |

11 | OPTIMADE provider "{{attributes.name}}" (id: {{id}}) 12 |

13 | 14 |

< back to the full provider list

15 |

General information

16 |
17 | {% if attributes.description %} 18 |

19 | Short description: {{ attributes.description }} 20 |

21 | {% endif %} 22 | {% if attributes.homepage %} 23 |

24 | Project homepage: {{ attributes.homepage }} 25 |

26 | {% endif %} 27 |

28 | Index Meta-Database URL: 29 | {% if attributes.base_url %} 30 | {{ attributes.base_url }} 31 | {% else %} 32 | This provider did not specify yet a base_url for its OPTIMADE implementation. 33 | {% endif %} 34 |

35 |

36 | Number of structures: {{ "{:,}".format(num_structures) }} 37 |

38 |
39 | 40 |

41 | Detailed information 42 |

43 |
(information checked on {{ last_check_time }})
44 |
45 | 46 | Index metaDB ({% if index_metadb.info_endpoint %}{% endif %}/info{% if index_metadb.info_endpoint %}{% endif %})State of the /info endpoint of the index meta-database 47 | {{ index_metadb.state }} 48 | {% if index_metadb.tooltip_lines %} {% for line in index_metadb.tooltip_lines %}{{ line }}
{% endfor %} {% else %} {{ index_metadb.state }} {% endif %} 49 | 50 | 51 | {% if index_metadb.version %} 52 | 53 | Index metaDB versionVersion of the index meta-database 54 | {{ index_metadb.version }} 55 | 56 | {% endif %} 57 | {% if index_metadb.links_state %} 58 | 59 | Index metaDB ({% if index_metadb.links_endpoint %}{% endif %}/links{% if index_metadb.links_endpoint %}{% endif %})State of the /links endpoint of the index meta-database 60 | {{ index_metadb.links_state }} 61 | {% if index_metadb.links_tooltip_lines %} {% for line in index_metadb.links_tooltip_lines %}{{ line }}
{% endfor %} {% else %} {{ index_metadb.links_state }} {% endif %} 62 | 63 | 64 | {% endif %} 65 | 66 | 67 | {% if index_metadb.subdbs %} 68 |

Databases served by this provider

69 | 148 | {% endif %} 149 |
150 | 151 |
152 | 153 | {% endblock body %} 154 | -------------------------------------------------------------------------------- /make_ghpages/make_pages.py: -------------------------------------------------------------------------------- 1 | """Generate HTML pages for the OPTIMADE providers list.""" 2 | import datetime 3 | import json 4 | import os 5 | import shutil 6 | import signal 7 | import string 8 | import traceback 9 | import urllib.request 10 | from contextlib import contextmanager 11 | 12 | from jinja2 import Environment, PackageLoader, select_autoescape 13 | from optimade import __version__ 14 | from optimade.models import IndexInfoResponse, LinksResponse 15 | from optimade.utils import get_providers 16 | from optimade.validator import ImplementationValidator 17 | from optimade.validator.utils import ResponseError 18 | 19 | # Subfolders 20 | OUT_FOLDER = "out" 21 | STATIC_FOLDER = "static" 22 | HTML_FOLDER = ( 23 | "providers" # Name for subfolder where HTMLs for providers are going to be sitting 24 | ) 25 | TEMPLATES_FOLDER = "templates" 26 | 27 | # Absolute paths 28 | pwd = os.path.split(os.path.abspath(__file__))[0] 29 | STATIC_FOLDER_ABS = os.path.join(pwd, STATIC_FOLDER) 30 | 31 | VALIDATION_TIMEOUT = 600 32 | 33 | 34 | HTTP_USER_AGENT_HEADER = { 35 | "User-Agent": "urllib.request (automated dashboard build for providers.optimade.org)" 36 | } 37 | 38 | 39 | class DashboardTimeoutException(ResponseError): 40 | pass 41 | 42 | 43 | @contextmanager 44 | def time_limit(timeout: float): 45 | """A simple context manager that uses the signal module to raise 46 | an exception if the timeout is exceeded. 47 | 48 | Arguments: 49 | timeout: The desired timeout in seconds. 50 | 51 | """ 52 | 53 | def signal_handler(signal_number, frame): 54 | raise DashboardTimeoutException( 55 | f"Validation timed out after {timeout} seconds. The validation run did not complete and results should be externally verified." 56 | ) 57 | 58 | signal.signal(signal.SIGALRM, signal_handler) 59 | signal.alarm(timeout) 60 | try: 61 | yield 62 | finally: 63 | signal.alarm(0) 64 | 65 | 66 | def extract_url(value): 67 | """To be used in the URLs of the sub databases. 68 | 69 | Indeed, sometimes its a AnyUrl, sometimes a Link(AnyUrl) 70 | """ 71 | try: 72 | u = value.href 73 | except AttributeError: 74 | u = value 75 | return u.strip("/") 76 | 77 | 78 | def get_index_metadb_data(base_url): 79 | """Return some info after inspecting the base_url of this index_metadb.""" 80 | versions_to_test = ["v1", "v0.10", "v0"] 81 | 82 | provider_data = {} 83 | for version in versions_to_test: 84 | info_endpoint = f"{base_url}/{version}/info" 85 | try: 86 | info_req = urllib.request.Request( 87 | info_endpoint, headers=HTTP_USER_AGENT_HEADER 88 | ) 89 | with urllib.request.urlopen(info_req) as url_response: 90 | response_content = url_response.read() 91 | provider_data["info_endpoint"] = info_endpoint 92 | break 93 | except urllib.error.HTTPError as exc: 94 | if exc.code == 404: 95 | continue 96 | else: 97 | provider_data["state"] = "problem" 98 | provider_data[ 99 | "tooltip_lines" 100 | ] = "Generic error while fetching the data:\n{}".format( 101 | traceback.format_exc() 102 | ).splitlines() 103 | provider_data["color"] = "light-red" 104 | return provider_data 105 | else: 106 | # Did not break: no version found 107 | provider_data["state"] = "not found" 108 | provider_data["tooltip_lines"] = [ 109 | "I couldn't find the index meta-database, I tried the following versions: {}".format( 110 | ", ".join(versions_to_test) 111 | ) 112 | ] 113 | provider_data["color"] = "light-red" 114 | return provider_data 115 | 116 | provider_data["state"] = "found" 117 | provider_data["color"] = "green" 118 | provider_data["version"] = version 119 | 120 | provider_data["default_subdb"] = None 121 | # Let's continue, it was found 122 | try: 123 | json_response = json.loads(response_content) 124 | IndexInfoResponse(**json_response) 125 | except Exception: 126 | # Adapt the badge info 127 | provider_data["state"] = "validation error" 128 | provider_data["color"] = "orange" 129 | provider_data[ 130 | "tooltip_lines" 131 | ] = "Error while validating the Index MetaDB:\n{}".format( 132 | traceback.format_exc() 133 | ).splitlines() 134 | provider_data["version"] = version 135 | else: 136 | try: 137 | # For now I use this way of getting it 138 | provider_data["default_subdb"] = json_response["data"]["relationships"][ 139 | "default" 140 | ]["data"]["id"] 141 | except Exception: 142 | # For now, whatever the error, I just ignore it 143 | pass 144 | 145 | links_endpoint = f"{base_url}/{version}/links" 146 | try: 147 | links_req = urllib.request.Request( 148 | links_endpoint, headers=HTTP_USER_AGENT_HEADER 149 | ) 150 | with urllib.request.urlopen(links_req) as url_response: 151 | response_content = url_response.read() 152 | except urllib.error.HTTPError: 153 | provider_data["links_state"] = "problem" 154 | provider_data[ 155 | "links_tooltip_lines" 156 | ] = "Generic error while fetching the /links endpoint:\n{}".format( 157 | traceback.format_exc() 158 | ).splitlines() 159 | provider_data["links_color"] = "light-red" 160 | return provider_data 161 | 162 | provider_data["links_endpoint"] = links_endpoint 163 | provider_data["links_state"] = "found" 164 | provider_data["links_color"] = "green" 165 | 166 | try: 167 | links_json_response = json.loads(response_content) 168 | LinksResponse(**links_json_response) 169 | except Exception: 170 | # Adapt the badge info 171 | provider_data["links_state"] = "validation error" 172 | provider_data["links_color"] = "orange" 173 | provider_data[ 174 | "links_tooltip_lines" 175 | ] = "Error while validating the /links endpoint of the Index MetaDB:\n{}".format( 176 | traceback.format_exc() 177 | ).splitlines() 178 | return provider_data 179 | 180 | # We also filter out any non-child DB link type. 181 | all_linked_dbs = links_json_response["data"] 182 | subdbs = [ 183 | subdb 184 | for subdb in all_linked_dbs 185 | if subdb["attributes"].get("link_type", "UNKNOWN") == "child" 186 | ] 187 | print( 188 | f" [{len(all_linked_dbs)} links found, of which {len(subdbs)} child sub-dbs]" 189 | ) 190 | 191 | # Order putting the default first, and then the rest in alphabetical order (by key) 192 | # Note that False gets before True. 193 | provider_data["subdbs"] = sorted( 194 | subdbs, 195 | key=lambda subdb: (subdb["id"] != provider_data["default_subdb"], subdb["id"]), 196 | ) 197 | 198 | # Count the non-null ones 199 | non_null_subdbs = [ 200 | subdb for subdb in provider_data["subdbs"] if subdb["attributes"]["base_url"] 201 | ] 202 | provider_data["num_non_null_subdbs"] = len(non_null_subdbs) 203 | 204 | provider_data["subdb_validation"] = {} 205 | provider_data["subdb_properties"] = {} 206 | provider_data["num_structures"] = 0 207 | for subdb in non_null_subdbs: 208 | url = subdb["attributes"]["base_url"] 209 | if (aggregate := subdb["attributes"].get("aggregate")) is None: 210 | aggregate = "ok" 211 | if aggregate != "ok": 212 | results = {} 213 | print(f"\t\tSkipping {subdb['id']} as aggregate is set to {aggregate}.") 214 | results["failure_count"] = 0 215 | results["failure_messages"] = [] 216 | results["success_count"] = 0 217 | results["internal_failure_count"] = 0 218 | results["no_aggregate_reason"] = subdb["attributes"].get( 219 | "no_aggregate_reason", "No details given" 220 | ) 221 | properties = {} 222 | 223 | else: 224 | v1_url = url.strip("/") + "/v1" if not url.endswith("/v1") else "" 225 | properties = get_child_properties(v1_url) 226 | results = validate_childdb(v1_url) 227 | results["num_structures"] = _get_structure_count(v1_url) 228 | provider_data["num_structures"] += results["num_structures"] 229 | 230 | results["aggregate"] = aggregate 231 | 232 | provider_data["subdb_validation"][url] = results 233 | provider_data["subdb_properties"][url] = properties 234 | provider_data["subdb_validation"][url]["valid"] = not results.get("failure_count", 0) 235 | # Count errors apart from internal errors 236 | provider_data["subdb_validation"][url]["total_count"] = ( 237 | results.get("success_count", 0) + results.get("failure_count", 0) 238 | ) 239 | try: 240 | ratio = results.get("success_count", 0) / ( 241 | results.get("success_count", 0) + results.get("failure_count", 0) 242 | ) 243 | except ZeroDivisionError: 244 | ratio = 0 245 | # Use the red/green values from the badge css 246 | ratio = 2 * (max(0.5, ratio) - 0.5) 247 | green = (77, 175, 74) 248 | red = (228, 26, 28) 249 | colour = list(green) 250 | 251 | for ind, channel in enumerate(colour): 252 | gradient = red[ind] - green[ind] 253 | colour[ind] += gradient * (1 - ratio) 254 | 255 | colour = [str(int(channel)) for channel in colour] 256 | provider_data["subdb_validation"][url][ 257 | "_validator_results_colour" 258 | ] = f"rgb({','.join(colour)});" 259 | 260 | if provider_data["subdb_validation"][url].get("aggregate", "ok") != "ok": 261 | provider_data["subdb_validation"][url][ 262 | "_validator_results_colour" 263 | ] = "DarkGrey" 264 | 265 | return provider_data 266 | 267 | 268 | def get_html_provider_fname(provider_id): 269 | """Return a valid html filename given the provider ID.""" 270 | valid_characters = set(string.ascii_letters + string.digits + "_-") 271 | 272 | simple_string = "".join(c for c in provider_id if c in valid_characters) 273 | 274 | return "{}.html".format(simple_string) 275 | 276 | 277 | def validate_childdb(url: str) -> dict: 278 | """Run the optimade-python-tools validator on the child database. 279 | 280 | Parameters: 281 | url: the URL of the child database. 282 | 283 | Returns: 284 | dictionary representation of the validation results. 285 | 286 | """ 287 | import dataclasses 288 | from traceback import print_exc 289 | 290 | validator = ImplementationValidator( 291 | base_url=url, 292 | run_optional_tests=False, 293 | verbosity=0, 294 | read_timeout=100, 295 | http_headers={ 296 | "User-Agent": f"optimade-python-tools validator/{__version__} (automated dashboard build for providers.optimade.org)" 297 | }, 298 | ) 299 | 300 | try: 301 | with time_limit(VALIDATION_TIMEOUT): 302 | validator.validate_implementation() 303 | except DashboardTimeoutException: 304 | validator.results.failure_count += 1 305 | validator.results.failures_messages += [ 306 | f"ImplementationValidator for this provider ({url}) timed out after the configured {VALIDATION_TIMEOUT} seconds." 307 | ] 308 | except (Exception, SystemExit): 309 | print_exc() 310 | 311 | return dataclasses.asdict(validator.results) 312 | 313 | def get_child_properties(url: str) -> dict: 314 | """Get the properties served by the child database. 315 | 316 | Parameters: 317 | url: the URL of the child database. 318 | 319 | Returns: 320 | dictionary representation of the properties of the child database, 321 | broken down by entrypoint. 322 | 323 | """ 324 | try: 325 | properties = {} 326 | info_req = urllib.request.Request( 327 | f"{url}/info", headers=HTTP_USER_AGENT_HEADER 328 | ) 329 | with urllib.request.urlopen(info_req) as url_response: 330 | response_content = json.loads(url_response.read()) 331 | entry_types = response_content.get("data", {}).get("attributes", {}).get("entry_types_by_format", {}).get("json", []) 332 | for _type in entry_types: 333 | entry_info_req = urllib.request.Request( 334 | f"{url}/info/{_type}", headers=HTTP_USER_AGENT_HEADER 335 | ) 336 | with urllib.request.urlopen(entry_info_req) as url_response: 337 | response_content = json.loads(url_response.read()) 338 | properties[_type] = response_content.get("data", {}).get("properties", {}) 339 | 340 | return properties 341 | 342 | except Exception as exc: 343 | print(exc) 344 | return {} 345 | 346 | 347 | def _get_structure_count(url: str) -> int: 348 | """Try to get the number of structures hosted at the given URL.""" 349 | try: 350 | structures_req = urllib.request.Request( 351 | f"{url}/structures", headers=HTTP_USER_AGENT_HEADER 352 | ) 353 | with urllib.request.urlopen(structures_req) as url_response: 354 | response_content = json.loads(url_response.read()) 355 | # account for inconsistencies in the metadata by taking largest of available/returned data 356 | return max( 357 | response_content.get("meta", {}).get("data_available", 0), 358 | response_content.get("meta", {}).get("data_returned", 0), 359 | ) 360 | 361 | except Exception as exc: 362 | print(exc) 363 | return 0 364 | 365 | 366 | def make_pages(): 367 | """Create the rendered pages (index, and per-provider detail page).""" 368 | 369 | # Create output folder, copy static files 370 | if os.path.exists(OUT_FOLDER): 371 | shutil.rmtree(OUT_FOLDER) 372 | os.mkdir(OUT_FOLDER) 373 | os.mkdir(os.path.join(OUT_FOLDER, HTML_FOLDER)) 374 | shutil.copytree(STATIC_FOLDER_ABS, os.path.join(OUT_FOLDER, STATIC_FOLDER)) 375 | 376 | env = Environment( 377 | loader=PackageLoader("mod"), 378 | autoescape=select_autoescape(["html", "xml"]), 379 | ) 380 | 381 | env.filters["extract_url"] = extract_url 382 | 383 | providers = get_providers() 384 | if not providers: 385 | raise RuntimeError("Unable to retrieve providers list.") 386 | 387 | last_check_time = datetime.datetime.utcnow().strftime("%A %B %d, %Y at %H:%M UTC") 388 | 389 | all_provider_data = [] 390 | # Create HTML view for each provider 391 | for provider in providers: 392 | provider_data = {"id": provider["id"], "last_check_time": last_check_time} 393 | print(" - {}".format(provider["id"])) 394 | 395 | subpage = os.path.join(HTML_FOLDER, get_html_provider_fname(provider["id"])) 396 | subpage_abspath = os.path.join(OUT_FOLDER, subpage) 397 | 398 | provider_data["subpage"] = subpage 399 | provider_data["attributes"] = provider 400 | 401 | base_url = provider.get("base_url") 402 | 403 | if base_url is None: 404 | provider_data["index_metadb"] = { 405 | "state": "unspecified", 406 | "tooltip_lines": [ 407 | "The provider did not specify a base URL for the Index Meta-Database" 408 | ], 409 | "color": "dark-gray", 410 | } 411 | else: 412 | provider_data["index_metadb"] = {} 413 | try: 414 | index_metadb_data = get_index_metadb_data(base_url) 415 | provider_data["index_metadb"] = index_metadb_data 416 | except Exception as exc: 417 | print(exc) 418 | provider_data["index_metadb"] = { 419 | "state": "unknown", 420 | "tooltip_lines": "Generic error while fetching the data:\n{}".format( 421 | traceback.format_exc() 422 | ).splitlines(), 423 | "color": "orange", 424 | } 425 | 426 | provider_data[ 427 | "title" 428 | ] = f'{provider_data["attributes"].get("name")}: OPTIMADE provider dashboard' 429 | provider_data["num_structures"] = provider_data["index_metadb"].get( 430 | "num_structures", 0 431 | ) 432 | 433 | # Write provider html 434 | provider_html = env.get_template("singlepage.html").render(**provider_data) 435 | with open(subpage_abspath, "w") as f: 436 | f.write(provider_html) 437 | all_provider_data.append(provider_data) 438 | print(" - Page {} generated.".format(subpage)) 439 | 440 | all_data = {} 441 | all_data["providers"] = sorted( 442 | all_provider_data, key=lambda provider: provider["id"] 443 | ) 444 | all_data["globalsummary"] = { 445 | "with_base_url": sum( 446 | 1 for d in all_data["providers"] if d["attributes"].get("base_url") 447 | ), 448 | "num_sub_databases": sum( 449 | [ 450 | provider_data.get("index_metadb", {}).get("num_non_null_subdbs", 0) 451 | for provider_data in all_provider_data 452 | ] 453 | ), 454 | "num_structures": sum(prov["num_structures"] for prov in all_provider_data), 455 | } 456 | 457 | # Write main overview index 458 | print("[main index]") 459 | rendered = env.get_template("main_index.html").render(**all_data) 460 | outfile = os.path.join(OUT_FOLDER, "index.html") 461 | with open(outfile, "w") as f: 462 | f.write(rendered) 463 | print(" - index.html generated") 464 | 465 | 466 | if __name__ == "__main__": 467 | make_pages() 468 | --------------------------------------------------------------------------------