├── .github ├── actions │ └── mike-docs │ │ └── action.yaml └── workflows │ ├── lint-and-test.yaml │ └── release.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── CONTRIBUTING.md ├── _images │ ├── aai-favicon.png │ └── aai-logo-cropped.png ├── _scripts │ └── gen_api_ref_pages.py ├── _styles │ ├── extra.css │ ├── neoteroi-mkdocs.css │ └── theme.css ├── _theme_overrides │ └── main.html ├── cli │ ├── cli.md │ ├── comparisons.md │ ├── fixtures.md │ ├── index.md │ └── pyproject.md ├── guides │ ├── benchmarks.md │ ├── customization.md │ ├── index.md │ ├── organization.md │ └── runners.md ├── index.md ├── quickstart.md └── tutorials │ ├── bq.md │ ├── duckdb.md │ ├── huggingface.md │ ├── index.md │ ├── mnist.md │ ├── prefect.md │ ├── prefect_resources │ ├── deployments.png │ ├── detail_flow_runs.png │ └── flow_runs.png │ ├── streamlit.md │ └── streamlit_resources │ └── initial_ui.png ├── examples ├── bq │ ├── benchmarks.py │ └── bq.py ├── huggingface │ ├── benchmark.py │ ├── runner.py │ └── training.py ├── mnist │ ├── benchmarks.py │ └── mnist.py ├── prefect │ ├── pyproject.toml │ └── src │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── runner.py │ │ └── training.py ├── streamlit │ ├── pyproject.toml │ └── streamlit_example.py └── zenml │ ├── README.md │ ├── benchmarks.py │ └── pipeline.py ├── mkdocs.yml ├── pyproject.toml ├── src └── nnbench │ ├── __init__.py │ ├── __main__.py │ ├── cli.py │ ├── compare.py │ ├── config.py │ ├── context.py │ ├── core.py │ ├── fixtures.py │ ├── py.typed │ ├── reporter │ ├── __init__.py │ ├── console.py │ ├── file.py │ ├── mlflow.py │ ├── sqlite.py │ └── util.py │ ├── runner.py │ ├── types.py │ └── util.py ├── tests ├── __init__.py ├── benchmarks │ ├── argchecks.py │ ├── standard.py │ └── tags.py ├── cli │ ├── __init__.py │ ├── benchmarks │ │ ├── a.py │ │ └── b.py │ ├── conf.py │ └── test_parallel_exec.py ├── conftest.py ├── integration │ ├── __init__.py │ └── test_benchmark_family_memory_consumption.py ├── test_config.py ├── test_context.py ├── test_core.py ├── test_file_reporter.py ├── test_runner.py ├── test_types.py └── test_utils.py ├── uv.lock └── zizmor.yml /.github/actions/mike-docs/action.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/.github/actions/mike-docs/action.yaml -------------------------------------------------------------------------------- /.github/workflows/lint-and-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/.github/workflows/lint-and-test.yaml -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/.github/workflows/release.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/README.md -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.md -------------------------------------------------------------------------------- /docs/_images/aai-favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_images/aai-favicon.png -------------------------------------------------------------------------------- /docs/_images/aai-logo-cropped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_images/aai-logo-cropped.png -------------------------------------------------------------------------------- /docs/_scripts/gen_api_ref_pages.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_scripts/gen_api_ref_pages.py -------------------------------------------------------------------------------- /docs/_styles/extra.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_styles/extra.css -------------------------------------------------------------------------------- /docs/_styles/neoteroi-mkdocs.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_styles/neoteroi-mkdocs.css -------------------------------------------------------------------------------- /docs/_styles/theme.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_styles/theme.css -------------------------------------------------------------------------------- /docs/_theme_overrides/main.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/_theme_overrides/main.html -------------------------------------------------------------------------------- /docs/cli/cli.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/cli/cli.md -------------------------------------------------------------------------------- /docs/cli/comparisons.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/cli/comparisons.md -------------------------------------------------------------------------------- /docs/cli/fixtures.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/cli/fixtures.md -------------------------------------------------------------------------------- /docs/cli/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/cli/index.md -------------------------------------------------------------------------------- /docs/cli/pyproject.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/cli/pyproject.md -------------------------------------------------------------------------------- /docs/guides/benchmarks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/guides/benchmarks.md -------------------------------------------------------------------------------- /docs/guides/customization.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/guides/customization.md -------------------------------------------------------------------------------- /docs/guides/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/guides/index.md -------------------------------------------------------------------------------- /docs/guides/organization.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/guides/organization.md -------------------------------------------------------------------------------- /docs/guides/runners.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/guides/runners.md -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/index.md -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/quickstart.md -------------------------------------------------------------------------------- /docs/tutorials/bq.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/bq.md -------------------------------------------------------------------------------- /docs/tutorials/duckdb.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/duckdb.md -------------------------------------------------------------------------------- /docs/tutorials/huggingface.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/huggingface.md -------------------------------------------------------------------------------- /docs/tutorials/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/index.md -------------------------------------------------------------------------------- /docs/tutorials/mnist.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/mnist.md -------------------------------------------------------------------------------- /docs/tutorials/prefect.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/prefect.md -------------------------------------------------------------------------------- /docs/tutorials/prefect_resources/deployments.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/prefect_resources/deployments.png -------------------------------------------------------------------------------- /docs/tutorials/prefect_resources/detail_flow_runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/prefect_resources/detail_flow_runs.png -------------------------------------------------------------------------------- /docs/tutorials/prefect_resources/flow_runs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/prefect_resources/flow_runs.png -------------------------------------------------------------------------------- /docs/tutorials/streamlit.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/streamlit.md -------------------------------------------------------------------------------- /docs/tutorials/streamlit_resources/initial_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/docs/tutorials/streamlit_resources/initial_ui.png -------------------------------------------------------------------------------- /examples/bq/benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/bq/benchmarks.py -------------------------------------------------------------------------------- /examples/bq/bq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/bq/bq.py -------------------------------------------------------------------------------- /examples/huggingface/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/huggingface/benchmark.py -------------------------------------------------------------------------------- /examples/huggingface/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/huggingface/runner.py -------------------------------------------------------------------------------- /examples/huggingface/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/huggingface/training.py -------------------------------------------------------------------------------- /examples/mnist/benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/mnist/benchmarks.py -------------------------------------------------------------------------------- /examples/mnist/mnist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/mnist/mnist.py -------------------------------------------------------------------------------- /examples/prefect/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/prefect/pyproject.toml -------------------------------------------------------------------------------- /examples/prefect/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/prefect/src/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/prefect/src/benchmark.py -------------------------------------------------------------------------------- /examples/prefect/src/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/prefect/src/runner.py -------------------------------------------------------------------------------- /examples/prefect/src/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/prefect/src/training.py -------------------------------------------------------------------------------- /examples/streamlit/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/streamlit/pyproject.toml -------------------------------------------------------------------------------- /examples/streamlit/streamlit_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/streamlit/streamlit_example.py -------------------------------------------------------------------------------- /examples/zenml/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/zenml/README.md -------------------------------------------------------------------------------- /examples/zenml/benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/zenml/benchmarks.py -------------------------------------------------------------------------------- /examples/zenml/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/examples/zenml/pipeline.py -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/mkdocs.yml -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/nnbench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/__init__.py -------------------------------------------------------------------------------- /src/nnbench/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/__main__.py -------------------------------------------------------------------------------- /src/nnbench/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/cli.py -------------------------------------------------------------------------------- /src/nnbench/compare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/compare.py -------------------------------------------------------------------------------- /src/nnbench/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/config.py -------------------------------------------------------------------------------- /src/nnbench/context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/context.py -------------------------------------------------------------------------------- /src/nnbench/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/core.py -------------------------------------------------------------------------------- /src/nnbench/fixtures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/fixtures.py -------------------------------------------------------------------------------- /src/nnbench/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/nnbench/reporter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/__init__.py -------------------------------------------------------------------------------- /src/nnbench/reporter/console.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/console.py -------------------------------------------------------------------------------- /src/nnbench/reporter/file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/file.py -------------------------------------------------------------------------------- /src/nnbench/reporter/mlflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/mlflow.py -------------------------------------------------------------------------------- /src/nnbench/reporter/sqlite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/sqlite.py -------------------------------------------------------------------------------- /src/nnbench/reporter/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/reporter/util.py -------------------------------------------------------------------------------- /src/nnbench/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/runner.py -------------------------------------------------------------------------------- /src/nnbench/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/types.py -------------------------------------------------------------------------------- /src/nnbench/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/src/nnbench/util.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/benchmarks/argchecks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/benchmarks/argchecks.py -------------------------------------------------------------------------------- /tests/benchmarks/standard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/benchmarks/standard.py -------------------------------------------------------------------------------- /tests/benchmarks/tags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/benchmarks/tags.py -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- 1 | DELAY_SECONDS = 10 2 | -------------------------------------------------------------------------------- /tests/cli/benchmarks/a.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/cli/benchmarks/a.py -------------------------------------------------------------------------------- /tests/cli/benchmarks/b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/cli/benchmarks/b.py -------------------------------------------------------------------------------- /tests/cli/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/cli/conf.py -------------------------------------------------------------------------------- /tests/cli/test_parallel_exec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/cli/test_parallel_exec.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/integration/test_benchmark_family_memory_consumption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/integration/test_benchmark_family_memory_consumption.py -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_config.py -------------------------------------------------------------------------------- /tests/test_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_context.py -------------------------------------------------------------------------------- /tests/test_core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_core.py -------------------------------------------------------------------------------- /tests/test_file_reporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_file_reporter.py -------------------------------------------------------------------------------- /tests/test_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_runner.py -------------------------------------------------------------------------------- /tests/test_types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_types.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/tests/test_utils.py -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/uv.lock -------------------------------------------------------------------------------- /zizmor.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aai-institute/nnbench/HEAD/zizmor.yml --------------------------------------------------------------------------------