├── .coveragerc ├── .flake8 ├── .github └── workflows │ ├── build_and_upload_wheels.yaml │ ├── pip-install-tester.yaml │ └── unit-tests.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── codecov.yml ├── docs ├── Makefile ├── _pygments │ └── style.py ├── analysis_examples.rst ├── basic_tutorial.rst ├── conf.py ├── data_generation.rst ├── developer_guide.rst ├── examples │ ├── read │ │ ├── caliper_cali_query.py │ │ ├── caliper_cali_to_json.py │ │ ├── caliper_caliperreader.py │ │ ├── caliper_caliperreader_cali.py │ │ ├── caliper_json.py │ │ ├── cprofile.py │ │ ├── dag_literal.py │ │ ├── gprof_dot.py │ │ ├── hpctoolkit.py │ │ └── timemory.py │ └── tutorial │ │ ├── hatchet_query_examples.ipynb │ │ └── roundtrip-demo.ipynb ├── favicon.ico ├── getting_started.rst ├── images │ ├── basic-tutorial │ │ ├── 01-dataframe.png │ │ ├── 02-tree.png │ │ ├── 03-dataframe-sorted.png │ │ ├── 04-dataframe-with-new-column.png │ │ ├── 05-tree-inc-time.png │ │ ├── 06-tree-depth.png │ │ ├── 07-tree-invert-colormap.png │ │ ├── 08-dataframe-index-levels.png │ │ ├── 09-dataframe-drop-index-levels.png │ │ ├── 10-graphframe-filter.png │ │ ├── 11-graphframe-squash.png │ │ ├── 12-lulesh-64cores.png │ │ └── 13-divide-graphframes.png │ ├── diff-graph1.png │ ├── diff-graph2.png │ ├── diff-graph3.png │ ├── filter-dataframe.png │ ├── flat-function.png │ ├── jupyter-query-filter.png │ ├── jupyter-tree-overview.png │ ├── lulesh-diff-df.png │ ├── lulesh-line-plot.png │ ├── lulesh-load-imb.png │ ├── lulesh-mpi.png │ ├── lulesh-plot.png │ ├── query-dataframe.png │ ├── sample-dataframe.png │ ├── sample-graph.png │ ├── scale-offset-df.png │ ├── squash-graph.png │ ├── strong-scale-graph1.png │ ├── strong-scale-graph2.png │ ├── strong-scale-graph3.png │ ├── tutorial-output │ │ ├── caliper-cali-dataframe.png │ │ └── caliper-cali-graph.png │ ├── vis-dot.png │ ├── vis-flamegraph.png │ ├── vis-terminal.png │ ├── weak-scale-graph1.png │ ├── weak-scale-graph2.png │ └── weak-scale-graph3.png ├── index.rst ├── llnl.rst ├── publications.rst ├── query_lang.rst ├── requirements.txt ├── slides │ └── 2021-ECPAM-hatchet-tutorial.pdf ├── source │ ├── hatchet.cython_modules.libs.rst │ ├── hatchet.cython_modules.rst │ ├── hatchet.external.roundtrip.roundtrip.rst │ ├── hatchet.external.roundtrip.rst │ ├── hatchet.external.rst │ ├── hatchet.query.rst │ ├── hatchet.readers.rst │ ├── hatchet.rst │ ├── hatchet.util.rst │ ├── hatchet.vis.external.rst │ ├── hatchet.vis.rst │ ├── hatchet.writers.rst │ └── modules.rst └── user_guide.rst ├── hatchet ├── __init__.py ├── cython_modules │ ├── __init__.py │ ├── graphframe_modules.pyx │ ├── libs │ │ └── __init__.py │ └── reader_modules.pyx ├── external │ ├── __init__.py │ ├── console.py │ └── roundtrip │ │ ├── LICENSE │ │ ├── NOTICE │ │ ├── README.md │ │ ├── __init__.py │ │ ├── package.json │ │ ├── pyproject.toml │ │ ├── requirements.txt │ │ ├── roundtrip │ │ ├── __init__.py │ │ ├── manager.py │ │ ├── roundtrip.js │ │ ├── tests │ │ │ ├── data │ │ │ │ ├── test.html │ │ │ │ └── test.js │ │ │ └── rt_test.py │ │ └── version.py │ │ └── setup.py ├── frame.py ├── graph.py ├── graphframe.py ├── node.py ├── query │ ├── __init__.py │ ├── compat.py │ ├── compound.py │ ├── engine.py │ ├── errors.py │ ├── object_dialect.py │ ├── query.py │ └── string_dialect.py ├── readers │ ├── __init__.py │ ├── caliper_native_reader.py │ ├── caliper_reader.py │ ├── cprofile_reader.py │ ├── dataframe_reader.py │ ├── gprof_dot_reader.py │ ├── hdf5_reader.py │ ├── hpctoolkit_reader.py │ ├── hpctoolkit_reader_latest.py │ ├── json_reader.py │ ├── literal_reader.py │ ├── pyinstrument_reader.py │ ├── spotdb_reader.py │ ├── tau_reader.py │ └── timemory_reader.py ├── tests │ ├── caliper.py │ ├── callgrind.py │ ├── conftest.py │ ├── cprofile.py │ ├── data │ │ ├── caliper-cpi-json │ │ │ └── cpi-callpath-profile.json │ │ ├── caliper-example-cali │ │ │ └── example-profile.cali │ │ ├── caliper-lulesh-cali │ │ │ └── lulesh-annotation-profile.cali │ │ ├── caliper-lulesh-json │ │ │ └── lulesh-annotation-profile.json │ │ ├── caliper-ordered-cali-mpi │ │ │ └── 230525-151723_1930647_FSY8b8cKSEEi.cali │ │ ├── caliper-ordered-cali │ │ │ └── 230525-151052_1930517_eWbGeyrlBOPT.cali │ │ ├── caliper-ordered-duplicate-json │ │ │ └── caliper-region-dup-node-order.json │ │ ├── caliper-ordered-duplicate │ │ │ ├── duplicate-node-order-old.cali │ │ │ └── duplicate-node-order.cali │ │ ├── caliper-ordered-json │ │ │ └── caliper-region-node-order.json │ │ ├── caliper-sw4-cuda-activity-cali │ │ │ └── caliper_cuda_activity_profile.cali │ │ ├── caliper-sw4-cuda-activity-profile-cali │ │ │ └── caliper_cuda_activity_profile_summary_v2.cali │ │ ├── caliper-timeseries-cali │ │ │ ├── 230711-163829_2055495_OV0UMVFsW6yp.cali │ │ │ └── timeseries.cali │ │ ├── cprofile-hatchet-pstats │ │ │ ├── cprofile-cycle-py2.pstats │ │ │ └── cprofile-cycle.pstats │ │ ├── gprof2dot-cpi │ │ │ └── callgrind.dot.64042.0.1 │ │ ├── hpctoolkit-allgather-database │ │ │ ├── 1.osu_allgather-000000-000-a8c00471-152577-0.metric-db │ │ │ ├── 1.osu_allgather-000000-001-a8c00471-152577-0.metric-db │ │ │ ├── 1.osu_allgather-000001-000-a8c00471-152578-0.metric-db │ │ │ ├── 1.osu_allgather-000001-001-a8c00471-152578-0.metric-db │ │ │ ├── 1.osu_allgather-000002-000-a8c00471-152579-0.metric-db │ │ │ ├── 1.osu_allgather-000002-001-a8c00471-152579-0.metric-db │ │ │ ├── 1.osu_allgather-000003-000-a8c00471-152580-0.metric-db │ │ │ ├── 1.osu_allgather-000003-001-a8c00471-152580-0.metric-db │ │ │ ├── 1.osu_allgather-000004-000-a8c00471-152581-0.metric-db │ │ │ ├── 1.osu_allgather-000004-001-a8c00471-152581-0.metric-db │ │ │ ├── 1.osu_allgather-000005-000-a8c00471-152582-0.metric-db │ │ │ ├── 1.osu_allgather-000005-001-a8c00471-152582-0.metric-db │ │ │ ├── 1.osu_allgather-000006-000-a8c00471-152583-0.metric-db │ │ │ ├── 1.osu_allgather-000006-001-a8c00471-152583-0.metric-db │ │ │ ├── 1.osu_allgather-000007-000-a8c00471-152584-0.metric-db │ │ │ ├── 1.osu_allgather-000007-001-a8c00471-152584-0.metric-db │ │ │ ├── 1.osu_allgather-000008-000-a8c00471-152585-0.metric-db │ │ │ ├── 1.osu_allgather-000008-001-a8c00471-152585-0.metric-db │ │ │ ├── 1.osu_allgather-000009-000-a8c00471-152586-0.metric-db │ │ │ ├── 1.osu_allgather-000009-001-a8c00471-152586-0.metric-db │ │ │ ├── experiment.xml │ │ │ └── src │ │ │ │ └── osu_allgather.c │ │ ├── hpctoolkit-allgather-graphframe.csv │ │ ├── hpctoolkit-cpi-database │ │ │ ├── 1.cpi-000000-000-a8c00270-160443-0.metric-db │ │ │ ├── 1.cpi-000001-000-a8c00270-160444-0.metric-db │ │ │ ├── 1.cpi-000002-000-a8c00270-160445-0.metric-db │ │ │ ├── 1.cpi-000003-000-a8c00270-160446-0.metric-db │ │ │ ├── experiment.xml │ │ │ └── src │ │ │ │ └── cpi.c │ │ ├── hpctoolkit-cpi-graphframe.csv │ │ ├── hpctoolkit-gamess │ │ │ ├── FORMATS.md │ │ │ ├── cct.db │ │ │ ├── meta.db │ │ │ ├── metrics │ │ │ │ ├── METRICS.yaml.ex │ │ │ │ └── default.yaml │ │ │ ├── profile.db │ │ │ └── trace.db │ │ ├── json │ │ │ └── hatchet-graph-literal.json │ │ ├── pyinstrument-hatchet-json │ │ │ └── pyinstrument-hatchet-profile.json │ │ ├── spotdb │ │ │ ├── 0.cali │ │ │ ├── 1.cali │ │ │ ├── 2.cali │ │ │ └── 3.cali │ │ ├── tau-cpi-mpi │ │ │ ├── profile.0.0.0 │ │ │ ├── profile.1.0.0 │ │ │ ├── profile.2.0.0 │ │ │ └── profile.3.0.0 │ │ └── timemory │ │ │ └── wall.tree.json │ ├── dataframe_ops.py │ ├── executable.py │ ├── frame.py │ ├── graph.py │ ├── graph_literal.py │ ├── graph_ops.py │ ├── graphframe.py │ ├── hpctoolkit.py │ ├── hpctoolkit_latest.py │ ├── json_test.py │ ├── node.py │ ├── profiler.py │ ├── pyinstrument.py │ ├── query.py │ ├── query_compat.py │ ├── spotdb_test.py │ ├── tau.py │ ├── timemory_func.py │ └── timemory_test.py ├── util │ ├── __init__.py │ ├── colormaps.py │ ├── config.py │ ├── deprecated.py │ ├── dot.py │ ├── executable.py │ ├── profiler.py │ └── timer.py ├── version.py ├── vis │ ├── README.md │ ├── __init__.py │ ├── external │ │ └── __init__.py │ ├── loader.py │ ├── package.json │ ├── requirements.txt │ ├── scripts │ │ ├── boxplot.js │ │ ├── cct.js │ │ ├── cct │ │ │ ├── cct_chart_view.js │ │ │ ├── cct_color_manager.js │ │ │ ├── cct_controller.js │ │ │ ├── cct_globals.js │ │ │ ├── cct_menu_view.js │ │ │ ├── cct_model.js │ │ │ ├── cct_repr.js │ │ │ ├── cct_scented_slider_popup.js │ │ │ ├── cct_stats.js │ │ │ └── cct_tooltip_view.js │ │ └── utils │ │ │ └── view.js │ ├── static │ │ ├── boxplot_bundle.html │ │ ├── boxplot_bundle.js │ │ ├── cct_bundle.html │ │ └── cct_bundle.js │ ├── static_fixer.py │ ├── styles │ │ ├── boxplot.css │ │ └── cct.css │ ├── templates │ │ ├── boxplot.html │ │ └── cct.html │ └── webpack.config.js └── writers │ ├── __init__.py │ ├── dataframe_writer.py │ └── hdf5_writer.py ├── install.sh ├── license.py ├── logo-hex.png ├── logo.png ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── screenshot.png └── setup.py /.coveragerc: -------------------------------------------------------------------------------- 1 | # -*- conf -*- 2 | # .coveragerc to control coverage.py 3 | [run] 4 | parallel = True 5 | branch = True 6 | data_file = ./.coverage/coverage 7 | source = . 8 | omit = 9 | hatchet/tests/* 10 | docs/examples/* 11 | 12 | [report] 13 | # Regexes for lines to exclude from consideration 14 | exclude_lines = 15 | # Have to re-enable the standard pragma 16 | pragma: no cover 17 | 18 | # Don't complain about missing debug-only code: 19 | def __repr__ 20 | if self\.debug 21 | 22 | # Don't complain if tests don't hit defensive assertion code: 23 | raise AssertionError 24 | raise NotImplementedError 25 | 26 | # Don't complain if non-runnable code isn't run: 27 | if 0: 28 | if False: 29 | if __name__ == .__main__.: 30 | 31 | ignore_errors = True 32 | 33 | [html] 34 | directory = htmlcov 35 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # -*- conf -*- 2 | # flake8 settings for hatchet 3 | # 4 | # These are the flake8 settings recommended by Black 5 | # See https://github.com/psf/black 6 | # 7 | [flake8] 8 | max-line-length = 88 9 | select = C,E,F,W,B,B950 10 | ignore = E501,W503,E203 11 | builtins = IPython 12 | -------------------------------------------------------------------------------- /.github/workflows/build_and_upload_wheels.yaml: -------------------------------------------------------------------------------- 1 | name: Build wheels 2 | 3 | on: 4 | # Uncomment for testing through a PR 5 | # pull_request: 6 | # branches: [develop, releases/**] 7 | workflow_dispatch: 8 | release: 9 | types: 10 | - published 11 | push: 12 | tags: 13 | - 'v**' 14 | 15 | jobs: 16 | 17 | build_wheels_linux_3: 18 | name: Build wheels for Linux 19 | runs-on: ubuntu-22.04 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Set up QEMU to support non-x86 architectures 24 | uses: docker/setup-qemu-action@v2 25 | with: 26 | platforms: all 27 | 28 | - uses: pypa/cibuildwheel@v2.18 29 | env: 30 | CIBW_SKIP: pp* *musllinux* 31 | CIBW_ARCHS_LINUX: auto aarch64 ppc64le 32 | 33 | - uses: actions/upload-artifact@v4 34 | with: 35 | name: hatchet_build_artifacts_wheels_linux_3 36 | path: ./wheelhouse/*.whl 37 | 38 | build_sdist: 39 | name: Build sdist 40 | runs-on: ubuntu-22.04 41 | steps: 42 | - uses: actions/checkout@v4 43 | 44 | - name: Get PyPA build 45 | run: python -m pip install build 46 | 47 | - name: Build sdist 48 | run: python -m build -s 49 | 50 | - uses: actions/upload-artifact@v4 51 | with: 52 | name: hatchet_build_artifacts_sdist 53 | path: dist/*.tar.gz 54 | 55 | test_upload_to_pypi: 56 | needs: 57 | - build_wheels_linux_3 58 | # - build_wheels_macos_36_37 59 | # - build_wheels_macos_38_plus 60 | - build_sdist 61 | runs-on: ubuntu-22.04 62 | if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') 63 | steps: 64 | - uses: actions/download-artifact@v4 65 | with: 66 | pattern: hatchet_build_artifacts_* 67 | merge-multiple: true 68 | path: dist 69 | 70 | - name: Check build artifacts 71 | run: | 72 | ls -lah dist 73 | 74 | - uses: pypa/gh-action-pypi-publish@release/v1 75 | with: 76 | user: __token__ 77 | password: ${{ secrets.TEST_PYPI_API_TOKEN }} 78 | repository_url: https://test.pypi.org/legacy/ 79 | 80 | upload_to_pypi: 81 | needs: 82 | - build_wheels_linux_3 83 | # - build_wheels_macos_36_37 84 | # - build_wheels_macos_38_plus 85 | - build_sdist 86 | runs-on: ubuntu-22.04 87 | if: (github.event_name == 'release' && github.event.action == 'published') || github.event_name == 'workflow_dispatch' 88 | steps: 89 | - uses: actions/download-artifact@v4 90 | with: 91 | pattern: hatchet_build_artifacts_* 92 | merge-multiple: true 93 | path: dist 94 | 95 | - name: Check build artifacts 96 | run: | 97 | ls -lah dist 98 | 99 | - uses: pypa/gh-action-pypi-publish@release/v1 100 | with: 101 | user: __token__ 102 | password: ${{ secrets.PYPI_API_TOKEN }} 103 | # Uncomment the line below if you want to upload to PyPI 104 | # repository_url: https://test.pypi.org/legacy/ 105 | -------------------------------------------------------------------------------- /.github/workflows/pip-install-tester.yaml: -------------------------------------------------------------------------------- 1 | name: pip install tester 2 | 3 | on: 4 | # Uncomment these lines for testing 5 | # push: 6 | # branches: [ develop ] 7 | # pull_request: 8 | # branches: [ develop, releases/** ] 9 | # Uncomment these lines for production 10 | schedule: 11 | - cron: '0 0 1 JAN,MAY,AUG,NOV *' 12 | 13 | jobs: 14 | pip_test: 15 | 16 | runs-on: ${{ matrix.os }} 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | os: [ubuntu-22.04] 21 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 22 | hatchet-version: ["2022.2.0"] 23 | 24 | steps: 25 | - uses: actions/checkout@v4 26 | with: 27 | ref: ${{ format('v{0}', matrix.hatchet-version) }} 28 | 29 | - name: Set up Python ${{ matrix.python-version }} 30 | uses: actions/setup-python@v5 31 | with: 32 | python-version: ${{ matrix.python-version }} 33 | 34 | - name: Build unit test dependencies 35 | run: | 36 | python -m pip install --upgrade pip pytest 37 | 38 | - name: Build Python 3 dependencies 39 | run: | 40 | python -m pip install tables 41 | 42 | - name: Install Hatchet 43 | env: 44 | HATCHET_VERSION: ${{ matrix.hatchet-version }} 45 | run: | 46 | curr_dir=$(pwd) 47 | cd ~ 48 | python -m pip install llnl-hatchet==$HATCHET_VERSION 49 | cd $curr_dir 50 | 51 | - name: Show packages 52 | run: | 53 | python -m pip list 54 | 55 | - name: Prep tests 56 | env: 57 | HATCHET_VERSION: ${{ matrix.hatchet-version }} 58 | PYTHON_VERSION: ${{ matrix.python-version }} 59 | run: | 60 | cp -r ./hatchet/tests ~ 61 | test_dir="tests-$HATCHET_VERSION-$PYTHON_VERSION" 62 | mv ~/tests ~/$test_dir 63 | cat > ~/$test_dir/pytest.ini << EOF 64 | [pytest] 65 | addopts = --durations=20 -ra 66 | testpaths = . 67 | python_files = *.py 68 | EOF 69 | 70 | - name: Test Hatchet 71 | env: 72 | HATCHET_VERSION: ${{ matrix.hatchet-version }} 73 | PYTHON_VERSION: ${{ matrix.python-version }} 74 | run: | 75 | curr_dir=$(pwd) 76 | cd ~ 77 | test_dir="tests-$HATCHET_VERSION-$PYTHON_VERSION" 78 | pytest -vv $test_dir 79 | cd $curr_dir 80 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yaml: -------------------------------------------------------------------------------- 1 | name: unit tests 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop, releases/** ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | # TODO: add macos-latest 16 | os: [ubuntu-22.04] 17 | # TODO: add pypy2, pypy3 18 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v5 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Install Python3 dependencies 29 | run: | 30 | python -m pip install --upgrade pip pytest 31 | pip install -r requirements.txt 32 | # Optional Dependency for HDF Checkpointing 33 | pip install tables 34 | python setup.py install 35 | python setup.py build_ext --inplace 36 | python -m pip list 37 | 38 | - name: Update Black 39 | if: ${{ matrix.python-version == 3.9 }} 40 | run: | 41 | pip install flake8-pytest-importorskip 42 | pip install --upgrade click==8.0.4 43 | pip install black==24.4.2 44 | pip install flake8==4.0.1 45 | 46 | - name: Lint and Format Check with Flake8 and Black 47 | if: ${{ matrix.python-version == 3.9 }} 48 | run: | 49 | black --diff --check . 50 | flake8 51 | 52 | - name: Check License Headers 53 | run: | 54 | python license.py verify 55 | 56 | - name: Install coverage tools 57 | run: | 58 | pip install codecov 59 | pip install pytest-cov 60 | 61 | - name: Clone Caliper 62 | uses: actions/checkout@v4 63 | with: 64 | repository: LLNL/Caliper 65 | path: Caliper 66 | token: ${{ secrets.GITHUB_TOKEN }} 67 | 68 | - name: Build Caliper 69 | working-directory: Caliper 70 | run: | 71 | echo -e "PWD: $(pwd)" 72 | mkdir build && mkdir install 73 | cd build 74 | cmake -DCMAKE_INSTALL_PREFIX=../install ../ 75 | make VERBOSE=1 76 | make install 77 | export PATH=$(pwd)/../install/bin:${PATH} 78 | echo -e "PATH=${PATH}" 79 | cd $GITHUB_WORKSPACE 80 | echo -e "PWD:${PWD}" 81 | which cali-query 82 | 83 | - name: Test Caliper and Timemory Support with pytest 84 | run: | 85 | PYTHONPATH=. $(which pytest) --cov=./ --cov-report=xml 86 | 87 | - name: Upload coverage to Codecov 88 | uses: codecov/codecov-action@v4 89 | env: 90 | CODECOV_TOKEN: ${{ secrets.HATCHET_CODECOV_TOKEN }} 91 | with: 92 | directory: ./coverage/reports 93 | env_vars: OS,PYTHON 94 | files: /home/runner/work/hatchet/hatchet/coverage.xml 95 | flags: unittests 96 | verbose: true 97 | fail_ci_if_error: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .cache 3 | .pytest_cache 4 | .ipynb_checkpoints 5 | 6 | build 7 | docs/_build 8 | hatchet/cython_modules/libs/graphframe_modules.*.so 9 | hatchet/cython_modules/libs/reader_modules.*.so 10 | hatchet/cython_modules/*.c 11 | hatchet/vis/*node_modules* 12 | hatchet/vis/static/*_bundle* 13 | *package-lock.json 14 | 15 | dist/ 16 | llnl_hatchet.egg-info/ 17 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/psf/black 3 | rev: stable 4 | hooks: 5 | - id: black 6 | language_version: python3.6 7 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | title: Hatchet 3 | message: "If you use Hatchet, please cite it as below." 4 | repository-code: https://github.com/LLNL/hatchet 5 | preferred-citation: 6 | type: conference-paper 7 | doi: 10.1145/3295500.3356219 8 | url: https://github.com/LLNL/hatchet 9 | authors: 10 | - family-names: Bhatele 11 | given-names: Abhinav 12 | - family-names: Brink 13 | given-names: Stephanie 14 | - family-names: Gamblin 15 | given-names: Todd 16 | title: "Hatchet: Pruning the Overgrowth in Parallel Profiles" 17 | conference: 18 | name: "International Conference for High Performance Computing, Networking, Storage and Analysis (SC '19)" 19 | city: "Denver" 20 | region: "Colorado" 21 | country: "USA" 22 | date-start: 2019-11-17 23 | date-end: 2019-11-22 24 | year: 2019 25 | notes: LLNL-CODE-741008 26 | publisher: 27 | name: ACM 28 | city: "New York" 29 | region: "New York" 30 | country: "USA" 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2023, Lawrence Livermore National Security, LLC. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a 4 | copy of this software and associated documentation files (the "Software"), 5 | to deal in the Software without restriction, including without limitation 6 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 7 | and/or sell copies of the Software, and to permit persons to whom the 8 | Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 18 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include NOTICE 3 | include hatchet/vis/package.json 4 | include hatchet/vis/webpack.config.js 5 | recursive-include hatchet/vis/scripts * 6 | recursive-include hatchet/vis/static * 7 | recursive-include hatchet/vis/templates * 8 | recursive-include hatchet/vis/styles * 9 | recursive-include hatchet/external/roundtrip * 10 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | round: down 3 | range: 65...100 4 | 5 | ignore: 6 | - hatchet/tests/* 7 | - docs/examples/* 8 | 9 | comment: 10 | layout: "header, diff, changes, tree" 11 | behavior: new 12 | require_changes: false 13 | require_base: false 14 | require_head: true 15 | after_n_builds: 5 16 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | # Minimal makefile for Sphinx documentation 7 | # 8 | 9 | # You can set these variables from the command line. 10 | SPHINXOPTS = 11 | SPHINXBUILD = sphinx-build 12 | SPHINXPROJ = hatchet 13 | SOURCEDIR = . 14 | BUILDDIR = _build 15 | 16 | # Put it first so that "make" without argument is like "make help". 17 | help: 18 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 19 | 20 | .PHONY: help Makefile 21 | 22 | # Catch-all target: route all unknown targets to Sphinx using the new 23 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 24 | %: Makefile 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 26 | -------------------------------------------------------------------------------- /docs/_pygments/style.py: -------------------------------------------------------------------------------- 1 | # The name of the Pygments (syntax highlighting) style to use. 2 | from pygments.styles.default import DefaultStyle 3 | from pygments.token import Generic 4 | 5 | 6 | # modifications to the default style 7 | class HatchetStyle(DefaultStyle): 8 | styles = DefaultStyle.styles.copy() 9 | background_color = "#f4f4f8" 10 | styles[Generic.Output] = "#355" 11 | styles[Generic.Prompt] = "bold #346ec9" 12 | -------------------------------------------------------------------------------- /docs/developer_guide.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | Hatchet Project Developers. See the top-level LICENSE file for details. 3 | 4 | SPDX-License-Identifier: MIT 5 | 6 | *************** 7 | Developer Guide 8 | *************** 9 | 10 | Contributing to Hatchet 11 | ======================= 12 | 13 | If you want to contribute a new data reader, feature, or bugfix to Hatchet, 14 | please read below. This guide discusses the contributing workflow used in the 15 | Hatchet project, and the granularity of pull requests (PRs). 16 | 17 | Branches 18 | -------- 19 | The main branch in Hatchet that has the latest contributions is named 20 | ``develop``. All pull requests should start from ``develop`` and target 21 | ``develop``. 22 | 23 | There is a branch for each minor release series. Release branches originate 24 | from ``develop`` and have tags for each revision release in the series. 25 | 26 | Continuous Integration 27 | ---------------------- 28 | 29 | Hatchet uses `GitHub Actions `_ for 30 | Continuous Integration testing. This means that every time you submit a pull 31 | request, a series of tests are run to make sure you didn’t accidentally 32 | introduce any bugs into Hatchet. Your PR will not be accepted until it passes 33 | all of these tests. 34 | 35 | Currently, we perform 2 types of tests: 36 | 37 | Unit tests 38 | ^^^^^^^^^^ 39 | 40 | Unit tests ensure that Hatchet's core API is working as expected. If you add a 41 | new data reader or new functionality to the Hatchet API, you should add unit 42 | tests that provide adequate coverage for your code. You should also check that 43 | your changes pass all unit tests. You can do this by typing: 44 | 45 | .. code-block:: console 46 | 47 | $ pytest 48 | 49 | Style tests 50 | ^^^^^^^^^^^ 51 | 52 | Hatchet uses `Flake8 `_ to test for `PEP 8 53 | `_ compliance. You can check for 54 | compliance using: 55 | 56 | .. code-block:: console 57 | 58 | $ flake8 59 | 60 | Hatchet also uses python `Black `_ for 61 | code formatting. Format your files using: 62 | 63 | .. code-block:: console 64 | 65 | $ black -t 66 | 67 | Contributing Workflow 68 | --------------------- 69 | 70 | Hatchet is being actively developed, so the ``develop`` branch in Hatchet has 71 | new pull requests being merged often. The recommended way to contribute a pull 72 | request is to fork the Hatchet repo in your own space (if you already have a 73 | fork, make sure is it up-to-date), and then create a new branch off of 74 | ``develop``. 75 | 76 | We prefer that commits pertaining to different components of Hatchet (specific 77 | readers, the core graphframe API, query language, vis tools, etc.) prefix the 78 | component name in the commit message (for example ``: descriptive 79 | message``. 80 | 81 | GitHub provides a detailed `tutorial 82 | `_ 83 | on creating pull requests. 84 | -------------------------------------------------------------------------------- /docs/examples/read/caliper_cali_query.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | 10 | 11 | if __name__ == "__main__": 12 | # Path to caliper cali file. 13 | cali_file = ( 14 | "../../../hatchet/tests/data/caliper-lulesh-cali/lulesh-annotation-profile.cali" 15 | ) 16 | 17 | # Setup desired cali query. 18 | grouping_attribute = "function" 19 | default_metric = "sum(sum#time.duration),inclusive_sum(sum#time.duration)" 20 | query = "select function,%s group by %s format json-split" % ( 21 | default_metric, 22 | grouping_attribute, 23 | ) 24 | 25 | # Use hatchet's ``from_caliper`` API with the path to the cali file and the 26 | # query. This API will internally run ``cali-query`` on this file to 27 | # produce a json-split stream. The result is stored into Hatchet's 28 | # GraphFrame. 29 | gf = ht.GraphFrame.from_caliper(cali_file, query) 30 | 31 | # Printout the DataFrame component of the GraphFrame. 32 | print(gf.dataframe) 33 | 34 | # Printout the graph component of the GraphFrame. 35 | # Use "time (inc)" as the metric column to be displayed 36 | print(gf.tree(metric_column="time (inc)")) 37 | -------------------------------------------------------------------------------- /docs/examples/read/caliper_cali_to_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import subprocess 9 | import hatchet as ht 10 | 11 | 12 | if __name__ == "__main__": 13 | # Path to caliper cali file. 14 | cali_file = ( 15 | "../../../hatchet/tests/data/caliper-lulesh-cali/lulesh-annotation-profile.cali" 16 | ) 17 | 18 | # Setup desired cali query. 19 | cali_query = "cali-query" 20 | grouping_attribute = "function" 21 | default_metric = "sum(sum#time.duration),inclusive_sum(sum#time.duration)" 22 | query = "select function,%s group by %s format json-split" % ( 23 | default_metric, 24 | grouping_attribute, 25 | ) 26 | 27 | # Use ``cali-query`` here to produce the json-split stream. 28 | cali_json = subprocess.Popen( 29 | [cali_query, "-q", query, cali_file], stdout=subprocess.PIPE 30 | ) 31 | 32 | # Use hatchet's ``from_caliper`` API with the resulting json-split. 33 | # The result is stored into Hatchet's GraphFrame. 34 | gf = ht.GraphFrame.from_caliper(cali_json.stdout) 35 | 36 | # Printout the DataFrame component of the GraphFrame. 37 | print(gf.dataframe) 38 | 39 | # Printout the graph component of the GraphFrame. 40 | # Use "time (inc)" as the metric column to be displayed 41 | print(gf.tree(metric_column="time (inc)")) 42 | -------------------------------------------------------------------------------- /docs/examples/read/caliper_caliperreader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import caliperreader as cr 9 | 10 | import hatchet as ht 11 | 12 | 13 | if __name__ == "__main__": 14 | # Path to caliper cali file. 15 | cali_file = "../../../hatchet/tests/data/caliper-example-cali/example-profile.cali" 16 | 17 | r = cr.CaliperReader() 18 | r.read(cali_file) 19 | 20 | gf = ht.GraphFrame.from_caliperreader(r) 21 | 22 | # Printout the DataFrame component of the GraphFrame. 23 | print(gf.dataframe) 24 | 25 | # Printout the graph component of the GraphFrame with the specified metric. 26 | print(gf.tree(metric_column="avg#inclusive#sum#time.duration")) 27 | -------------------------------------------------------------------------------- /docs/examples/read/caliper_caliperreader_cali.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | 10 | 11 | if __name__ == "__main__": 12 | # Path to caliper cali file. 13 | cali_file = "../../../hatchet/tests/data/caliper-example-cali/example-profile.cali" 14 | 15 | gf = ht.GraphFrame.from_caliperreader(cali_file) 16 | 17 | # Printout the DataFrame component of the GraphFrame. 18 | print(gf.dataframe) 19 | 20 | # Printout the graph component of the GraphFrame with the specified metric. 21 | print(gf.tree(metric_column="avg#inclusive#sum#time.duration")) 22 | -------------------------------------------------------------------------------- /docs/examples/read/caliper_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | 10 | 11 | if __name__ == "__main__": 12 | # Path to caliper json-split file. 13 | json_file = "../../../hatchet/tests/data/caliper-cpi-json/cpi-callpath-profile.json" 14 | 15 | # Use hatchet's ``from_caliper`` API with the resulting json-split. 16 | # The result is stored into Hatchet's GraphFrame. 17 | gf = ht.GraphFrame.from_caliper(json_file) 18 | 19 | # Printout the DataFrame component of the GraphFrame. 20 | print(gf.dataframe) 21 | 22 | # Printout the graph component of the GraphFrame. 23 | # Because no metric parameter is specified, ``time`` is used by default. 24 | print(gf.tree()) 25 | -------------------------------------------------------------------------------- /docs/examples/read/cprofile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | from sys import version_info as python_version 10 | 11 | 12 | if __name__ == "__main__": 13 | # Path to pstats file. 14 | # A pstats file produced with a particualar version of python (2 or 3) must be read in with that version 15 | if python_version[0] == 2: 16 | pstats_file = "../../../hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle-py2.pstats" 17 | elif python_version[0] == 3: 18 | pstats_file = ( 19 | "../../../hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle.pstats" 20 | ) 21 | 22 | # Use hatchet's ``from_cprofile`` API to read in the pstats file. The result 23 | # is stored into Hatchet's GraphFrame. 24 | gf = ht.GraphFrame.from_cprofile(pstats_file) 25 | 26 | # Printout the DataFrame component of the GraphFrame. 27 | print(gf.dataframe) 28 | 29 | # Printout the graph component of the GraphFrame. 30 | # Because no metric parameter is specified, ``time`` is used by default. 31 | print(gf.tree()) 32 | -------------------------------------------------------------------------------- /docs/examples/read/gprof_dot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | 10 | 11 | if __name__ == "__main__": 12 | # Path to DOT file. 13 | dot_file = "../../../hatchet/tests/data/gprof2dot-cpi/callgrind.dot.64042.0.1" 14 | 15 | # Use hatchet's ``from_gprof_dot`` API to read in the DOT file. The result 16 | # is stored into Hatchet's GraphFrame. 17 | gf = ht.GraphFrame.from_gprof_dot(dot_file) 18 | 19 | # Printout the DataFrame component of the GraphFrame. 20 | print(gf.dataframe) 21 | 22 | # Printout the graph component of the GraphFrame. 23 | # Because no metric parameter is specified, ``time`` is used by default. 24 | print(gf.tree()) 25 | -------------------------------------------------------------------------------- /docs/examples/read/hpctoolkit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import hatchet as ht 9 | 10 | 11 | if __name__ == "__main__": 12 | # Path to HPCToolkit database directory. 13 | dirname = "../../../hatchet/tests/data/hpctoolkit-cpi-database" 14 | 15 | # Use hatchet's ``from_hpctoolkit`` API to read in the HPCToolkit database. 16 | # The result is stored into Hatchet's GraphFrame. 17 | gf = ht.GraphFrame.from_hpctoolkit(dirname) 18 | 19 | # Printout the DataFrame component of the GraphFrame. 20 | print(gf.dataframe) 21 | 22 | # Printout the graph component of the GraphFrame. 23 | # Use "time (inc)" as the metric column to be displayed 24 | print(gf.tree(metric_column="time (inc)")) 25 | -------------------------------------------------------------------------------- /docs/examples/read/timemory.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 4 | # Hatchet Project Developers. See the top-level LICENSE file for details. 5 | # 6 | # SPDX-License-Identifier: MIT 7 | 8 | import argparse 9 | import hatchet as ht 10 | 11 | if __name__ == "__main__": 12 | 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument( 15 | "-m", "--metric", default="sum", type=str, help="Metric to display" 16 | ) 17 | args, argv = parser.parse_known_args() 18 | files = argv[:] 19 | if not files: 20 | files.append("../../../hatchet/tests/data/timemory/wall.tree.json") 21 | 22 | for json_file in files: 23 | # Use hatchet's ``from_timemory`` API with the hierarchical json output. 24 | # The result is stored into Hatchet's GraphFrame. 25 | gf = ht.GraphFrame.from_timemory(json_file) 26 | 27 | # Printout the DataFrame component of the GraphFrame. 28 | print(gf.dataframe) 29 | 30 | # Printout the graph component of the GraphFrame. 31 | # one dimensional components use standardize metric labels, 32 | # e.g., sum, sum.inc, mean, mean.inc, etc. 33 | # but multi-dimensional data, such as the current-peak-rss 34 | # (which reports the peak RSS at the start of the marker 35 | # and at the end) and hardware-counter data, create 36 | # unique suffixes, e.g. sum.start-peak-rss, 37 | # sum.start-peak-rss, et.c 38 | print(gf.tree(args.metric)) 39 | -------------------------------------------------------------------------------- /docs/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/favicon.ico -------------------------------------------------------------------------------- /docs/getting_started.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | Hatchet Project Developers. See the top-level LICENSE file for details. 3 | 4 | SPDX-License-Identifier: MIT 5 | 6 | *************** 7 | Getting Started 8 | *************** 9 | 10 | Prerequisites 11 | ============= 12 | 13 | Hatchet has the following minimum requirements, which must be installed before Hatchet is run: 14 | 15 | #. Python 2 (2.7) or 3 (3.5 - 3.8) 16 | #. matplotlib 17 | #. pydot 18 | #. numpy, and 19 | #. pandas 20 | 21 | Hatchet is available on `GitHub `_. 22 | 23 | 24 | Installation 25 | ============ 26 | 27 | You can get hatchet from its `GitHub repository 28 | `_ using this command: 29 | 30 | .. code-block:: console 31 | 32 | $ git clone https://github.com/llnl/hatchet.git 33 | 34 | This will create a directory called ``hatchet``. 35 | 36 | Install and Build Hatchet 37 | ------------------------- 38 | 39 | To build hatchet and update your PYTHONPATH, run the following shell script 40 | from the hatchet root directory: 41 | 42 | .. code-block:: console 43 | 44 | $ source ./install.sh 45 | 46 | Note: The ``source`` keyword is required to update your PYTHONPATH environment 47 | variable. It is not necessary if you have already manually added the hatchet 48 | directory to your PYTHONPATH. 49 | 50 | Alternatively, you can install hatchet using pip: 51 | 52 | .. code-block:: console 53 | 54 | $ pip install llnl-hatchet 55 | 56 | Check Installation 57 | ------------------ 58 | 59 | After installing hatchet, you should be able to import hatchet when running the Python interpreter in interactive mode: 60 | 61 | .. code-block:: console 62 | 63 | $ python 64 | Python 3.7.4 (default, Jul 11 2019, 01:08:00) 65 | [Clang 10.0.1 (clang-1001.0.46.4)] on darwin 66 | Type "help", "copyright", "credits" or "license" for more information. 67 | >>> 68 | 69 | Typing ``import hatchet`` at the prompt should succeed without any error 70 | messages: 71 | 72 | .. code-block:: console 73 | 74 | >>> import hatchet 75 | >>> 76 | 77 | Supported data formats 78 | ====================== 79 | 80 | Currently, hatchet supports the following data formats as input: 81 | 82 | * `HPCToolkit `_ database: This is generated 83 | by using ``hpcprof-mpi`` to post-process the raw measurements directory 84 | output by HPCToolkit. 85 | * Caliper `Cali `_ file: 86 | This is the format in which caliper outputs raw performance data by default. 87 | * Caliper `Json-split 88 | `_ file: This is 89 | generated by either running cali-query on the raw caliper data or by enabling 90 | the mpireport service when using caliper. 91 | * `DOT `_ format: This is 92 | generated by using `gprof2dot `_ on 93 | ``gprof`` or ``callgrind`` output. 94 | * String literal: Hatchet can read as input a list of dictionaries that 95 | represents a graph. 96 | * List: Hatchet can also read a list of lists that represents a graph. 97 | 98 | For more details on the different input file formats, refer to the 99 | :doc:`User Guide `. 100 | -------------------------------------------------------------------------------- /docs/images/basic-tutorial/01-dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/01-dataframe.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/02-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/02-tree.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/03-dataframe-sorted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/03-dataframe-sorted.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/04-dataframe-with-new-column.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/04-dataframe-with-new-column.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/05-tree-inc-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/05-tree-inc-time.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/06-tree-depth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/06-tree-depth.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/07-tree-invert-colormap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/07-tree-invert-colormap.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/08-dataframe-index-levels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/08-dataframe-index-levels.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/09-dataframe-drop-index-levels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/09-dataframe-drop-index-levels.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/10-graphframe-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/10-graphframe-filter.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/11-graphframe-squash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/11-graphframe-squash.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/12-lulesh-64cores.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/12-lulesh-64cores.png -------------------------------------------------------------------------------- /docs/images/basic-tutorial/13-divide-graphframes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/basic-tutorial/13-divide-graphframes.png -------------------------------------------------------------------------------- /docs/images/diff-graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/diff-graph1.png -------------------------------------------------------------------------------- /docs/images/diff-graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/diff-graph2.png -------------------------------------------------------------------------------- /docs/images/diff-graph3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/diff-graph3.png -------------------------------------------------------------------------------- /docs/images/filter-dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/filter-dataframe.png -------------------------------------------------------------------------------- /docs/images/flat-function.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/flat-function.png -------------------------------------------------------------------------------- /docs/images/jupyter-query-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/jupyter-query-filter.png -------------------------------------------------------------------------------- /docs/images/jupyter-tree-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/jupyter-tree-overview.png -------------------------------------------------------------------------------- /docs/images/lulesh-diff-df.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/lulesh-diff-df.png -------------------------------------------------------------------------------- /docs/images/lulesh-line-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/lulesh-line-plot.png -------------------------------------------------------------------------------- /docs/images/lulesh-load-imb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/lulesh-load-imb.png -------------------------------------------------------------------------------- /docs/images/lulesh-mpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/lulesh-mpi.png -------------------------------------------------------------------------------- /docs/images/lulesh-plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/lulesh-plot.png -------------------------------------------------------------------------------- /docs/images/query-dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/query-dataframe.png -------------------------------------------------------------------------------- /docs/images/sample-dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/sample-dataframe.png -------------------------------------------------------------------------------- /docs/images/sample-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/sample-graph.png -------------------------------------------------------------------------------- /docs/images/scale-offset-df.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/scale-offset-df.png -------------------------------------------------------------------------------- /docs/images/squash-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/squash-graph.png -------------------------------------------------------------------------------- /docs/images/strong-scale-graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/strong-scale-graph1.png -------------------------------------------------------------------------------- /docs/images/strong-scale-graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/strong-scale-graph2.png -------------------------------------------------------------------------------- /docs/images/strong-scale-graph3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/strong-scale-graph3.png -------------------------------------------------------------------------------- /docs/images/tutorial-output/caliper-cali-dataframe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/tutorial-output/caliper-cali-dataframe.png -------------------------------------------------------------------------------- /docs/images/tutorial-output/caliper-cali-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/tutorial-output/caliper-cali-graph.png -------------------------------------------------------------------------------- /docs/images/vis-dot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/vis-dot.png -------------------------------------------------------------------------------- /docs/images/vis-flamegraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/vis-flamegraph.png -------------------------------------------------------------------------------- /docs/images/vis-terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/vis-terminal.png -------------------------------------------------------------------------------- /docs/images/weak-scale-graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/weak-scale-graph1.png -------------------------------------------------------------------------------- /docs/images/weak-scale-graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/weak-scale-graph2.png -------------------------------------------------------------------------------- /docs/images/weak-scale-graph3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/images/weak-scale-graph3.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | Hatchet Project Developers. See the top-level LICENSE file for details. 3 | 4 | SPDX-License-Identifier: MIT 5 | 6 | .. hatchet documentation master file, created by 7 | sphinx-quickstart on Tue Jun 26 08:43:21 2018. 8 | You can adapt this file completely to your liking, but it should at least 9 | contain the root `toctree` directive. 10 | 11 | ####### 12 | Hatchet 13 | ####### 14 | 15 | Hatchet is a Python-based library that allows `Pandas 16 | `_ dataframes to be indexed by structured tree and 17 | graph data. It is intended for analyzing performance data that has a hierarchy 18 | (for example, serial or parallel profiles that represent calling context trees, 19 | call graphs, nested regions' timers, etc.). Hatchet implements various 20 | operations to analyze a single hierarchical data set or compare multiple data 21 | sets, and its API facilitates analyzing such data programmatically. 22 | 23 | You can get hatchet from its `GitHub repository 24 | `_: 25 | 26 | .. code-block:: console 27 | 28 | $ git clone https://github.com/llnl/hatchet.git 29 | 30 | or install it using pip: 31 | 32 | .. code-block:: console 33 | 34 | $ pip install llnl-hatchet 35 | 36 | For Lawrence Livermore National Laboratory users, we recommend using the 37 | hatchet installation directly. For more information, see :doc:`Using Hatchet 38 | on LLNL Systems `. 39 | 40 | If you are new to hatchet and want to start using it, see :doc:`Getting Started 41 | `, or refer to the full :doc:`User Guide ` below. 42 | 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | :caption: User Docs 47 | 48 | getting_started 49 | llnl 50 | user_guide 51 | query_lang 52 | data_generation 53 | analysis_examples 54 | 55 | .. toctree:: 56 | :maxdepth: 2 57 | :caption: Tutorials 58 | 59 | basic_tutorial 60 | 61 | 62 | If you encounter bugs while using hatchet, you can report them by opening an issue on `GitHub `_. 63 | 64 | If you are referencing hatchet in a publication, please cite the 65 | following `paper `_: 66 | 67 | * Abhinav Bhatele, Stephanie Brink, and Todd Gamblin. Hatchet: Pruning 68 | the Overgrowth in Parallel Profiles. In Proceedings of the International 69 | Conference for High Performance Computing, Networking, Storage and Analysis 70 | (SC '19). ACM, New York, NY, USA. 71 | `DOI `_ 72 | 73 | .. toctree:: 74 | :maxdepth: 2 75 | :caption: Developer Docs 76 | 77 | developer_guide 78 | 79 | .. toctree:: 80 | :maxdepth: 2 81 | :caption: References 82 | 83 | publications 84 | 85 | .. toctree:: 86 | :maxdepth: 2 87 | :caption: API Docs 88 | 89 | Hatchet API Docs 90 | 91 | 92 | ################## 93 | Indices and tables 94 | ################## 95 | 96 | * :ref:`genindex` 97 | * :ref:`modindex` 98 | * :ref:`search` 99 | -------------------------------------------------------------------------------- /docs/llnl.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | Hatchet Project Developers. See the top-level LICENSE file for details. 3 | 4 | SPDX-License-Identifier: MIT 5 | 6 | ***************************** 7 | Using Hatchet on LLNL Systems 8 | ***************************** 9 | 10 | Hatchet installations are available on both Intel and IBM systems at Lawrence 11 | Livermore National Laboratory. 12 | 13 | To use one of these global installations, add the following to your Python 14 | script or Jupyter notebook. This code allows you to use hatchet and its 15 | dependencies. 16 | 17 | .. code-block:: python 18 | :caption: Starter commands to find hatchet and its dependencies 19 | 20 | import sys 21 | import platform 22 | import datetime as dt 23 | from IPython.display import HTML, display 24 | 25 | input_deploy_dir_str = "/usr/gapps/spot/live/" 26 | machine = platform.uname().machine 27 | 28 | sys.path.append(input_deploy_dir_str + "/hatchet-venv/" + machine + "/lib/python3.7/site-packages") 29 | sys.path.append(input_deploy_dir_str + "/hatchet/" + machine) 30 | sys.path.append(input_deploy_dir_str + "/spotdb") 31 | 32 | import hatchet 33 | import spotdb 34 | 35 | 36 | The following Python script loads a single SPOT/cali file into hatchet using 37 | Hatchet's ``from_spotdb()``. This returns a list of hatchet ``GraphFrames``, 38 | and we use ``pop()`` to access the single ``GraphFrame`` in the list. 39 | 40 | .. code-block:: python 41 | :caption: Python script to load a single SPOT file into hatchet 42 | 43 | import sys 44 | import platform 45 | import datetime as dt 46 | from IPython.display import HTML, display 47 | 48 | input_deploy_dir_str = "/usr/gapps/spot/live/" 49 | machine = platform.uname().machine 50 | 51 | sys.path.append(input_deploy_dir_str + "/hatchet-venv/" + machine + "/lib/python3.7/site-packages") 52 | sys.path.append(input_deploy_dir_str + "/hatchet/" + machine) 53 | sys.path.append(input_deploy_dir_str + "/spotdb") 54 | 55 | import hatchet 56 | import spotdb 57 | 58 | input_db_uri_str = "./mpi" 59 | input_run_ids_str = "c5UcO9xwAUKNVVFg1_0.cali" 60 | 61 | db = spotdb.connect(input_db_uri_str) 62 | runs = input_run_ids_str.split(',') 63 | 64 | gfs = hatchet.GraphFrame.from_spotdb(db, runs) 65 | gf = gfs.pop() 66 | 67 | launchdate = dt.datetime.fromtimestamp(int(gf.metadata["launchdate"])) 68 | jobsize = int(gf.metadata.get("jobsize", 1)) 69 | 70 | print("launchdate: {}, jobsize: {}".format(launchdate, jobsize)) 71 | print(gf.tree()) 72 | display(HTML(gf.dataframe.to_html())) 73 | 74 | 75 | The following Python script loads multiple SPOT/cali files (most likely 76 | contained in the same directory) into hatchet using Hatchet's 77 | ``from_spotdb()``. The files are specified as a single string, and commas 78 | delineate each file. The result is a list of hatchet ``GraphFrames``, one for 79 | each file. 80 | 81 | .. code-block:: python 82 | :caption: Python script to load multiple SPOT files into hatchet 83 | 84 | import sys 85 | import platform 86 | import datetime as dt 87 | from IPython.display import HTML, display 88 | 89 | input_deploy_dir_str = "/usr/gapps/spot/live/" 90 | machine = platform.uname().machine 91 | 92 | sys.path.append(input_deploy_dir_str + "/hatchet-venv/" + machine + "/lib/python3.7/site-packages") 93 | sys.path.append(input_deploy_dir_str + "/hatchet/" + machine) 94 | sys.path.append(input_deploy_dir_str + "/spotdb") 95 | 96 | import hatchet 97 | import spotdb 98 | 99 | input_db_uri_str = "./mpi" 100 | input_run_ids_str = "./mpi/cQ-CGJlYj-uFT2yv-_0.cali,./mpi/cQ-CGJlYj-uFT2yv-_1.cali,./mpi/cQ-CGJlYj-uFT2yv-_2.cali" 101 | 102 | db = spotdb.connect(input_db_uri_str) 103 | runs = input_run_ids_str.split(',') 104 | 105 | gfs = hatchet.GraphFrame.from_spotdb(db, runs) 106 | 107 | for idx, gf in enumerate(gfs): 108 | launchdate = dt.datetime.fromtimestamp(int(gf.metadata["launchdate"])) 109 | jobsize = int(gf.metadata.get("jobsize", 1)) 110 | print("launchdate: {}, jobsize: {}".format(launchdate, jobsize)) 111 | print(gf.tree()) 112 | display(HTML(gf.dataframe.to_html())) 113 | -------------------------------------------------------------------------------- /docs/publications.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | Hatchet Project Developers. See the top-level LICENSE file for details. 3 | 4 | SPDX-License-Identifier: MIT 5 | 6 | ****************************** 7 | Publications and Presentations 8 | ****************************** 9 | 10 | Publications 11 | ============ 12 | 13 | - Stephanie Brink, Ian Lumsden, Connor Scully-Allison, Katy Williams, Olga Pearce, Todd Gamblin, Michela Taufer, Katherine Isaacs, Abhinav Bhatele. `Usability and Performance Improvements in Hatchet `_. Presented at the `ProTools 2020 Workshop `_, held in conjunction with the International Conference for High Performance Computing, Networking, Storage and Analysis (SC '20), held virtually. 14 | 15 | - Abhinav Bhatele, Stephanie Brink, and Todd Gamblin. `Hatchet: Pruning the Overgrowth in Parallel Profiles `_. In Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (SC '19), Denver, CO. 16 | 17 | Posters 18 | ======= 19 | 20 | - Ian Lumsden. `Graph-Based Profiling Analysis using Hatchet `_. Presented at SC '20. `Slides `_ | `Video Presentation `_ 21 | 22 | Tutorials 23 | ========= 24 | 25 | - Automated Performance Analysis with Caliper, SPOT, and Hatchet, ECP Annual Meeting, April 12, 2021. :download:`pdf ` | `YouTube `_ 26 | 27 | - Performance Analysis using Hatchet, LLNL, July 29/31, 2020. 28 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | pandas 3 | pydot 4 | numpy 5 | PyYAML 6 | cython 7 | multiprocess 8 | textX 9 | caliper-reader 10 | -------------------------------------------------------------------------------- /docs/slides/2021-ECPAM-hatchet-tutorial.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/docs/slides/2021-ECPAM-hatchet-tutorial.pdf -------------------------------------------------------------------------------- /docs/source/hatchet.cython_modules.libs.rst: -------------------------------------------------------------------------------- 1 | hatchet.cython\_modules.libs package 2 | ==================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.cython\_modules.libs.graphframe\_modules module 8 | ------------------------------------------------------- 9 | 10 | .. automodule:: hatchet.cython_modules.libs.graphframe_modules 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.cython\_modules.libs.reader\_modules module 16 | --------------------------------------------------- 17 | 18 | .. automodule:: hatchet.cython_modules.libs.reader_modules 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: hatchet.cython_modules.libs 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/hatchet.cython_modules.rst: -------------------------------------------------------------------------------- 1 | hatchet.cython\_modules package 2 | =============================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | hatchet.cython_modules.libs 11 | 12 | Submodules 13 | ---------- 14 | 15 | hatchet.cython\_modules.graphframe\_modules module 16 | -------------------------------------------------- 17 | 18 | .. automodule:: hatchet.cython_modules.graphframe_modules 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hatchet.cython\_modules.reader\_modules module 24 | ---------------------------------------------- 25 | 26 | .. automodule:: hatchet.cython_modules.reader_modules 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | Module contents 32 | --------------- 33 | 34 | .. automodule:: hatchet.cython_modules 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /docs/source/hatchet.external.roundtrip.roundtrip.rst: -------------------------------------------------------------------------------- 1 | hatchet.external.roundtrip.roundtrip package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.external.roundtrip.roundtrip.manager module 8 | --------------------------------------------------- 9 | 10 | .. automodule:: hatchet.external.roundtrip.roundtrip.manager 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.external.roundtrip.roundtrip.version module 16 | --------------------------------------------------- 17 | 18 | .. automodule:: hatchet.external.roundtrip.roundtrip.version 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: hatchet.external.roundtrip.roundtrip 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/hatchet.external.roundtrip.rst: -------------------------------------------------------------------------------- 1 | hatchet.external.roundtrip package 2 | ================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | hatchet.external.roundtrip.roundtrip 11 | 12 | Submodules 13 | ---------- 14 | 15 | hatchet.external.roundtrip.setup module 16 | --------------------------------------- 17 | 18 | .. automodule:: hatchet.external.roundtrip.setup 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: hatchet.external.roundtrip 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/hatchet.external.rst: -------------------------------------------------------------------------------- 1 | hatchet.external package 2 | ======================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | hatchet.external.roundtrip 11 | 12 | Submodules 13 | ---------- 14 | 15 | hatchet.external.console module 16 | ------------------------------- 17 | 18 | .. automodule:: hatchet.external.console 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: hatchet.external 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/hatchet.query.rst: -------------------------------------------------------------------------------- 1 | hatchet.query package 2 | ===================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.query.compat module 8 | --------------------------- 9 | 10 | .. automodule:: hatchet.query.compat 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.query.compound module 16 | ----------------------------- 17 | 18 | .. automodule:: hatchet.query.compound 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hatchet.query.engine module 24 | --------------------------- 25 | 26 | .. automodule:: hatchet.query.engine 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hatchet.query.errors module 32 | --------------------------- 33 | 34 | .. automodule:: hatchet.query.errors 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hatchet.query.object\_dialect module 40 | ------------------------------------ 41 | 42 | .. automodule:: hatchet.query.object_dialect 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hatchet.query.query module 48 | -------------------------- 49 | 50 | .. automodule:: hatchet.query.query 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hatchet.query.string\_dialect module 56 | ------------------------------------ 57 | 58 | .. automodule:: hatchet.query.string_dialect 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | Module contents 64 | --------------- 65 | 66 | .. automodule:: hatchet.query 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | -------------------------------------------------------------------------------- /docs/source/hatchet.readers.rst: -------------------------------------------------------------------------------- 1 | hatchet.readers package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.readers.caliper\_native\_reader module 8 | ---------------------------------------------- 9 | 10 | .. automodule:: hatchet.readers.caliper_native_reader 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.readers.caliper\_reader module 16 | -------------------------------------- 17 | 18 | .. automodule:: hatchet.readers.caliper_reader 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hatchet.readers.cprofile\_reader module 24 | --------------------------------------- 25 | 26 | .. automodule:: hatchet.readers.cprofile_reader 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hatchet.readers.dataframe\_reader module 32 | ---------------------------------------- 33 | 34 | .. automodule:: hatchet.readers.dataframe_reader 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hatchet.readers.gprof\_dot\_reader module 40 | ----------------------------------------- 41 | 42 | .. automodule:: hatchet.readers.gprof_dot_reader 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hatchet.readers.hdf5\_reader module 48 | ----------------------------------- 49 | 50 | .. automodule:: hatchet.readers.hdf5_reader 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hatchet.readers.hpctoolkit\_reader module 56 | ----------------------------------------- 57 | 58 | .. automodule:: hatchet.readers.hpctoolkit_reader 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | hatchet.readers.json\_reader module 64 | ----------------------------------- 65 | 66 | .. automodule:: hatchet.readers.json_reader 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | hatchet.readers.literal\_reader module 72 | -------------------------------------- 73 | 74 | .. automodule:: hatchet.readers.literal_reader 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | hatchet.readers.pyinstrument\_reader module 80 | ------------------------------------------- 81 | 82 | .. automodule:: hatchet.readers.pyinstrument_reader 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | hatchet.readers.spotdb\_reader module 88 | ------------------------------------- 89 | 90 | .. automodule:: hatchet.readers.spotdb_reader 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | hatchet.readers.tau\_reader module 96 | ---------------------------------- 97 | 98 | .. automodule:: hatchet.readers.tau_reader 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | hatchet.readers.timemory\_reader module 104 | --------------------------------------- 105 | 106 | .. automodule:: hatchet.readers.timemory_reader 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | Module contents 112 | --------------- 113 | 114 | .. automodule:: hatchet.readers 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | -------------------------------------------------------------------------------- /docs/source/hatchet.rst: -------------------------------------------------------------------------------- 1 | hatchet package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | hatchet.cython_modules 11 | hatchet.external 12 | hatchet.query 13 | hatchet.readers 14 | hatchet.util 15 | hatchet.vis 16 | hatchet.writers 17 | 18 | Submodules 19 | ---------- 20 | 21 | hatchet.caliper\_json module 22 | ---------------------------- 23 | 24 | .. automodule:: hatchet.caliper_json 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | 29 | hatchet.frame module 30 | -------------------- 31 | 32 | .. automodule:: hatchet.frame 33 | :members: 34 | :undoc-members: 35 | :show-inheritance: 36 | 37 | hatchet.graph module 38 | -------------------- 39 | 40 | .. automodule:: hatchet.graph 41 | :members: 42 | :undoc-members: 43 | :show-inheritance: 44 | 45 | hatchet.graphframe module 46 | ------------------------- 47 | 48 | .. automodule:: hatchet.graphframe 49 | :members: 50 | :undoc-members: 51 | :show-inheritance: 52 | 53 | hatchet.node module 54 | ------------------- 55 | 56 | .. automodule:: hatchet.node 57 | :members: 58 | :undoc-members: 59 | :show-inheritance: 60 | 61 | hatchet.version module 62 | ---------------------- 63 | 64 | .. automodule:: hatchet.version 65 | :members: 66 | :undoc-members: 67 | :show-inheritance: 68 | 69 | Module contents 70 | --------------- 71 | 72 | .. automodule:: hatchet 73 | :members: 74 | :undoc-members: 75 | :show-inheritance: 76 | -------------------------------------------------------------------------------- /docs/source/hatchet.util.rst: -------------------------------------------------------------------------------- 1 | hatchet.util package 2 | ==================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.util.colormaps module 8 | ----------------------------- 9 | 10 | .. automodule:: hatchet.util.colormaps 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.util.config module 16 | -------------------------- 17 | 18 | .. automodule:: hatchet.util.config 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hatchet.util.deprecated module 24 | ------------------------------ 25 | 26 | .. automodule:: hatchet.util.deprecated 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | hatchet.util.dot module 32 | ----------------------- 33 | 34 | .. automodule:: hatchet.util.dot 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | hatchet.util.executable module 40 | ------------------------------ 41 | 42 | .. automodule:: hatchet.util.executable 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | hatchet.util.profiler module 48 | ---------------------------- 49 | 50 | .. automodule:: hatchet.util.profiler 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | hatchet.util.timer module 56 | ------------------------- 57 | 58 | .. automodule:: hatchet.util.timer 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | Module contents 64 | --------------- 65 | 66 | .. automodule:: hatchet.util 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | -------------------------------------------------------------------------------- /docs/source/hatchet.vis.external.rst: -------------------------------------------------------------------------------- 1 | hatchet.vis.external package 2 | ============================ 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: hatchet.vis.external 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/hatchet.vis.rst: -------------------------------------------------------------------------------- 1 | hatchet.vis package 2 | =================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | hatchet.vis.external 11 | 12 | Submodules 13 | ---------- 14 | 15 | hatchet.vis.loader module 16 | ------------------------- 17 | 18 | .. automodule:: hatchet.vis.loader 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | hatchet.vis.static\_fixer module 24 | -------------------------------- 25 | 26 | .. automodule:: hatchet.vis.static_fixer 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | Module contents 32 | --------------- 33 | 34 | .. automodule:: hatchet.vis 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | -------------------------------------------------------------------------------- /docs/source/hatchet.writers.rst: -------------------------------------------------------------------------------- 1 | hatchet.writers package 2 | ======================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | hatchet.writers.dataframe\_writer module 8 | ---------------------------------------- 9 | 10 | .. automodule:: hatchet.writers.dataframe_writer 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | hatchet.writers.hdf5\_writer module 16 | ----------------------------------- 17 | 18 | .. automodule:: hatchet.writers.hdf5_writer 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: hatchet.writers 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | hatchet 2 | ======= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | hatchet 8 | -------------------------------------------------------------------------------- /hatchet/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | # make flake8 unused names in this file. 7 | # flake8: noqa: F401 8 | 9 | from .graphframe import GraphFrame 10 | from .query import QueryMatcher 11 | from .version import __version__ 12 | -------------------------------------------------------------------------------- /hatchet/cython_modules/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/cython_modules/graphframe_modules.pyx: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import cython 7 | import numpy as np 8 | from libc.math cimport floor 9 | 10 | 11 | # Description: All cython code related to graphframe.py 12 | @cython.boundscheck(False) 13 | def insert_one_for_self_nodes(const long snio_len, short[:] self_missing_node, const long[:] snio_indices): 14 | """Adds a '1' where rows are in self but not in other.""" 15 | for i in range(snio_len): 16 | self_missing_node[snio_indices[i]] = 1 17 | 18 | cpdef fast_not_isin(const unsigned long long[:,:] arr1, const unsigned long long[:,:] arr2, const long arr1_len, const long arr2_len): 19 | """Check if elements in arr1 exist in arr2. 20 | 21 | A fast check to see if each element in arr1 exists in arr2. This returns a 22 | vector mask equivalent to what the operation ~df.isin( ... ) would return. 23 | 24 | Arguments: 25 | arr1 (unsigned long long [][]): The array of values we are searching for. 26 | arr2 (unsigned long long [][]): The sorted array of values we are searching in. 27 | 28 | Returns: 29 | (bool []): A boolean mask over arr1 indicating whether each element is or is not in 30 | the function. True indicates that an element is not in arr2. 31 | """ 32 | result = np.zeros(len(arr1), dtype=np.bool_) 33 | cdef long index = -1 34 | cdef unsigned long long prior = -1 35 | cdef long i = 0 36 | 37 | for i in range(arr1_len): 38 | # mini optimization: don't perform binsearch if we already know node is/isnot in graph 39 | if prior == arr1[i][0]: 40 | result[arr1[i][1]] = result[arr1[i-1][1]] 41 | else: 42 | index = binary_search(arr2, arr2_len, arr1[i][0]) 43 | if index == -1: 44 | result[arr1[i][1]] = True 45 | else: 46 | result[arr1[i][1]] = False 47 | 48 | prior = arr1[i][0] 49 | 50 | return result 51 | 52 | cdef binary_search(const unsigned long long[:,:] array, const long arr_len, const unsigned long long key): 53 | """Basic Binary Search. 54 | 55 | Arguments: 56 | array (unsigned long long [][]): Array we are searching in (must be sorted). 57 | 58 | Returns: The index of key if found, -1 if not found. 59 | """ 60 | cdef long L = 0 61 | cdef long R = arr_len - 1 62 | cdef long midpt = 0 63 | while (L <= R): 64 | midpt = floor((L+R)/2) 65 | if array[midpt][0] < key: 66 | L = midpt + 1 67 | elif array[midpt][0] > key: 68 | R = midpt - 1 69 | else: 70 | return midpt 71 | return -1 72 | -------------------------------------------------------------------------------- /hatchet/cython_modules/libs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/cython_modules/reader_modules.pyx: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import cython 7 | 8 | def subtract_exclusive_metric_vals(long nid, long parent_nid, double[:] metrics, long num_stmt_nodes, long stride): 9 | cdef long ref_nid = nid 10 | cdef long ref_pnid = parent_nid 11 | 12 | # compiler directive 13 | with cython.boundscheck(False): 14 | # we are modifying metrics in place here 15 | # since they are passed by refrence via their 16 | # memory 17 | for i in range(num_stmt_nodes): 18 | metrics[ref_pnid-1] -= metrics[ref_nid-1] 19 | 20 | ref_nid += stride 21 | ref_pnid += stride 22 | -------------------------------------------------------------------------------- /hatchet/external/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | 7 | class VersionError(Exception): 8 | """ 9 | Define a version error class with the same features of the base exception class. 10 | Allows us to catch the very specific ipython version error and throw a warning. 11 | """ 12 | 13 | pass 14 | 15 | 16 | try: 17 | import IPython 18 | 19 | # Testing IPython version 20 | if int(IPython.__version__.split(".")[0]) > 7: 21 | raise VersionError() 22 | 23 | from .roundtrip.roundtrip.manager import Roundtrip 24 | 25 | # Refrencing Roundtrip here to resolve scope issues with import 26 | Roundtrip 27 | 28 | except ImportError: 29 | pass 30 | 31 | except VersionError: 32 | if IPython.get_ipython() is not None: 33 | print( 34 | "Warning: Roundtrip module could not be loaded. Requires jupyter notebook version <= 7.x." 35 | ) 36 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Humans, Data, and Computers Lab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/NOTICE: -------------------------------------------------------------------------------- 1 | This work was supported by the United States Department of Defense through DTIC Contract FA8075-14-D-0002-0007, 2 | the National Science Foundation under NSF III-1656958, and the Department of Energy under DE-SC0022044. 3 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/README.md: -------------------------------------------------------------------------------- 1 | ## Roundtrip 2 | 3 | A python/javascript library for loading and managing Javascript code and visualization in Jupyter notebook cells. 4 | 5 | This library supports: 6 | - Loading HTML, CSS, and JavaScript Files 7 | - Loading Webpack-Generated Files 8 | - Passing Jupyter-scoped data into Javascript code 9 | - Returnng Javascript generated data back to the calling Jupyter Notebook 10 | - Binding data between a Jupyter notebook and embedded Javascript visualizations 11 | - Automatic updating of bound visualizations enabling: 12 | - fluid gui-scripting based workflows 13 | - linked views across cells 14 | 15 | ### Important Links 16 | - [Roundtrip Wiki](https://github.com/hdc-arizona/roundtrip/wiki) 17 | 18 | 19 | ### Try It Out 20 | 1) Install [Jupyter notebook](https://jupyter.org/install) & [Node.js](https://nodejs.org/en/download/) 21 | 2) Clone this repository: 22 | ```bash 23 | git clone https://github.com/hdc-arizona/roundtrip.git 24 | ``` 25 | 3) Run the automatic installer: 26 | ```bash 27 | cd roundtrip 28 | pip3 install roundtrip-lib 29 | chmod +x buuild_examples.sh 30 | ./build_examples.sh 31 | ``` 32 | 4) Start a jupyter server from the `roundtrip` base directory: 33 | ```bash 34 | jupyter notebook 35 | ``` 36 | 5) From your web browser navigate to the `docs/examples/` folder and open the `Manual Workflow Example` notebook. 37 | 38 | On load, you may need to clean the output by running `Restart & Clear Output` 39 | from the `Kernel` menu in Jupyter. 40 | 41 | Running the cells in the first example will demonstrate: 42 | 43 | 1. The loading of a real pandas dataset into a javascript visualization 44 | 2. Interaction with the visualization 45 | 3. Returning data back from the visualization to the Jupyter notebook 46 | 47 | Once you understand the functionality in this notebook please open the `Advanced Workflow Example` 48 | 49 | Running the cells in this example will demonstrate: 50 | 1. The `?` operator and how it links python data with visualization data 51 | 2. How the `?` can provide linked-view functionality between cells 52 | 3. How the linking of data and automatic updating of cells can be easily turned off by removing the `?` 53 | 4. How cells update when data is update inside the jupyter notebook as well as in the visualizations 54 | 55 | 56 | ### License 57 | 58 | Roundtrip is distributed under the terms of the MIT license. 59 | 60 | All contributions must be made under the MIT license. Copyrights in the 61 | Roundtrip project are retained by contributors. No copyright assignment is 62 | required to contribute to Roundtrip. 63 | 64 | See [LICENSE](https://github.com/hdc-arizona/roundtrip/blob/master/LICENSE) and 65 | [NOTICE](https://github.com/hdc-arizona/roundtrip/blob/master/NOTICE) for details. 66 | 67 | SPDX-License-Identifier: MIT 68 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/external/roundtrip/__init__.py -------------------------------------------------------------------------------- /hatchet/external/roundtrip/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "roundtrip", 3 | "version": "0.2.1", 4 | "description": "An interface for loading Javascript (notably D3 visualizations) into Jupyter Notebooks. Supports transferring data from Python Jupyter cells to Javascript—and back.", 5 | "private": true, 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/hdc-arizona/roundtrip.git" 12 | }, 13 | "keywords": [], 14 | "author": "", 15 | "license": "ISC", 16 | "bugs": { 17 | "url": "https://github.com/hdc-arizona/roundtrip/issues" 18 | }, 19 | "homepage": "https://github.com/hdc-arizona/roundtrip#readme", 20 | "devDependencies": { 21 | "@babel/core": "^7.16.0", 22 | "@babel/preset-env": "^7.16.0", 23 | "@babel/preset-react": "^7.16.0", 24 | "babel-loader": "^8.2.3", 25 | "html-webpack-plugin": "^5.3.2", 26 | "webpack": "^5.58.2", 27 | "webpack-cli": "^4.9.0" 28 | }, 29 | "dependencies": { 30 | "css-loader": "^6.5.0", 31 | "d3": "^7.0.4", 32 | "react": "^17.0.2", 33 | "react-dom": "^17.0.2", 34 | "style-loader": "^3.3.1" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "roundtrip" 3 | version = "0.2" 4 | description = "A Python library for the loading of JS visualizations into Jupyter notebooks." 5 | authors = [ 6 | "Connor Scully-Allison " 8 | ] 9 | license = "MIT" 10 | 11 | [tool.black] 12 | line-length = 88 13 | target-version = ['py27', 'py35', 'py36', 'py37', 'py38'] 14 | include = '\.pyi?$' 15 | exclude = ''' 16 | /( 17 | \.eggs 18 | | \.git 19 | | \.hg 20 | | \.mypy_cache 21 | | \.tox 22 | | \.venv 23 | | _build 24 | | buck-out 25 | | build 26 | | dist 27 | )/ 28 | ''' -------------------------------------------------------------------------------- /hatchet/external/roundtrip/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | beautifulsoup4 4 | ipython 5 | pathlib 6 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/roundtrip/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021-2022 HDC Lab and Roundtrip Project Developers. 2 | # See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/roundtrip/tests/data/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 |
Loaded successfully.
6 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/roundtrip/tests/data/test.js: -------------------------------------------------------------------------------- 1 | RT = window.Roundtrip; 2 | 3 | data_from_py = RT["data_from_py"] 4 | 5 | data_to_py = data_from_py + 10 6 | 7 | RT["to_py"] = data_to_py -------------------------------------------------------------------------------- /hatchet/external/roundtrip/roundtrip/tests/rt_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from IPython.testing.globalipapp import start_ipython 4 | 5 | from hatchet.external.roundtrip.roundtrip.manager import RoundTrip 6 | 7 | no_rt = False 8 | if sys.version_info[0] < 3: 9 | no_rt = True 10 | 11 | local_dir = os.path.dirname(os.path.abspath(__file__)) 12 | 13 | ip = start_ipython() 14 | 15 | if not no_rt: 16 | RT = RoundTrip(ipy_shell=ip, test=True) 17 | 18 | 19 | def test_init_load_web_files(): 20 | if not no_rt: 21 | # verify that the RT Javascript was loaded 22 | assert ( 23 | open(os.path.join(local_dir, "../roundtrip.js"), "r").read() 24 | in RT.rt_js._repr_javascript_() 25 | ) 26 | 27 | # test loading multiple files 28 | RT.load_web_files( 29 | [ 30 | str(os.path.join(local_dir, "data/test.html")), 31 | str(os.path.join(local_dir, "data/test.js")), 32 | ] 33 | ) 34 | 35 | assert len(RT.bridges) != 0 36 | 37 | RT.initialize() 38 | 39 | for b in RT.bridges: 40 | for s in RT.bridges[b].active_scripts: 41 | assert ( 42 | open(os.path.join(local_dir, "data/test.js"), "r").read() 43 | in s._repr_javascript_() 44 | ) 45 | 46 | 47 | def test_data_transfer(): 48 | if not no_rt: 49 | js_var_down = "data_from_py" 50 | js_var_up = "data_to_py" 51 | 52 | RT.data_to_js(5, js_variable=js_var_down) 53 | RT.fetch_data(js_var_up, "retrieved_from_js") 54 | 55 | down = False 56 | up = False 57 | 58 | for b in RT.bridges: 59 | for s in RT.bridges[b].active_scripts: 60 | if ( 61 | "window.Roundtrip['{}']".format(js_var_down) 62 | in s._repr_javascript_() 63 | ): 64 | down = True 65 | elif "Roundtrip['{}']".format(js_var_up) in s._repr_javascript_(): 66 | up = True 67 | 68 | assert down is True 69 | assert up is True 70 | 71 | 72 | def test_var_transfer(): 73 | if not no_rt: 74 | js_var_down = "var_from_py" 75 | js_var_up = "var_to_py" 76 | py_var = "var_test" 77 | 78 | down = False 79 | up = False 80 | 81 | # load a var in the user namespace 82 | ip.run_cell(raw_cell="{} = 1".format(py_var)) 83 | 84 | RT.var_to_js(py_var, js_variable=js_var_down) 85 | RT.fetch_data(js_var_up, "retrieved_from_js") 86 | 87 | for b in RT.bridges: 88 | for s in RT.bridges[b].active_scripts: 89 | if ( 90 | "window.Roundtrip['{}']".format(js_var_down) 91 | in s._repr_javascript_() 92 | and py_var in s._repr_javascript_() 93 | ): 94 | down = True 95 | elif "Roundtrip['{}']".format(js_var_up) in s._repr_javascript_(): 96 | up = True 97 | 98 | assert down is True 99 | assert up is True 100 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/roundtrip/version.py: -------------------------------------------------------------------------------- 1 | __version_info__ = ("0", "2", "1") 2 | __version__ = ".".join(__version_info__) 3 | -------------------------------------------------------------------------------- /hatchet/external/roundtrip/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | from codecs import open 3 | from os import path 4 | 5 | here = path.abspath(path.dirname(__file__)) 6 | 7 | with open("README.md", "r", encoding="utf-8") as fh: 8 | long_description = fh.read() 9 | 10 | # Get the version in a safe way 11 | # per python docs: https://packaging.python.org/guides/single-sourcing-package-version/ 12 | version = {} 13 | with open("./roundtrip/version.py") as fp: 14 | exec(fp.read(), version) 15 | 16 | 17 | setup( 18 | name="roundtrip-lib", 19 | version=version["__version__"], 20 | description="A Python library for loading JS visualizations into jupyter notebooks.", 21 | long_description=long_description, 22 | long_description_content_type="text/markdown", 23 | url="https://github.com/hdc-arizona/roundtrip", 24 | author="Connor Scully-Allison", 25 | author_email="cscullyallison@email.arizona.edu", 26 | license="MIT", 27 | keywords="", 28 | packages=["roundtrip"], 29 | install_requires=["numpy", "pandas", "beautifulsoup4", "ipython", "pathlib"], 30 | include_package_data=True, 31 | ) 32 | -------------------------------------------------------------------------------- /hatchet/frame.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from functools import total_ordering 7 | 8 | 9 | @total_ordering 10 | class Frame: 11 | """The frame index for a node. The node only stores its frame. 12 | 13 | Arguments: 14 | attrs (dict): dictionary of attributes and values 15 | """ 16 | 17 | def __init__(self, attrs=None, **kwargs): 18 | """Construct a frame from a dictionary, or from immediate kwargs. 19 | 20 | Arguments: 21 | attrs (dict, optional): dictionary of attributes for this Frame 22 | 23 | Keyword arguments are optional, but if they are provided, they 24 | will be used to update the dictionary. Keys in kwargs take 25 | precedence over anything in the attrs dictionary. 26 | 27 | So, these are all functionally equivalent:: 28 | 29 | Frame({"name": "foo", "file": "bar.c"}) 30 | Frame(name="foo", file="bar.c") 31 | Frame({"name": "foo"}, file="bar.c") 32 | Frame({"name": "foo", "file": "baz.h"}, file="bar.c") 33 | 34 | """ 35 | # attributes dictionary 36 | self.attrs = attrs if attrs else {} 37 | 38 | # add keyword arguments, if any. 39 | if kwargs: 40 | self.attrs.update(kwargs) 41 | 42 | if not self.attrs: 43 | raise ValueError("Frame must be constructed with attributes!") 44 | 45 | # add type to frame if type is not in the attributes dict or kwargs 46 | if "type" not in self.attrs: 47 | self.attrs["type"] = "None" 48 | 49 | self._tuple_repr = None 50 | 51 | def __eq__(self, other): 52 | return self.tuple_repr == other.tuple_repr 53 | 54 | def __lt__(self, other): 55 | return self.tuple_repr < other.tuple_repr 56 | 57 | def __gt__(self, other): 58 | return self.tuple_repr > other.tuple_repr 59 | 60 | def __hash__(self): 61 | return hash(self.tuple_repr) 62 | 63 | def __str__(self): 64 | """str() with sorted attributes, so output is deterministic.""" 65 | return "{%s}" % ", ".join("'%s': '%s'" % (k, v) for k, v in self.tuple_repr) 66 | 67 | def __repr__(self): 68 | return "Frame(%s)" % self 69 | 70 | @property 71 | def tuple_repr(self): 72 | """Make a tuple of attributes and values based on reader.""" 73 | if not self._tuple_repr: 74 | self._tuple_repr = tuple(sorted((k, v) for k, v in self.attrs.items())) 75 | return self._tuple_repr 76 | 77 | def copy(self): 78 | return Frame(self.attrs.copy()) 79 | 80 | def __getitem__(self, name): 81 | return self.attrs[name] 82 | 83 | def get(self, name, default=None): 84 | return self.attrs.get(name, default) 85 | 86 | def values(self, names): 87 | """Return a tuple of attribute values from this Frame.""" 88 | if isinstance(names, (list, tuple)): 89 | return tuple(self.attrs.get(name) for name in names) 90 | else: 91 | return self.attrs.get(names) 92 | -------------------------------------------------------------------------------- /hatchet/query/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | # Make flake8 ignore unused names in this file 7 | # flake8: noqa: F401 8 | 9 | from .query import Query 10 | from .compound import ( 11 | CompoundQuery, 12 | ConjunctionQuery, 13 | DisjunctionQuery, 14 | ExclusiveDisjunctionQuery, 15 | NegationQuery, 16 | ) 17 | from .object_dialect import ObjectQuery 18 | from .string_dialect import StringQuery, parse_string_dialect 19 | from .engine import QueryEngine 20 | from .errors import ( 21 | InvalidQueryPath, 22 | InvalidQueryFilter, 23 | RedundantQueryFilterWarning, 24 | BadNumberNaryQueryArgs, 25 | ) 26 | 27 | from .compat import ( 28 | AbstractQuery, 29 | NaryQuery, 30 | AndQuery, 31 | IntersectionQuery, 32 | OrQuery, 33 | UnionQuery, 34 | XorQuery, 35 | SymDifferenceQuery, 36 | NotQuery, 37 | QueryMatcher, 38 | CypherQuery, 39 | parse_cypher_query, 40 | ) 41 | 42 | 43 | def combine_via_conjunction(query0, query1): 44 | return ConjunctionQuery(query0, query1) 45 | 46 | 47 | def combine_via_disjunction(query0, query1): 48 | return DisjunctionQuery(query0, query1) 49 | 50 | 51 | def combine_via_exclusive_disjunction(query0, query1): 52 | return ExclusiveDisjunctionQuery(query0, query1) 53 | 54 | 55 | def negate_query(query): 56 | return NegationQuery(query) 57 | 58 | 59 | Query.__and__ = combine_via_conjunction 60 | Query.__or__ = combine_via_disjunction 61 | Query.__xor__ = combine_via_exclusive_disjunction 62 | Query.__not__ = negate_query 63 | 64 | 65 | CompoundQuery.__and__ = combine_via_conjunction 66 | CompoundQuery.__or__ = combine_via_disjunction 67 | CompoundQuery.__xor__ = combine_via_exclusive_disjunction 68 | CompoundQuery.__not__ = negate_query 69 | 70 | 71 | def is_hatchet_query(query_obj): 72 | return ( 73 | issubclass(type(query_obj), Query) 74 | or issubclass(type(query_obj), CompoundQuery) 75 | or issubclass(type(query_obj), AbstractQuery) 76 | ) 77 | 78 | 79 | # Uncomment when accessing old-style queries using 80 | # 'from hatchet.query import *' is removed 81 | # 82 | # __all__ = [ 83 | # "Query", 84 | # "CompoundQuery", 85 | # "ConjunctionQuery", 86 | # "DisjunctionQuery", 87 | # "ExclusiveDisjunctionQuery", 88 | # "NegationQuery", 89 | # "ObjectQuery", 90 | # "StringQuery", 91 | # "parse_string_dialect", 92 | # "InvalidQueryFilter", 93 | # "InvalidQueryPath", 94 | # "RedundantQueryFilterWarning", 95 | # "BadNumberNaryQueryArgs", 96 | # ] 97 | -------------------------------------------------------------------------------- /hatchet/query/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | 7 | class InvalidQueryPath(Exception): 8 | """Raised when a query does not have the correct syntax""" 9 | 10 | 11 | class InvalidQueryFilter(Exception): 12 | """Raised when a query filter does not have a valid syntax""" 13 | 14 | 15 | class RedundantQueryFilterWarning(Warning): 16 | """Warned when a query filter does nothing or is redundant""" 17 | 18 | 19 | class BadNumberNaryQueryArgs(Exception): 20 | """Raised when a query filter does not have a valid syntax""" 21 | 22 | 23 | class MultiIndexModeMismatch(Exception): 24 | """Raised when an ObjectQuery or StringQuery object 25 | is set to use multi-indexed data, but no multi-indexed 26 | data is provided""" 27 | -------------------------------------------------------------------------------- /hatchet/query/query.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from .errors import InvalidQueryPath 7 | 8 | 9 | class Query(object): 10 | """Class for representing and building Hatchet Call Path Queries""" 11 | 12 | def __init__(self): 13 | """Create new Query""" 14 | self.query_pattern = [] 15 | 16 | def match(self, quantifier=".", predicate=lambda row: True): 17 | """Start a query with a root node described by the arguments. 18 | 19 | Arguments: 20 | quantifier (".", "*", "+", or int, optional): the quantifier for this node (tells how many graph nodes to match) 21 | predicate (Callable, optional): the predicate for this node (used to determine whether a graph node matches this query node) 22 | 23 | Returns: 24 | (Query): returns self so that this method can be chained with subsequent calls to "rel"/"relation" 25 | """ 26 | if len(self.query_pattern) != 0: 27 | self.query_pattern = [] 28 | self._add_node(quantifier, predicate) 29 | return self 30 | 31 | def rel(self, quantifier=".", predicate=lambda row: True): 32 | """Add a new node to the end of the query. 33 | 34 | Arguments: 35 | quantifier (".", "*", "+", or int, optional): the quantifier for this node (tells how many graph nodes to match) 36 | predicate (Callable, optional): the predicate for this node (used to determine whether a graph node matches this query node) 37 | 38 | Returns: 39 | (Query): returns self so that this method can be chained with subsequent calls to "rel"/"relation" 40 | """ 41 | if len(self.query_pattern) == 0: 42 | raise InvalidQueryPath( 43 | "Queries in the base Query Language must start with a call to 'match'" 44 | ) 45 | self._add_node(quantifier, predicate) 46 | return self 47 | 48 | def relation(self, quantifer=".", predicate=lambda row: True): 49 | """Alias to Query.rel. Add a new node to the end of the query. 50 | 51 | Arguments: 52 | quantifier (".", "*", "+", or int, optional): the quantifier for this node (tells how many graph nodes to match) 53 | predicate (Callable, optional): the predicate for this node (used to determine whether a graph node matches this query node) 54 | 55 | Returns: 56 | (Query): returns self so that this method can be chained with subsequent calls to "rel"/"relation" 57 | """ 58 | return self.rel(quantifer, predicate) 59 | 60 | def __len__(self): 61 | """Returns the length of the query.""" 62 | return len(self.query_pattern) 63 | 64 | def __iter__(self): 65 | """Allows users to iterate over the Query like a list.""" 66 | return iter(self.query_pattern) 67 | 68 | def _add_node(self, quantifer=".", predicate=lambda row: True): 69 | """Add a node to the query. 70 | 71 | Arguments: 72 | quantifier (".", "*", "+", or int, optional): the quantifier for this node (tells how many graph nodes to match) 73 | predicate (Callable, optional): the predicate for this node (used to determine whether a graph node matches this query node) 74 | """ 75 | assert isinstance(quantifer, int) or isinstance(quantifer, str) 76 | assert callable(predicate) 77 | if isinstance(quantifer, int): 78 | for _ in range(quantifer): 79 | self.query_pattern.append((".", predicate)) 80 | elif quantifer == "+": 81 | self.query_pattern.append((".", predicate)) 82 | self.query_pattern.append(("*", predicate)) 83 | else: 84 | assert quantifer == "." or quantifer == "*" 85 | self.query_pattern.append((quantifer, predicate)) 86 | -------------------------------------------------------------------------------- /hatchet/readers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/readers/dataframe_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import hatchet.graphframe 7 | from hatchet.node import Node 8 | from hatchet.graph import Graph 9 | 10 | from abc import abstractmethod 11 | 12 | # TODO The ABC class was introduced in Python 3.4. 13 | # When support for earlier versions is (eventually) dropped, 14 | # this entire "try-except" block can be reduced to: 15 | # from abc import ABC 16 | try: 17 | from abc import ABC 18 | except ImportError: 19 | from abc import ABCMeta 20 | 21 | ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) 22 | 23 | 24 | def _get_node_from_df_iloc(df, ind): 25 | node = None 26 | if isinstance(df.iloc[ind].name, tuple): 27 | node = df.iloc[ind].name[0] 28 | elif isinstance(df.iloc[ind].name, Node): 29 | node = df.iloc[ind].name 30 | else: 31 | raise InvalidDataFrameIndex( 32 | "DataFrame index elements must be either a tuple or a Node" 33 | ) 34 | return node 35 | 36 | 37 | def _get_parents_and_children(df): 38 | rel_dict = {} 39 | for i in range(len(df)): 40 | node = _get_node_from_df_iloc(df, i) 41 | if node not in rel_dict: 42 | rel_dict[node] = {} 43 | rel_dict[node]["parents"] = df.iloc[i].loc["parents"] 44 | rel_dict[node]["children"] = df.iloc[i].loc["children"] 45 | return rel_dict 46 | 47 | 48 | def _reconstruct_graph(df, rel_dict): 49 | node_list = sorted(list(df.index.to_frame()["node"])) 50 | for i in range(len(df)): 51 | node = _get_node_from_df_iloc(df, i) 52 | if len(node.children) == 0: 53 | node.children = [node_list[nid] for nid in rel_dict[node]["children"]] 54 | if len(node.parents) == 0: 55 | node.parents = [node_list[nid] for nid in rel_dict[node]["parents"]] 56 | roots = [node for node in node_list if len(node.parents) == 0] 57 | return Graph(roots) 58 | 59 | 60 | class DataframeReader(ABC): 61 | """Abstract Base Class for reading in checkpointing files.""" 62 | 63 | def __init__(self, filename): 64 | self.filename = filename 65 | 66 | @abstractmethod 67 | def _read_dataframe_from_file(self, **kwargs): 68 | pass 69 | 70 | def read(self, **kwargs): 71 | df = self._read_dataframe_from_file(**kwargs) 72 | rel_dict = _get_parents_and_children(df) 73 | graph = _reconstruct_graph(df, rel_dict) 74 | graph.enumerate_traverse() 75 | exc_metrics = df.iloc[0, df.columns.get_loc("exc_metrics")] 76 | inc_metrics = df.iloc[0, df.columns.get_loc("inc_metrics")] 77 | default_metric = df.iloc[0, df.columns.get_loc("default_metric")] 78 | df.drop( 79 | columns=[ 80 | "children", 81 | "parents", 82 | "exc_metrics", 83 | "inc_metrics", 84 | "default_metric", 85 | ], 86 | inplace=True, 87 | ) 88 | return hatchet.graphframe.GraphFrame( 89 | graph, 90 | df, 91 | exc_metrics=exc_metrics, 92 | inc_metrics=inc_metrics, 93 | default_metric=default_metric, 94 | ) 95 | 96 | 97 | class InvalidDataFrameIndex(Exception): 98 | """Raised when the DataFrame index is of an invalid type.""" 99 | -------------------------------------------------------------------------------- /hatchet/readers/hdf5_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import warnings 7 | import pandas as pd 8 | import sys 9 | from .dataframe_reader import DataframeReader 10 | 11 | 12 | class HDF5Reader(DataframeReader): 13 | def __init__(self, filename): 14 | # TODO Remove Arguments when Python 2.7 support is dropped 15 | if sys.version_info[0] == 2: 16 | super(HDF5Reader, self).__init__(filename) 17 | else: 18 | super().__init__(filename) 19 | 20 | def _read_dataframe_from_file(self, **kwargs): 21 | df = None 22 | with warnings.catch_warnings(): 23 | warnings.filterwarnings("ignore", category=Warning) 24 | df = pd.read_hdf(self.filename, **kwargs) 25 | return df 26 | -------------------------------------------------------------------------------- /hatchet/readers/json_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import json 7 | 8 | import pandas as pd 9 | 10 | import hatchet.graphframe 11 | from hatchet.node import Node 12 | from hatchet.graph import Graph 13 | from hatchet.frame import Frame 14 | 15 | 16 | class JsonReader: 17 | """Create a GraphFrame from a json string of the following format. 18 | 19 | Return: 20 | (GraphFrame): graphframe containing data from dictionaries 21 | """ 22 | 23 | def __init__(self, json_spec): 24 | """Read from a json string specification of a graphframe 25 | 26 | json (string): Json specification of a graphframe. 27 | """ 28 | self.spec_dict = json.loads(json_spec) 29 | 30 | def read(self): 31 | roots = [] 32 | for graph_spec in self.spec_dict["graph"]: 33 | # turn frames into nodes 34 | for nid, value in graph_spec.items(): 35 | graph_spec[nid]["data"] = Node(Frame(value["data"]), hnid=int(nid)) 36 | 37 | # connect nodes 38 | for nid, value in graph_spec.items(): 39 | for child in value["children"]: 40 | child = str(child) 41 | value["data"].add_child(graph_spec[child]["data"]) 42 | graph_spec[child]["data"].add_parent(value["data"]) 43 | 44 | for nid, value in graph_spec.items(): 45 | if len(value["data"].parents) == 0: 46 | roots.append(value["data"]) 47 | 48 | grph = Graph(roots) 49 | 50 | # make the dataframes 51 | dataframe = pd.DataFrame(self.spec_dict["dataframe"]) 52 | for graph_spec in self.spec_dict["graph"]: 53 | dataframe["node"] = dataframe["node"].map( 54 | lambda n: graph_spec[str(n)]["data"] if (str(n) in graph_spec) else n 55 | ) 56 | dataframe.set_index(self.spec_dict["dataframe_indices"], inplace=True) 57 | 58 | return hatchet.graphframe.GraphFrame( 59 | grph, 60 | dataframe, 61 | self.spec_dict["exclusive_metrics"], 62 | self.spec_dict["inclusive_metrics"], 63 | ) 64 | -------------------------------------------------------------------------------- /hatchet/readers/pyinstrument_reader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import json 7 | 8 | import pandas as pd 9 | 10 | import hatchet.graphframe 11 | from hatchet.node import Node 12 | from hatchet.graph import Graph 13 | from hatchet.frame import Frame 14 | 15 | 16 | class PyinstrumentReader: 17 | def __init__(self, filename): 18 | self.pyinstrument_json_filename = filename 19 | self.graph_dict = {} 20 | self.list_roots = [] 21 | self.node_dicts = [] 22 | 23 | def create_graph(self): 24 | def parse_node_literal(child_dict, hparent): 25 | """Create node_dict for one node and then call the function 26 | recursively on all children.""" 27 | 28 | hnode = Node( 29 | Frame({"name": child_dict["function"], "type": "function"}), hparent 30 | ) 31 | 32 | child_node_dict = { 33 | "node": hnode, 34 | "name": child_dict["function"], 35 | "file": child_dict["file_path_short"], 36 | "line": child_dict["line_no"], 37 | "time": child_dict["time"], 38 | "time (inc)": child_dict["time"], 39 | "is_application_code": child_dict["is_application_code"], 40 | } 41 | 42 | hparent.add_child(hnode) 43 | self.node_dicts.append(child_node_dict) 44 | 45 | if "children" in child_dict: 46 | for child in child_dict["children"]: 47 | # Pyinstrument's time metric actually stores inclusive time. 48 | # To calculate exclusive time, we subtract the children's time 49 | # from the parent's time. 50 | child_node_dict["time"] -= child["time"] 51 | parse_node_literal(child, hnode) 52 | 53 | # start with creating a node_dict for each root 54 | graph_root = Node( 55 | Frame( 56 | {"name": self.graph_dict["root_frame"]["function"], "type": "function"} 57 | ), 58 | None, 59 | ) 60 | 61 | node_dict = { 62 | "node": graph_root, 63 | "name": self.graph_dict["root_frame"]["function"], 64 | "file": self.graph_dict["root_frame"]["file_path_short"], 65 | "line": self.graph_dict["root_frame"]["line_no"], 66 | "time": self.graph_dict["root_frame"]["time"], 67 | "time (inc)": self.graph_dict["root_frame"]["time"], 68 | "is_application_code": self.graph_dict["root_frame"]["is_application_code"], 69 | } 70 | 71 | self.node_dicts.append(node_dict) 72 | self.list_roots.append(graph_root) 73 | 74 | # call recursively on all children of root 75 | if "children" in self.graph_dict["root_frame"]: 76 | for child in self.graph_dict["root_frame"]["children"]: 77 | # Pyinstrument's time metric actually stores inclusive time. 78 | # To calculate exclusive time, we subtract the children's time 79 | # from the parent's time. 80 | node_dict["time"] -= child["time"] 81 | parse_node_literal(child, graph_root) 82 | 83 | graph = Graph(self.list_roots) 84 | graph.enumerate_traverse() 85 | 86 | return graph 87 | 88 | def read(self): 89 | with open(self.pyinstrument_json_filename) as pyinstrument_json: 90 | self.graph_dict = json.load(pyinstrument_json) 91 | 92 | graph = self.create_graph() 93 | 94 | dataframe = pd.DataFrame(data=self.node_dicts) 95 | dataframe.set_index(["node"], inplace=True) 96 | dataframe.sort_index(inplace=True) 97 | 98 | return hatchet.graphframe.GraphFrame(graph, dataframe, ["time"], ["time (inc)"]) 99 | -------------------------------------------------------------------------------- /hatchet/tests/callgrind.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import numpy as np 7 | 8 | from hatchet import GraphFrame 9 | from hatchet.readers.gprof_dot_reader import GprofDotReader 10 | 11 | roots = [ 12 | "psm_no_lock", 13 | "(below main)", 14 | "0x0000000000000ff0", 15 | "_dl_catch_error'2", 16 | "hwloc_topology_load", 17 | "ibv_get_device_list", 18 | "psm_allocate_vbufs", 19 | ] 20 | 21 | 22 | def test_graphframe(calc_pi_callgrind_dot): 23 | """Sanity test a GraphFrame object with known data.""" 24 | gf = GraphFrame.from_gprof_dot(str(calc_pi_callgrind_dot)) 25 | 26 | assert len(gf.dataframe.groupby("name")) == 105 27 | 28 | for col in gf.dataframe.columns: 29 | if col in ("time (inc)", "time"): 30 | assert gf.dataframe[col].dtype == np.float64 31 | elif col in ("name", "module", "node"): 32 | assert gf.dataframe[col].dtype == object 33 | 34 | # TODO: add tests to confirm values in dataframe 35 | 36 | 37 | def test_read_calc_pi_database(calc_pi_callgrind_dot): 38 | """Sanity check the GprofDot reader by examining a known input.""" 39 | reader = GprofDotReader(str(calc_pi_callgrind_dot)) 40 | 41 | list_roots = reader.create_graph() 42 | root_names = [] 43 | for root in list_roots: 44 | root_names.append(root.frame.attrs["name"]) 45 | 46 | assert all(rt in root_names for rt in roots) 47 | -------------------------------------------------------------------------------- /hatchet/tests/cprofile.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import numpy as np 7 | import re 8 | 9 | from hatchet import GraphFrame 10 | 11 | 12 | def test_graphframe(hatchet_cycle_pstats): 13 | gf = GraphFrame.from_cprofile(str(hatchet_cycle_pstats)) 14 | 15 | assert len(gf.dataframe.groupby("file")) == 4 16 | assert len(gf.dataframe.groupby("name")) == 9 17 | 18 | gf.dataframe.reset_index(inplace=True) 19 | 20 | for col in gf.dataframe.columns: 21 | if col in ("time (inc)", "time"): 22 | assert gf.dataframe[col].dtype == np.float64 23 | elif col in ("line", "numcalls", "nativecalls"): 24 | assert gf.dataframe[col].dtype == np.int64 25 | elif col in ("name", "type", "file", "module", "node"): 26 | assert gf.dataframe[col].dtype == object 27 | 28 | 29 | def test_tree(monkeypatch, hatchet_cycle_pstats): 30 | monkeypatch.setattr("sys.stdout.isatty", (lambda: False)) 31 | gf = GraphFrame.from_cprofile(str(hatchet_cycle_pstats)) 32 | 33 | output = gf.tree(metric_column="time") 34 | 35 | assert "g pstats_reader_test.py" in output 36 | assert " ~" in output 37 | 38 | output = gf.tree(metric_column="time (inc)") 39 | 40 | assert "f pstats_reader_test.py" in output 41 | assert re.match("(.|\n)*recursive(.|\n)*recursive", output) 42 | -------------------------------------------------------------------------------- /hatchet/tests/data/caliper-lulesh-cali/lulesh-annotation-profile.cali: -------------------------------------------------------------------------------- 1 | __rec=node,id=24,attr=10,data=588,parent=3 2 | __rec=node,id=27,attr=8,data=cali.channel,parent=24 3 | __rec=node,id=25,attr=8,data=cali.caliper.version,parent=24 4 | __rec=node,id=26,attr=25,data=2.2.0-dev 5 | __rec=node,id=28,attr=27,data=default,parent=26 6 | __rec=node,id=96,attr=10,data=21,parent=5 7 | __rec=node,id=97,attr=8,data=min#time.duration,parent=96 8 | __rec=node,id=105,attr=8,data=max#time.duration,parent=96 9 | __rec=node,id=111,attr=8,data=sum#time.duration,parent=96 10 | __rec=node,id=117,attr=8,data=avg#time.duration,parent=96 11 | __rec=node,id=147,attr=10,data=21,parent=1 12 | __rec=node,id=148,attr=8,data=count,parent=147 13 | __rec=ctx,ref=28,attr=97=105=111=117=148,data=0.001297=0.001297=0.001297=0.001297=1 14 | __rec=node,id=19,attr=10,data=268,parent=3 15 | __rec=node,id=20,attr=8,data=function,parent=19 16 | __rec=node,id=155,attr=20,data=main,parent=28 17 | __rec=ctx,ref=155,attr=97=105=111=117=148,data=0.000285=0.024836=0.025121=0.012561=2 18 | __rec=node,id=21,attr=8,data=loop,parent=19 19 | __rec=node,id=165,attr=21,data=lulesh.cycle,parent=155 20 | __rec=ctx,ref=165,attr=97=105=111=117=148,data=0.000004=0.000015=0.000206=0.000005=41 21 | __rec=node,id=166,attr=20,data=TimeIncrement,parent=165 22 | __rec=ctx,ref=166,attr=97=105=111=117=148,data=0.000008=0.000022=0.000102=0.000010=10 23 | __rec=node,id=167,attr=20,data=LagrangeLeapFrog,parent=165 24 | __rec=ctx,ref=167,attr=97=105=111=117=148,data=0.000004=0.000021=0.000226=0.000006=40 25 | __rec=node,id=168,attr=20,data=LagrangeNodal,parent=167 26 | __rec=ctx,ref=168,attr=97=105=111=117=148,data=0.000003=0.001466=0.012400=0.000620=20 27 | __rec=node,id=169,attr=20,data=CalcForceForNodes,parent=168 28 | __rec=ctx,ref=169,attr=97=105=111=117=148,data=0.000004=0.000385=0.001854=0.000093=20 29 | __rec=node,id=170,attr=20,data=CalcVolumeForceForElems,parent=169 30 | __rec=ctx,ref=170,attr=97=105=111=117=148,data=0.000007=0.000241=0.001841=0.000061=30 31 | __rec=node,id=171,attr=20,data=IntegrateStressForElems,parent=170 32 | __rec=ctx,ref=171,attr=97=105=111=117=148,data=0.005258=0.006681=0.056681=0.005668=10 33 | __rec=node,id=172,attr=20,data=CalcHourglassControlForElems,parent=170 34 | __rec=ctx,ref=172,attr=97=105=111=117=148,data=0.000675=0.006053=0.059643=0.002982=20 35 | __rec=node,id=173,attr=20,data=CalcFBHourglassForceForElems,parent=172 36 | __rec=ctx,ref=173,attr=97=105=111=117=148,data=0.007265=0.008769=0.080608=0.008061=10 37 | __rec=node,id=174,attr=20,data=LagrangeElements,parent=167 38 | __rec=ctx,ref=174,attr=97=105=111=117=148,data=0.000004=0.000146=0.001367=0.000034=40 39 | __rec=node,id=175,attr=20,data=CalcLagrangeElements,parent=174 40 | __rec=ctx,ref=175,attr=97=105=111=117=148,data=0.000004=0.000368=0.002862=0.000143=20 41 | __rec=node,id=176,attr=20,data=CalcKinematicsForElems,parent=175 42 | __rec=ctx,ref=176,attr=97=105=111=117=148,data=0.004971=0.006349=0.055700=0.005570=10 43 | __rec=node,id=177,attr=20,data=CalcQForElems,parent=174 44 | __rec=ctx,ref=177,attr=97=105=111=117=148,data=0.000227=0.003135=0.031685=0.001584=20 45 | __rec=node,id=178,attr=20,data=CalcMonotonicQForElems,parent=177 46 | __rec=ctx,ref=178,attr=97=105=111=117=148,data=0.001310=0.001522=0.014334=0.001433=10 47 | __rec=node,id=179,attr=20,data=ApplyMaterialPropertiesForElems,parent=174 48 | __rec=ctx,ref=179,attr=97=105=111=117=148,data=0.000004=0.000219=0.002410=0.000020=120 49 | __rec=node,id=180,attr=20,data=EvalEOSForElems,parent=179 50 | __rec=ctx,ref=180,attr=97=105=111=117=148,data=0.000016=0.000282=0.044917=0.000098=460 51 | __rec=node,id=181,attr=20,data=CalcEnergyForElems,parent=180 52 | __rec=ctx,ref=181,attr=97=105=111=117=148,data=0.000051=0.001470=0.095906=0.000274=350 53 | __rec=node,id=182,attr=20,data=CalcTimeConstraintsForElems,parent=167 54 | __rec=ctx,ref=182,attr=97=105=111=117=148,data=0.000413=0.000518=0.004628=0.000463=10 55 | __rec=globals,ref=28 56 | -------------------------------------------------------------------------------- /hatchet/tests/data/caliper-ordered-duplicate-json/caliper-region-dup-node-order.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | [ 0, 0, 0.009792, null ], 4 | [ 1, 0, 0.018887, 0 ], 5 | [ 2, 0, 0.000154, 1 ], 6 | [ 3, 0, 0.000100, 2 ], 7 | [ 4, 0, 0.000159, 3 ], 8 | [ 5, 0, 0.038220, 4 ], 9 | [ 6, 0, 0.005107, 5 ], 10 | [ 7, 0, 0.004684, 6 ], 11 | [ 8, 0, 0.161500, 7 ], 12 | [ 8, 0, 0.163350, 8 ], 13 | [ 10, 0, 0.239820, 9 ], 14 | [ 11, 0, 0.003603, 10 ], 15 | [ 12, 0, 0.007004, 11 ], 16 | [ 13, 0, 0.168298, 12 ], 17 | [ 14, 0, 0.097482, 13 ], 18 | [ 15, 0, 0.038570, 14 ], 19 | [ 27, 0, 0.006099, 15 ], 20 | [ 29, 0, 0.102499, 16 ], 21 | [ 30, 0, 0.190896, 17 ], 22 | [ 61, 0, 0.010730, 18 ] 23 | ], 24 | "columns": [ "Node order", "mpi.rank", "time", "path" ], 25 | "column_metadata": [ { "is_value": true, "attribute.alias": "Node order" }, { "is_value": true }, { "is_value": true, "attribute.alias": "time", "attribute.unit": "sec" }, { "is_value": false } ], 26 | "nodes": [ { "label": "main", "column": "path" }, { "label": "lulesh.cycle", "column": "path", "parent": 0 }, { "label": "TimeIncrement", "column": "path", "parent": 1 }, { "label": "LagrangeLeapFrog", "column": "path", "parent": 1 }, { "label": "LagrangeNodal", "column": "path", "parent": 3 }, { "label": "CalcForceForNodes", "column": "path", "parent": 4 }, { "label": "CalcVolumeForceForElems", "column": "path", "parent": 5 }, { "label": "IntegrateStressForElems", "column": "path", "parent": 6 }, { "label": "CalcHourglassControlForElems", "column": "path", "parent": 6 }, { "label": "CalcFBHourglassForceForElems", "column": "path", "parent": 8 }, { "label": "LagrangeElements", "column": "path", "parent": 3 }, { "label": "CalcLagrangeElements", "column": "path", "parent": 10 }, { "label": "CalcKinematicsForElems", "column": "path", "parent": 11 }, { "label": "CalcQForElems", "column": "path", "parent": 10 }, { "label": "CalcMonotonicQForElems", "column": "path", "parent": 13 }, { "label": "ApplyMaterialPropertiesForElems", "column": "path", "parent": 10 }, { "label": "EvalEOSForElems", "column": "path", "parent": 15 }, { "label": "CalcEnergyForElems", "column": "path", "parent": 16 }, { "label": "CalcTimeConstraintsForElems", "column": "path", "parent": 3 } ], 27 | "mpi.world.size": "1", 28 | "cali.caliper.version": "2.10.0-dev", 29 | "figure_of_merit": "218.037069", 30 | "elapsed_time": "1.238322", 31 | "Compiler Flags": "", 32 | "Built by": "yokelson", 33 | "Compiler Version": "10.3.1", 34 | "Compiler Name": "GNU", 35 | "region_balance": "1", 36 | "region_cost": "1", 37 | "num_regions": "11", 38 | "problem_size": "30", 39 | "iterations": "10", 40 | "threads": "1", 41 | "jobsize": "1", 42 | "cluster": "quartz", 43 | "cmdline": "[/usr/WS2/yokelson/caliper-examples/apps/LULESH/build/./lulesh2.0,-i,10]", 44 | "libraries": "[linux-vdso.so.1,/usr/workspace/yokelson/Caliper/install/lib64/libcaliper.so.2,/usr/tce/packages/mvapich2/mvapich2-2.3.6-gcc-10.3.1/lib/libmpi.so.12,/usr/tce/packages/mvapich2/mvapich2-2.3.6-gcc-10.3.1/lib/libmpicxx.so.12,/lib64/libstdc++.so.6,/lib64/libm.so.6,/lib64/libgcc_s.so.1,/lib64/libc.so.6,/lib64/libdl.so.2,/lib64/libpthread.so.0,/lib64/ld-linux-x86-64.so.2,/lib64/librt.so.1,/lib64/libgfortran.so.5,/lib64/libnuma.so.1,/lib64/libudev.so.1,/lib64/libpciaccess.so.0,/lib64/libXNVCtrl.so.0,/lib64/libXext.so.6,/lib64/libX11.so.6,/lib64/libxml2.so.2,/lib64/libibverbs.so.1,/lib64/libpsm2.so.2,/lib64/libquadmath.so.0,/usr/tce/backend/installations/linux-rhel8-x86_64/gcc-8.5.0/zlib-1.2.13-pwvjwcyuqiscffdc2x2lr7ti355xwohq/lib/libz.so.1,/lib64/libmount.so.1,/lib64/libxcb.so.1,/lib64/liblzma.so.5,/lib64/libnl-route-3.so.200,/lib64/libnl-3.so.200,/lib64/libblkid.so.1,/lib64/libuuid.so.1,/lib64/libselinux.so.1,/lib64/libXau.so.6,/lib64/libpcre2-8.so.0,/lib64/libnss_files.so.2,/usr/lib64/libibverbs/libefa-rdmav34.so,/usr/lib64/libibverbs/libvmw_pvrdma-rdmav34.so,/usr/lib64/libibverbs/libbnxt_re-rdmav34.so,/usr/lib64/libibverbs/librxe-rdmav34.so,/usr/lib64/libibverbs/libirdma-rdmav34.so,/usr/lib64/libibverbs/libmlx4-rdmav34.so,/usr/lib64/libibverbs/libhns-rdmav34.so,/usr/lib64/libibverbs/libhfi1verbs-rdmav34.so,/usr/lib64/libibverbs/libmlx5-rdmav34.so,/usr/lib64/libibverbs/libsiw-rdmav34.so,/usr/lib64/libibverbs/libcxgb4-rdmav34.so,/usr/lib64/libibverbs/libqedr-rdmav34.so]", 45 | "executablepath": "/usr/WS2/yokelson/caliper-examples/apps/LULESH/build/lulesh2.0", 46 | "launchdate": "1685046480", 47 | "user": "yokelson", 48 | "cali.channel": "hatchet-region-profile" 49 | } 50 | -------------------------------------------------------------------------------- /hatchet/tests/data/caliper-ordered-json/caliper-region-node-order.json: -------------------------------------------------------------------------------- 1 | { 2 | "data": [ 3 | [ 0, 0, 0.009792, null ], 4 | [ 1, 0, 0.018887, 0 ], 5 | [ 2, 0, 0.000154, 1 ], 6 | [ 3, 0, 0.000100, 2 ], 7 | [ 4, 0, 0.000159, 3 ], 8 | [ 5, 0, 0.038220, 4 ], 9 | [ 6, 0, 0.005107, 5 ], 10 | [ 7, 0, 0.004684, 6 ], 11 | [ 8, 0, 0.161500, 7 ], 12 | [ 9, 0, 0.163350, 8 ], 13 | [ 10, 0, 0.239820, 9 ], 14 | [ 11, 0, 0.003603, 10 ], 15 | [ 12, 0, 0.007004, 11 ], 16 | [ 13, 0, 0.168298, 12 ], 17 | [ 14, 0, 0.097482, 13 ], 18 | [ 15, 0, 0.038570, 14 ], 19 | [ 27, 0, 0.006099, 15 ], 20 | [ 29, 0, 0.102499, 16 ], 21 | [ 30, 0, 0.190896, 17 ], 22 | [ 61, 0, 0.010730, 18 ] 23 | ], 24 | "columns": [ "Node order", "mpi.rank", "time", "path" ], 25 | "column_metadata": [ { "is_value": true, "attribute.alias": "Node order" }, { "is_value": true }, { "is_value": true, "attribute.alias": "time", "attribute.unit": "sec" }, { "is_value": false } ], 26 | "nodes": [ { "label": "main", "column": "path" }, { "label": "lulesh.cycle", "column": "path", "parent": 0 }, { "label": "TimeIncrement", "column": "path", "parent": 1 }, { "label": "LagrangeLeapFrog", "column": "path", "parent": 1 }, { "label": "LagrangeNodal", "column": "path", "parent": 3 }, { "label": "CalcForceForNodes", "column": "path", "parent": 4 }, { "label": "CalcVolumeForceForElems", "column": "path", "parent": 5 }, { "label": "IntegrateStressForElems", "column": "path", "parent": 6 }, { "label": "CalcHourglassControlForElems", "column": "path", "parent": 6 }, { "label": "CalcFBHourglassForceForElems", "column": "path", "parent": 8 }, { "label": "LagrangeElements", "column": "path", "parent": 3 }, { "label": "CalcLagrangeElements", "column": "path", "parent": 10 }, { "label": "CalcKinematicsForElems", "column": "path", "parent": 11 }, { "label": "CalcQForElems", "column": "path", "parent": 10 }, { "label": "CalcMonotonicQForElems", "column": "path", "parent": 13 }, { "label": "ApplyMaterialPropertiesForElems", "column": "path", "parent": 10 }, { "label": "EvalEOSForElems", "column": "path", "parent": 15 }, { "label": "CalcEnergyForElems", "column": "path", "parent": 16 }, { "label": "CalcTimeConstraintsForElems", "column": "path", "parent": 3 } ], 27 | "mpi.world.size": "1", 28 | "cali.caliper.version": "2.10.0-dev", 29 | "figure_of_merit": "218.037069", 30 | "elapsed_time": "1.238322", 31 | "Compiler Flags": "", 32 | "Built by": "yokelson", 33 | "Compiler Version": "10.3.1", 34 | "Compiler Name": "GNU", 35 | "region_balance": "1", 36 | "region_cost": "1", 37 | "num_regions": "11", 38 | "problem_size": "30", 39 | "iterations": "10", 40 | "threads": "1", 41 | "jobsize": "1", 42 | "cluster": "quartz", 43 | "cmdline": "[/usr/WS2/yokelson/caliper-examples/apps/LULESH/build/./lulesh2.0,-i,10]", 44 | "libraries": "[linux-vdso.so.1,/usr/workspace/yokelson/Caliper/install/lib64/libcaliper.so.2,/usr/tce/packages/mvapich2/mvapich2-2.3.6-gcc-10.3.1/lib/libmpi.so.12,/usr/tce/packages/mvapich2/mvapich2-2.3.6-gcc-10.3.1/lib/libmpicxx.so.12,/lib64/libstdc++.so.6,/lib64/libm.so.6,/lib64/libgcc_s.so.1,/lib64/libc.so.6,/lib64/libdl.so.2,/lib64/libpthread.so.0,/lib64/ld-linux-x86-64.so.2,/lib64/librt.so.1,/lib64/libgfortran.so.5,/lib64/libnuma.so.1,/lib64/libudev.so.1,/lib64/libpciaccess.so.0,/lib64/libXNVCtrl.so.0,/lib64/libXext.so.6,/lib64/libX11.so.6,/lib64/libxml2.so.2,/lib64/libibverbs.so.1,/lib64/libpsm2.so.2,/lib64/libquadmath.so.0,/usr/tce/backend/installations/linux-rhel8-x86_64/gcc-8.5.0/zlib-1.2.13-pwvjwcyuqiscffdc2x2lr7ti355xwohq/lib/libz.so.1,/lib64/libmount.so.1,/lib64/libxcb.so.1,/lib64/liblzma.so.5,/lib64/libnl-route-3.so.200,/lib64/libnl-3.so.200,/lib64/libblkid.so.1,/lib64/libuuid.so.1,/lib64/libselinux.so.1,/lib64/libXau.so.6,/lib64/libpcre2-8.so.0,/lib64/libnss_files.so.2,/usr/lib64/libibverbs/libefa-rdmav34.so,/usr/lib64/libibverbs/libvmw_pvrdma-rdmav34.so,/usr/lib64/libibverbs/libbnxt_re-rdmav34.so,/usr/lib64/libibverbs/librxe-rdmav34.so,/usr/lib64/libibverbs/libirdma-rdmav34.so,/usr/lib64/libibverbs/libmlx4-rdmav34.so,/usr/lib64/libibverbs/libhns-rdmav34.so,/usr/lib64/libibverbs/libhfi1verbs-rdmav34.so,/usr/lib64/libibverbs/libmlx5-rdmav34.so,/usr/lib64/libibverbs/libsiw-rdmav34.so,/usr/lib64/libibverbs/libcxgb4-rdmav34.so,/usr/lib64/libibverbs/libqedr-rdmav34.so]", 45 | "executablepath": "/usr/WS2/yokelson/caliper-examples/apps/LULESH/build/lulesh2.0", 46 | "launchdate": "1685046480", 47 | "user": "yokelson", 48 | "cali.channel": "hatchet-region-profile" 49 | } 50 | -------------------------------------------------------------------------------- /hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle-py2.pstats: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle-py2.pstats -------------------------------------------------------------------------------- /hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle.pstats: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/cprofile-hatchet-pstats/cprofile-cycle.pstats -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000000-000-a8c00471-152577-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000000-000-a8c00471-152577-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000000-001-a8c00471-152577-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000000-001-a8c00471-152577-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000001-000-a8c00471-152578-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000001-000-a8c00471-152578-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000001-001-a8c00471-152578-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000001-001-a8c00471-152578-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000002-000-a8c00471-152579-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000002-000-a8c00471-152579-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000002-001-a8c00471-152579-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000002-001-a8c00471-152579-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000003-000-a8c00471-152580-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000003-000-a8c00471-152580-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000003-001-a8c00471-152580-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000003-001-a8c00471-152580-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000004-000-a8c00471-152581-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000004-000-a8c00471-152581-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000004-001-a8c00471-152581-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000004-001-a8c00471-152581-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000005-000-a8c00471-152582-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000005-000-a8c00471-152582-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000005-001-a8c00471-152582-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000005-001-a8c00471-152582-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000006-000-a8c00471-152583-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000006-000-a8c00471-152583-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000006-001-a8c00471-152583-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000006-001-a8c00471-152583-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000007-000-a8c00471-152584-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000007-000-a8c00471-152584-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000007-001-a8c00471-152584-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000007-001-a8c00471-152584-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000008-000-a8c00471-152585-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000008-000-a8c00471-152585-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000008-001-a8c00471-152585-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000008-001-a8c00471-152585-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000009-000-a8c00471-152586-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000009-000-a8c00471-152586-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000009-001-a8c00471-152586-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-allgather-database/1.osu_allgather-000009-001-a8c00471-152586-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000000-000-a8c00270-160443-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000000-000-a8c00270-160443-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000001-000-a8c00270-160444-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000001-000-a8c00270-160444-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000002-000-a8c00270-160445-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000002-000-a8c00270-160445-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000003-000-a8c00270-160446-0.metric-db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-cpi-database/1.cpi-000003-000-a8c00270-160446-0.metric-db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-cpi-database/src/cpi.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ 2 | /* 3 | * (C) 2001 by Argonne National Laboratory. 4 | * See COPYRIGHT in top-level directory. 5 | */ 6 | 7 | #include "mpi.h" 8 | #include 9 | #include 10 | 11 | double f(double); 12 | 13 | double f(double a) 14 | { 15 | return (4.0 / (1.0 + a*a)); 16 | } 17 | 18 | int main(int argc,char *argv[]) 19 | { 20 | int n, myid, numprocs, i; 21 | double PI25DT = 3.141592653589793238462643; 22 | double mypi, pi, h, sum, x; 23 | double startwtime = 0.0, endwtime; 24 | int namelen; 25 | char processor_name[MPI_MAX_PROCESSOR_NAME]; 26 | 27 | MPI_Init(&argc,&argv); 28 | MPI_Comm_size(MPI_COMM_WORLD,&numprocs); 29 | MPI_Comm_rank(MPI_COMM_WORLD,&myid); 30 | MPI_Get_processor_name(processor_name,&namelen); 31 | 32 | fprintf(stdout,"Process %d of %d is on %s\n", 33 | myid, numprocs, processor_name); 34 | fflush(stdout); 35 | 36 | n = 10000; /* default # of rectangles */ 37 | if (myid == 0) 38 | startwtime = MPI_Wtime(); 39 | 40 | MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD); 41 | 42 | h = 1.0 / (double) n; 43 | sum = 0.0; 44 | /* A slightly better approach starts from large i and works back */ 45 | for (i = myid + 1; i <= n; i += numprocs) 46 | { 47 | x = h * ((double)i - 0.5); 48 | sum += f(x); 49 | } 50 | mypi = h * sum; 51 | 52 | MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 53 | 54 | if (myid == 0) { 55 | endwtime = MPI_Wtime(); 56 | printf("pi is approximately %.16f, Error is %.16f\n", 57 | pi, fabs(pi - PI25DT)); 58 | printf("wall clock time = %f\n", endwtime-startwtime); 59 | fflush(stdout); 60 | } 61 | 62 | MPI_Finalize(); 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-gamess/cct.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-gamess/cct.db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-gamess/meta.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-gamess/meta.db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-gamess/profile.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-gamess/profile.db -------------------------------------------------------------------------------- /hatchet/tests/data/hpctoolkit-gamess/trace.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/tests/data/hpctoolkit-gamess/trace.db -------------------------------------------------------------------------------- /hatchet/tests/data/json/hatchet-graph-literal.json: -------------------------------------------------------------------------------- 1 | {"graph":[{"0":{"data":{"name":"foo","type":"function"},"children":[1,4,12]},"1":{"data":{"name":"bar","type":"None"},"children":[2,3]},"2":{"data":{"name":"baz","type":"function"},"children":[]},"3":{"data":{"name":"grault","type":"None"},"children":[]},"4":{"data":{"name":"qux","type":"function"},"children":[5]},"5":{"data":{"name":"quux","type":"None"},"children":[6]},"6":{"data":{"name":"corge","type":"function"},"children":[7,11,10]},"7":{"data":{"name":"bar","type":"None"},"children":[8,9]},"8":{"data":{"name":"baz","type":"function"},"children":[]},"9":{"data":{"name":"grault","type":"None"},"children":[]},"10":{"data":{"name":"garply","type":"function"},"children":[]},"11":{"data":{"name":"grault","type":"None"},"children":[]},"12":{"data":{"name":"waldo","type":"function"},"children":[13,19]},"13":{"data":{"name":"fred","type":"function"},"children":[14,15]},"14":{"data":{"name":"plugh","type":"function"},"children":[]},"15":{"data":{"name":"xyzzy","type":"function"},"children":[16]},"16":{"data":{"name":"thud","type":"function"},"children":[17,18]},"17":{"data":{"name":"baz","type":"function"},"children":[]},"18":{"data":{"name":"garply","type":"function"},"children":[]},"19":{"data":{"name":"garply","type":"function"},"children":[]}},{"20":{"data":{"name":"waldo","type":"function"},"children":[21]},"21":{"data":{"name":"bar","type":"None"},"children":[22,23]},"22":{"data":{"name":"baz","type":"function"},"children":[]},"23":{"data":{"name":"grault","type":"None"},"children":[]}}],"dataframe_indices":["node"],"dataframe":[{"node":0,"name":"foo","time (inc)":135,"time":0},{"node":1,"name":"bar","time (inc)":20,"time":5},{"node":2,"name":"baz","time (inc)":5,"time":5},{"node":3,"name":"grault","time (inc)":10,"time":10},{"node":4,"name":"qux","time (inc)":60,"time":0},{"node":5,"name":"quux","time (inc)":60,"time":5},{"node":6,"name":"corge","time (inc)":55,"time":10},{"node":7,"name":"bar","time (inc)":20,"time":5},{"node":8,"name":"baz","time (inc)":5,"time":5},{"node":9,"name":"grault","time (inc)":10,"time":10},{"node":10,"name":"garply","time (inc)":15,"time":15},{"node":11,"name":"grault","time (inc)":10,"time":10},{"node":12,"name":"waldo","time (inc)":55,"time":0},{"node":13,"name":"fred","time (inc)":40,"time":5},{"node":14,"name":"plugh","time (inc)":5,"time":5},{"node":15,"name":"xyzzy","time (inc)":30,"time":5},{"node":16,"name":"thud","time (inc)":25,"time":5},{"node":17,"name":"baz","time (inc)":5,"time":5},{"node":18,"name":"garply","time (inc)":15,"time":15},{"node":19,"name":"garply","time (inc)":15,"time":15},{"node":20,"name":"waldo","time (inc)":30,"time":10},{"node":21,"name":"bar","time (inc)":20,"time":5},{"node":22,"name":"baz","time (inc)":5,"time":5},{"node":23,"name":"grault","time (inc)":10,"time":10}],"inclusive_metrics":["time (inc)"],"exclusive_metrics":["time"]} -------------------------------------------------------------------------------- /hatchet/tests/executable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import os 7 | import stat 8 | 9 | import pytest 10 | 11 | from hatchet.util.executable import which 12 | 13 | 14 | @pytest.fixture 15 | def mock_cali_query(tmpdir): 16 | """Create a mock cali-query path.""" 17 | tmpdir = tmpdir.mkdir("tmp-bin") 18 | cali_query = tmpdir.join("cali-query") 19 | with cali_query.open("w") as file: 20 | file.write("") 21 | 22 | st = os.stat(str(cali_query)) 23 | os.chmod(str(cali_query), st.st_mode | stat.S_IEXEC) 24 | 25 | # save current PATH variable 26 | old_path = os.environ.get("PATH") 27 | # append tmpdir to PATH variable 28 | os.environ["PATH"] = "%s:%s" % (str(tmpdir), old_path) 29 | # send it 30 | yield tmpdir 31 | # restore original PATH variable 32 | os.environ["PATH"] = old_path 33 | 34 | 35 | def test_which(mock_cali_query): 36 | assert which("cali-query") 37 | -------------------------------------------------------------------------------- /hatchet/tests/frame.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import pytest 7 | 8 | from hatchet.frame import Frame 9 | 10 | 11 | def test_constructors(): 12 | f1 = Frame({"name": "foo", "file": "bar.c"}) 13 | f2 = Frame(name="foo", file="bar.c") 14 | f3 = Frame({"name": "foo"}, file="bar.c") 15 | f4 = Frame({"name": "foo", "file": "baz.h"}, file="bar.c") 16 | 17 | assert f1 == f2 == f3 == f4 18 | 19 | 20 | def test_no_attrs(): 21 | with pytest.raises(ValueError): 22 | Frame({}) 23 | 24 | with pytest.raises(ValueError): 25 | Frame() 26 | 27 | 28 | def test_comparison(): 29 | assert Frame(a=1) == Frame(a=1) 30 | 31 | assert Frame(a=1) < Frame(a=2) 32 | assert Frame(a=1) != Frame(a=2) 33 | assert Frame(a=2) > Frame(a=1) 34 | 35 | assert Frame(a=1) < Frame(b=1) 36 | assert Frame(a=1) != Frame(b=1) 37 | assert Frame(b=1) > Frame(a=1) 38 | 39 | assert Frame(a=1, b=1) < Frame(b=1, c=1) 40 | assert Frame(a=1, b=1) != Frame(b=1, c=1) 41 | assert Frame(b=1, c=1) > Frame(a=1, b=1) 42 | 43 | 44 | def test_copy(): 45 | f = Frame(a=1) 46 | assert f == f.copy() 47 | assert not (f != f.copy()) 48 | assert not (f > f.copy()) 49 | assert not (f < f.copy()) 50 | 51 | f = Frame(a="foo", b="bar", c="baz") 52 | assert f == f.copy() 53 | assert not (f != f.copy()) 54 | assert not (f > f.copy()) 55 | assert not (f < f.copy()) 56 | 57 | 58 | def test_getitem(): 59 | f = Frame(a=1, b=2, c=3) 60 | assert f["a"] == 1 61 | assert f["b"] == 2 62 | assert f["c"] == 3 63 | 64 | assert f.get("a") == 1 65 | assert f.get("b") == 2 66 | assert f.get("c") == 3 67 | 68 | assert f.get("d") is None 69 | assert f.get("d", "foo") == "foo" 70 | 71 | with pytest.raises(KeyError): 72 | assert f["d"] 73 | 74 | 75 | def test_values(): 76 | f = Frame(a=1, b=2, c=3) 77 | assert f.values("a") == 1 78 | assert f.values(["b"]) == (2,) 79 | assert f.values(("a", "a", "a")) == (1, 1, 1) 80 | assert f.values(["a", "b"]) == (1, 2) 81 | assert f.values(["a", "b", "c"]) == (1, 2, 3) 82 | assert f.values(["b", "c"]) == (2, 3) 83 | assert f.values("c") == 3 84 | 85 | assert f.values(["foo", "a", "bar", "c"]) == (None, 1, None, 3) 86 | 87 | 88 | def test_repr(): 89 | assert ( 90 | repr(Frame(foo="baz", bar="quux")) 91 | == "Frame({'bar': 'quux', 'foo': 'baz', 'type': 'None'})" 92 | ) 93 | 94 | 95 | def test_str(): 96 | assert ( 97 | str(Frame(foo="baz", bar="quux")) 98 | == "{'bar': 'quux', 'foo': 'baz', 'type': 'None'}" 99 | ) 100 | -------------------------------------------------------------------------------- /hatchet/tests/graph.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from hatchet.node import Node 7 | from hatchet.frame import Frame 8 | from hatchet.graph import Graph 9 | 10 | 11 | def test_from_lists(): 12 | """Ensure we can traverse roots in correct order without repeating a 13 | shared subdag. 14 | """ 15 | d = Node(Frame(name="d")) 16 | diamond_subdag = Node.from_lists(("a", ("b", d), ("c", d))) 17 | 18 | g = Graph.from_lists(("e", "f", diamond_subdag), ("g", diamond_subdag, "h")) 19 | assert list(g.traverse(attrs="name")) == ["e", "a", "b", "d", "c", "f", "g", "h"] 20 | 21 | 22 | def test_len_chain(): 23 | graph = Graph.from_lists(("a", "b", "c", "d", "e")) 24 | assert len(graph) == 5 25 | 26 | 27 | def test_len_diamond(): 28 | d = Node(Frame(name="d")) 29 | graph = Graph.from_lists(("a", ("b", d), ("c", d))) 30 | assert len(graph) == 4 31 | 32 | 33 | def test_len_tree(): 34 | graph = Graph.from_lists(("a", ("b", "d"), ("c", "d"))) 35 | assert len(graph) == 5 36 | 37 | 38 | def test_copy(): 39 | d = Node(Frame(name="d")) 40 | diamond_subdag = Node.from_lists(("a", ("b", d), ("c", d))) 41 | g = Graph.from_lists(("e", "f", diamond_subdag), ("g", diamond_subdag, "h")) 42 | 43 | gc = g.copy() 44 | 45 | assert gc == g 46 | assert gc.node_ordering == g.node_ordering 47 | 48 | 49 | def test_union_dag(): 50 | # make graphs g1, g2, and g3, where you know g3 is the union of g1 and g2 51 | c = Node.from_lists(("c", "d")) 52 | g1 = Graph.from_lists(("a", ("b", c), ("e", c, "f"))) 53 | 54 | d = Node(Frame(name="d")) 55 | g2 = Graph.from_lists(("a", ("b", ("c", d)), ("e", d, "f"))) 56 | 57 | d2 = Node(Frame(name="d")) 58 | c2 = Node.from_lists(("c", d2)) 59 | g3 = Graph.from_lists(("a", ("b", c2), ("e", c2, d2, "f"))) 60 | 61 | assert g1 != g2 62 | 63 | g4 = g1.union(g2) 64 | 65 | assert g4 == g3 66 | 67 | 68 | def test_dag_is_not_tree(): 69 | g = Graph.from_lists(("b", "c"), ("d", "e")) 70 | assert not g.is_tree() 71 | 72 | d = Node(Frame(name="d")) 73 | diamond_subdag = Node.from_lists(("a", ("b", d), ("c", d))) 74 | g = Graph([diamond_subdag]) 75 | assert not g.is_tree() 76 | 77 | g = Graph.from_lists(("e", "f", diamond_subdag), ("g", diamond_subdag, "h")) 78 | assert not g.is_tree() 79 | 80 | 81 | def test_trees_are_trees(): 82 | g = Graph.from_lists(("a",)) 83 | assert g.is_tree() 84 | 85 | g = Graph.from_lists(("a", ("b", ("c")))) 86 | assert g.is_tree() 87 | 88 | g = Graph.from_lists(("a", "b", "c")) 89 | assert g.is_tree() 90 | 91 | g = Graph.from_lists( 92 | ("a", ("b", "e", "f", "g"), ("c", "e", "f", "g"), ("d", "e", "f", "g")) 93 | ) 94 | assert g.is_tree() 95 | -------------------------------------------------------------------------------- /hatchet/tests/graph_ops.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from hatchet import GraphFrame 7 | 8 | 9 | def test_graph_equal(mock_graph_literal): 10 | gf = GraphFrame.from_literal(mock_graph_literal) 11 | other = GraphFrame.from_literal(mock_graph_literal) 12 | 13 | assert gf.graph == other.graph 14 | 15 | 16 | def test_graph_not_equal(mock_graph_literal, calc_pi_hpct_db): 17 | gf = GraphFrame.from_literal(mock_graph_literal) 18 | other = GraphFrame.from_hpctoolkit(str(calc_pi_hpct_db)) 19 | 20 | assert gf.graph != other.graph 21 | 22 | 23 | def test_dag_not_equal(mock_dag_literal1, mock_dag_literal2): 24 | gf = GraphFrame.from_literal(mock_dag_literal1) 25 | other = GraphFrame.from_literal(mock_dag_literal2) 26 | 27 | assert gf.graph != other.graph 28 | 29 | 30 | def test_union_dag_same_structure(mock_dag_literal1): 31 | # make graphs g1 and g2 that you know are equal 32 | gf = GraphFrame.from_literal(mock_dag_literal1) 33 | other = GraphFrame.from_literal(mock_dag_literal1) 34 | 35 | g1 = gf.graph 36 | g2 = other.graph 37 | 38 | assert g1 == g2 39 | 40 | g3 = g1.union(g2) 41 | assert g3 is not g1 42 | assert g3 is not g2 43 | assert g3 == g1 44 | assert g3 == g2 45 | -------------------------------------------------------------------------------- /hatchet/tests/json_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from hatchet import GraphFrame 7 | 8 | 9 | def test_read_json(json_graphframe_specification): 10 | jgs = "" 11 | with open(json_graphframe_specification, "r") as f: 12 | jgs = f.read() 13 | gf = GraphFrame.from_json(jgs) 14 | 15 | assert len(gf.dataframe) == 24 16 | assert len(gf.graph) == 24 17 | assert gf.graph.roots[0].frame["name"] == "foo" 18 | 19 | 20 | def test_write_json(json_graphframe_specification): 21 | jgs = "" 22 | with open(json_graphframe_specification, "r") as f: 23 | jgs = f.read() 24 | gf = GraphFrame.from_json(jgs) 25 | json_out = gf.to_json() 26 | 27 | assert "".join(sorted("".join(sorted(jgs.split())))) == "".join( 28 | sorted("".join(json_out.split())) 29 | ) 30 | -------------------------------------------------------------------------------- /hatchet/tests/node.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import pytest 7 | 8 | from hatchet.node import Node, MultiplePathError 9 | from hatchet.frame import Frame 10 | from hatchet.graph import Graph 11 | 12 | 13 | def test_from_lists(): 14 | node = Node.from_lists("a") 15 | assert node.frame == Frame(name="a") 16 | 17 | a = Frame(name="a") 18 | b = Frame(name="b") 19 | c = Frame(name="c") 20 | 21 | node = Node.from_lists(["a", ["b", "c"]]) 22 | 23 | assert node.frame == a 24 | assert node.children[0].frame == b 25 | assert node.children[0].children[0].frame == c 26 | 27 | 28 | def test_from_lists_value_error(): 29 | with pytest.raises(ValueError): 30 | Node.from_lists(object()) 31 | 32 | 33 | def test_traverse_pre(): 34 | node = Node(Frame(name="a")) 35 | assert list(node.traverse(attrs="name")) == ["a"] 36 | 37 | node = Node.from_lists(["a", ["b", "d", "e"], ["c", "f", "g"]]) 38 | assert list(node.traverse(attrs="name")) == ["a", "b", "d", "e", "c", "f", "g"] 39 | 40 | 41 | def test_traverse_post(): 42 | node = Node.from_lists(["a", ["b", "d", "e"], ["c", "f", "g"]]) 43 | assert list(node.traverse(order="post", attrs="name")) == [ 44 | "d", 45 | "e", 46 | "b", 47 | "f", 48 | "g", 49 | "c", 50 | "a", 51 | ] 52 | 53 | 54 | def test_traverse_dag(): 55 | d = Node(Frame(name="d")) 56 | node = Node.from_lists(["a", ["b", d], ["c", d]]) 57 | assert list(node.traverse(attrs="name")) == ["a", "b", "d", "c"] 58 | 59 | 60 | def test_node_repr(): 61 | d = Node(Frame(a=1, b=2, c=3)) 62 | assert repr(d) == "Node({'a': 1, 'b': 2, 'c': 3, 'type': 'None'})" 63 | 64 | 65 | def test_path(): 66 | d = Node(Frame(name="d", type="function")) 67 | node = Node.from_lists(["a", ["b", d]]) 68 | 69 | assert d.path() == ( 70 | Node(Frame(name="a")), 71 | Node(Frame(name="b")), 72 | Node(Frame(name="d", type="function")), 73 | ) 74 | assert d.parents[0].path() == (Node(Frame(name="a")), Node(Frame(name="b"))) 75 | assert node.path() == (Node(Frame(name="a")),) 76 | 77 | 78 | def test_paths(): 79 | d = Node(Frame(name="d")) 80 | Node.from_lists(["a", ["b", d], ["c", d]]) 81 | with pytest.raises(MultiplePathError): 82 | d.path() 83 | 84 | assert d.paths() == [ 85 | (Node(Frame(name="a")), Node(Frame(name="b")), Node(Frame(name="d"))), 86 | (Node(Frame(name="a")), Node(Frame(name="c")), Node(Frame(name="d"))), 87 | ] 88 | 89 | 90 | def test_traverse_paths(): 91 | d = Node(Frame(name="d")) 92 | diamond_subdag = Node.from_lists(("a", ("b", d), ("c", d))) 93 | 94 | g = Graph.from_lists(("e", "f", diamond_subdag), ("g", diamond_subdag, "h")) 95 | assert list(g.traverse(attrs="name")) == ["e", "a", "b", "d", "c", "f", "g", "h"] 96 | 97 | 98 | def check_dag_equal(): 99 | chain = Node.from_lists(("a", ("b", ("c", ("d",))))) 100 | 101 | d = Node(Frame(name="d")) 102 | diamond = Node.from_lists(("a", ("b", d), ("c", d))) 103 | 104 | tree = Node.from_lists( 105 | ("a", ("b", "e", "f", "g"), ("c", "e", "f", "g"), ("d", "e", "f", "g")) 106 | ) 107 | 108 | assert chain.dag_equal(chain) 109 | assert chain.dag_equal(chain.copy()) 110 | 111 | assert diamond.dag_equal(diamond) 112 | assert diamond.dag_equal(diamond.copy()) 113 | 114 | assert tree.dag_equal(tree) 115 | assert tree.dag_equal(tree.copy()) 116 | 117 | assert not chain.dag_equal(tree) 118 | assert not chain.dag_equal(diamond) 119 | 120 | assert not tree.dag_equal(chain) 121 | assert not tree.dag_equal(diamond) 122 | 123 | assert not diamond.dag_equal(chain) 124 | assert not diamond.dag_equal(tree) 125 | -------------------------------------------------------------------------------- /hatchet/tests/profiler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import hatchet as ht 7 | import pstats 8 | import os 9 | import pytest 10 | import sys 11 | from hatchet.util.profiler import Profiler 12 | 13 | 14 | decorated_calls = [ 15 | "copy", 16 | "deepcopy", 17 | "drop_index_levels", 18 | "filter", 19 | "squash", 20 | "tree", 21 | "to_dot", 22 | "__add__", 23 | "__imul__", 24 | "__isub__", 25 | "traverse", 26 | "copy", 27 | "__len__", 28 | ] 29 | 30 | 31 | def f(): 32 | """Dummy function for profiling""" 33 | for i in range(1000): 34 | for j in range(1000): 35 | i * j 36 | 37 | 38 | def test_profiler(): 39 | """Test the profiler start/stop, ensure that f() has been profiled""" 40 | prf = Profiler() 41 | 42 | prf.start() 43 | f() 44 | prf.stop() 45 | 46 | sts = pstats.Stats(prf._prf) 47 | assert "stats" in sts.__dict__ 48 | 49 | fn_names = [] 50 | for stat in sts.__dict__["stats"]: 51 | fn_names.append(stat[2]) 52 | 53 | assert "f" in fn_names 54 | 55 | assert isinstance(prf.__str__(), str) 56 | assert "(f)" in prf.__str__() 57 | 58 | os.remove(prf._output + ".pstats") 59 | 60 | 61 | def test_write_file(): 62 | """Test that write_to_file, writes a profile to the correct file.""" 63 | prf = Profiler() 64 | 65 | prf.start() 66 | f() 67 | prf.stop() 68 | 69 | prf.write_to_file("test.pstats") 70 | 71 | assert os.path.exists("test.pstats") 72 | 73 | sts = pstats.Stats("test.pstats") 74 | fn_names = [] 75 | for stat in sts.__dict__["stats"]: 76 | fn_names.append(stat[2]) 77 | 78 | assert "f" in fn_names 79 | 80 | os.remove("test.pstats") 81 | os.remove(prf._output + ".pstats") 82 | 83 | 84 | @pytest.mark.skipif(sys.version_info > (3, 8), reason="Temporarily allow this to fail.") 85 | def test_profiling_calc_pi(calc_pi_hpct_db): 86 | """Test debug wrapper as called from hpctoolkit.""" 87 | prf = Profiler() 88 | output_file = prf._output + ".pstats" 89 | 90 | prf.start() 91 | gf = ht.GraphFrame.from_hpctoolkit(str(calc_pi_hpct_db)) 92 | 93 | gf.copy() 94 | gf2 = gf.deepcopy() 95 | gf.tree() 96 | gf.to_dot() 97 | 98 | gf3 = gf + gf2 99 | gf *= gf2 100 | gf3 -= gf2 101 | 102 | gf.graph.traverse() 103 | gf.graph.copy() 104 | len(gf.graph) 105 | 106 | gf2 = gf.filter(lambda x: x["time"] > 0.01) 107 | 108 | gf2.squash() 109 | gf.drop_index_levels() 110 | 111 | prf.stop() 112 | 113 | assert os.path.exists(output_file) 114 | 115 | sts = pstats.Stats(output_file) 116 | 117 | fn_names = [] 118 | for stat in sts.__dict__["stats"]: 119 | fn_names.append(stat[2]) 120 | 121 | for call in decorated_calls: 122 | assert call in fn_names 123 | 124 | os.remove(output_file) 125 | -------------------------------------------------------------------------------- /hatchet/tests/pyinstrument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import json 7 | 8 | import numpy as np 9 | 10 | from hatchet import GraphFrame 11 | 12 | 13 | def test_graphframe(hatchet_pyinstrument_json): 14 | """Sanity test a GraphFrame object with known data.""" 15 | 16 | def test_children(child_dict, gf): 17 | 18 | df = gf.dataframe.loc[ 19 | (gf.dataframe["name"] == child_dict["function"]) 20 | & (gf.dataframe["file"] == child_dict["file_path_short"]) 21 | & (gf.dataframe["line"] == child_dict["line_no"]) 22 | & (gf.dataframe["time (inc)"] == child_dict["time"]) 23 | & (gf.dataframe["is_application_code"] == child_dict["is_application_code"]) 24 | ] 25 | 26 | assert len(df.index) == 1 27 | assert df.index[0].frame["name"] == child_dict["function"] 28 | 29 | child_number = 0 30 | if "children" in child_dict: 31 | for child in child_dict["children"]: 32 | # to calculate the number of children 33 | child_number += 1 34 | test_children(child, gf) 35 | 36 | # number of children should be the same. 37 | assert len(df.index[0].children) == child_number 38 | 39 | # create a graphframe using from_pyinstrument 40 | gf = GraphFrame.from_pyinstrument(str(hatchet_pyinstrument_json)) 41 | 42 | graph_dict = [] 43 | # read directly from the input file to compare it with the graphframe. 44 | with open(str(hatchet_pyinstrument_json)) as pyinstrument_json: 45 | graph_dict = json.load(pyinstrument_json) 46 | 47 | # roots should be the same 48 | assert graph_dict["root_frame"]["function"] == gf.graph.roots[0].frame["name"] 49 | 50 | child_number = 0 51 | if "children" in graph_dict["root_frame"]: 52 | for child in graph_dict["root_frame"]["children"]: 53 | # to calculate the number of children 54 | child_number += 1 55 | test_children(child, gf) 56 | 57 | # number of children should be the same. 58 | assert len(gf.graph.roots[0].children) == child_number 59 | 60 | assert len(gf.dataframe.groupby("name")) == 44 61 | 62 | gf.dataframe.reset_index(inplace=True) 63 | 64 | for col in gf.dataframe.columns: 65 | if col in ("time (inc)", "time"): 66 | assert gf.dataframe[col].dtype == np.float64 67 | elif col in ("line"): 68 | assert gf.dataframe[col].dtype == np.int64 69 | elif col in ("is_application_code"): 70 | assert gf.dataframe[col].dtype == bool 71 | elif col in ("name", "type", "file", "node"): 72 | assert gf.dataframe[col].dtype == object 73 | 74 | 75 | def test_tree(monkeypatch, hatchet_pyinstrument_json): 76 | """Sanity test a GraphFrame object with known data.""" 77 | monkeypatch.setattr("sys.stdout.isatty", (lambda: False)) 78 | gf = GraphFrame.from_pyinstrument(str(hatchet_pyinstrument_json)) 79 | 80 | output = gf.tree(metric_column="time") 81 | 82 | assert "0.000 examples.py" in output 83 | assert "0.025 read hatchet/readers/caliper_reader.py" in output 84 | 85 | output = gf.tree(metric_column="time (inc)") 86 | 87 | assert "0.478 examples.py" in output 88 | assert "0.063 from_caliper_json hatchet/graphframe.py" in output 89 | 90 | 91 | def test_graphframe_to_literal(hatchet_pyinstrument_json): 92 | """Sanity test a GraphFrame object with known data.""" 93 | gf = GraphFrame.from_pyinstrument(str(hatchet_pyinstrument_json)) 94 | graph_literal = gf.to_literal() 95 | 96 | gf2 = GraphFrame.from_literal(graph_literal) 97 | 98 | assert len(gf.graph) == len(gf2.graph) 99 | -------------------------------------------------------------------------------- /hatchet/tests/spotdb_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import pytest 7 | 8 | from hatchet import GraphFrame 9 | from hatchet.readers.spotdb_reader import SpotDatasetReader, SpotDBReader 10 | 11 | spotdb = pytest.importorskip("spotdb") 12 | 13 | 14 | def test_spot_dataset_reader(): 15 | """Sanity-check the Spot dataset reader""" 16 | 17 | regionprofile = { 18 | "a/b/c": {"m": 20, "m#inclusive": 20}, 19 | "a/b": {"m#inclusive": 40}, 20 | "a": {"m#inclusive": 42}, 21 | } 22 | metadata = {"launchdate": 123456789} 23 | attr_info = { 24 | "m": {"type": "double"}, 25 | "m#inclusive": {"type": "int", "alias": "M Alias"}, 26 | } 27 | 28 | reader = SpotDatasetReader(regionprofile, metadata, attr_info) 29 | gf = reader.read(default_metric="M Alias (inc)") 30 | 31 | assert len(gf.dataframe) == 3 32 | assert set(gf.dataframe.columns) == {"name", "m", "M Alias (inc)"} 33 | 34 | assert gf.metadata["launchdate"] == metadata["launchdate"] 35 | assert gf.default_metric == "M Alias (inc)" 36 | 37 | 38 | @pytest.mark.skipif(not spotdb, reason="spotdb module not available") 39 | def test_spotdb_reader(spotdb_data): 40 | """Sanity check for the SpotDB reader""" 41 | 42 | db = spotdb_data 43 | 44 | reader = SpotDBReader(db) 45 | gfs = reader.read() 46 | 47 | assert len(gfs) == 4 48 | 49 | metrics = {"Total time (inc)", "Avg time/rank (inc)"} 50 | 51 | assert len(gfs[0].dataframe) > 2 52 | assert gfs[0].default_metric == "Total time (inc)" 53 | assert metrics < set(gfs[0].dataframe.columns) 54 | assert metrics < set(gfs[3].dataframe.columns) 55 | 56 | assert "launchdate" in gfs[0].metadata.keys() 57 | 58 | 59 | @pytest.mark.skipif(not spotdb, reason="spotdb module not available") 60 | def test_from_spotdb(spotdb_data): 61 | """Sanity check for GraphFrame.from_spotdb""" 62 | 63 | db = spotdb_data 64 | runs = db.get_all_run_ids() 65 | gfs = GraphFrame.from_spotdb(spotdb_data, runs[0:2]) 66 | 67 | assert len(gfs) == 2 68 | 69 | metrics = {"Total time (inc)", "Avg time/rank (inc)"} 70 | 71 | assert len(gfs[0].dataframe) > 2 72 | assert gfs[0].default_metric == "Total time (inc)" 73 | assert metrics < set(gfs[0].dataframe.columns) 74 | 75 | assert "launchdate" in gfs[0].metadata.keys() 76 | -------------------------------------------------------------------------------- /hatchet/tests/tau.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import numpy as np 7 | 8 | from hatchet import GraphFrame 9 | 10 | 11 | def test_graphframe(tau_profile_dir): 12 | """Sanity test a GraphFrame object with known data.""" 13 | gf = GraphFrame.from_tau(str(tau_profile_dir)) 14 | 15 | for col in gf.dataframe.columns: 16 | if col in ("time (inc)", "time"): 17 | assert gf.dataframe[col].dtype == np.float64 18 | elif col in ("line"): 19 | assert gf.dataframe[col].dtype == np.int64 20 | elif col in ("name", "node"): 21 | assert gf.dataframe[col].dtype == object 22 | 23 | # TODO: add tests to confirm values in dataframe 24 | 25 | 26 | def test_tree(monkeypatch, tau_profile_dir): 27 | """Sanity test a GraphFrame object with known data.""" 28 | monkeypatch.setattr("sys.stdout.isatty", (lambda: False)) 29 | gf = GraphFrame.from_tau(str(tau_profile_dir)) 30 | 31 | # check the tree for rank 0 32 | output = gf.tree(metric_column="time", rank=0) 33 | 34 | assert "449.000 .TAU application" in output 35 | assert "4458.000 MPI_Finalize()" in output 36 | assert "218.000 MPI_Bcast()" in output 37 | 38 | # check the tree for rank 1 39 | output = gf.tree(metric_column="time", rank=1) 40 | 41 | assert "419.000 .TAU application" in output 42 | assert "4894.000 MPI_Finalize()" in output 43 | assert "333.000 MPI_Bcast()" in output 44 | 45 | 46 | def test_children(tau_profile_dir): 47 | gf = GraphFrame.from_tau(str(tau_profile_dir)) 48 | root = gf.graph.roots 49 | root_children = [ 50 | "MPI_Init()", 51 | "MPI_Comm_size()", 52 | "MPI_Comm_rank()", 53 | "MPI_Get_processor_name()", 54 | "MPI_Bcast()", 55 | "MPI_Reduce()", 56 | "MPI_Finalize()", 57 | ] 58 | 59 | # check if only one root node is created 60 | assert len(root) == 1 61 | 62 | # check if root has right children #TODO: can be improved 63 | for child in root[0].children: 64 | assert child.frame["name"] in root_children 65 | 66 | 67 | def test_graphframe_to_literal(tau_profile_dir): 68 | """Sanity test a GraphFrame object with known data.""" 69 | gf = GraphFrame.from_tau(str(tau_profile_dir)) 70 | graph_literal = gf.to_literal() 71 | 72 | gf_literal = GraphFrame.from_literal(graph_literal) 73 | 74 | assert len(gf.graph) == len(gf_literal.graph) 75 | -------------------------------------------------------------------------------- /hatchet/tests/timemory_func.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import numpy as np 7 | 8 | timemory_avail = True 9 | try: 10 | from timemory.profiler import Profiler 11 | from timemory.trace import Tracer 12 | from timemory.component import WallClock, CpuClock 13 | except ImportError: 14 | timemory_avail = False 15 | 16 | class FakeComponent: 17 | def __init__(self, *args, **kwargs): 18 | pass 19 | 20 | @staticmethod 21 | def id(): 22 | pass 23 | 24 | class FakeProfiler: 25 | def __init__(self, *args, **kwargs): 26 | pass 27 | 28 | def __call__(self, func): 29 | pass 30 | 31 | WallClock = FakeComponent 32 | CpuClock = FakeComponent 33 | Profiler = FakeProfiler 34 | Tracer = FakeProfiler 35 | 36 | 37 | ncnt = 0 38 | components = [WallClock.id(), CpuClock.id()] 39 | 40 | 41 | def eval_func(arr, tol): 42 | """Dummy tolerance-checking function for profiling which introduces some arbitrary branching""" 43 | global ncnt 44 | max = np.max(arr) 45 | if ncnt % 3 != 0: 46 | avg = np.mean(arr) 47 | else: 48 | avg = max 49 | ncnt = ncnt + 1 50 | return True if avg < tol and max < tol else False 51 | 52 | 53 | @Profiler(components) 54 | def prof_func(arr, tol): 55 | """Dummy function for profiling""" 56 | while not eval_func(arr, tol): 57 | arr = arr - np.power(arr, 3) 58 | 59 | 60 | @Tracer(components) 61 | def trace_func(arr, tol): 62 | """Dummy function for tracing""" 63 | while not eval_func(arr, tol): 64 | arr = arr - np.power(arr, 3) 65 | -------------------------------------------------------------------------------- /hatchet/tests/timemory_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import numpy as np 7 | 8 | from hatchet import GraphFrame 9 | 10 | import pytest 11 | 12 | timemory_avail = True 13 | try: 14 | import timemory 15 | except ImportError: 16 | timemory_avail = False 17 | 18 | 19 | @pytest.mark.skipif(not timemory_avail, reason="timemory package not available") 20 | def test_graphframe(timemory_json_data): 21 | """Sanity test a GraphFrame object with known data.""" 22 | from timemory.component import WallClock 23 | 24 | wc_s = WallClock.id() # string identifier 25 | wc_v = WallClock.index() # enumeration id 26 | gf = GraphFrame.from_timemory(timemory_json_data, [wc_s]) 27 | 28 | assert len(gf.dataframe) == timemory.size([wc_v])[wc_v] 29 | 30 | for col in gf.dataframe.columns: 31 | if col in ("sum.wall_clock.inc", "sum.wall_clock"): 32 | assert gf.dataframe[col].dtype == np.float64 33 | elif col in ("nid", "rank"): 34 | assert gf.dataframe[col].dtype == np.int64 35 | elif col in ("name", "node"): 36 | assert gf.dataframe[col].dtype == object 37 | 38 | 39 | @pytest.mark.skipif(not timemory_avail, reason="timemory package not available") 40 | def test_tree(monkeypatch, timemory_json_data): 41 | """Sanity test a GraphFrame object with known data.""" 42 | monkeypatch.setattr("sys.stdout.isatty", (lambda: False)) 43 | gf = GraphFrame.from_timemory(timemory_json_data) 44 | 45 | output = gf.tree(metric_column="sum.wall_clock") 46 | print(output) 47 | 48 | 49 | @pytest.mark.skipif(not timemory_avail, reason="timemory package not available") 50 | def test_graphframe_to_literal(timemory_json_data): 51 | """Sanity test a GraphFrame object with known data.""" 52 | gf = GraphFrame.from_timemory(timemory_json_data) 53 | graph_literal = gf.to_literal() 54 | 55 | assert len(graph_literal) == len(gf.graph.roots) 56 | 57 | 58 | @pytest.mark.skipif(not timemory_avail, reason="timemory package not available") 59 | def test_default_metric(timemory_json_data): 60 | """Validation test for GraphFrame object using default metric field""" 61 | gf = GraphFrame.from_timemory(timemory_json_data) 62 | 63 | for func in ["tree", "to_dot", "to_flamegraph"]: 64 | lhs = "{}".format(getattr(gf, func)(gf.default_metric)) 65 | rhs = "{}".format(getattr(gf, func)()) 66 | assert lhs == rhs 67 | -------------------------------------------------------------------------------- /hatchet/util/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/util/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | dot_keywords = ["graph", "subgraph", "digraph", "node", "edge", "strict"] 7 | -------------------------------------------------------------------------------- /hatchet/util/deprecated.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import functools 7 | 8 | 9 | def deprecated_params(**old_to_new): 10 | def deco(f): 11 | @functools.wraps(f) 12 | def wrapper(*args, **kwargs): 13 | rename_kwargs(f.__name__, old_to_new, kwargs) 14 | return f(*args, **kwargs) 15 | 16 | return wrapper 17 | 18 | return deco 19 | 20 | 21 | def rename_kwargs(fname, old_to_new, kwargs): 22 | for old, new in old_to_new.items(): 23 | if old in kwargs: 24 | if new in kwargs: 25 | raise TypeError( 26 | '{}() received both "{}=" and "{}=".'.format(fname, old, new) 27 | ) 28 | 29 | # if parameter has been removed 30 | if not new: 31 | raise ValueError( 32 | '{}() parameter "{}=" will be removed in a future release. Please remove it from this script.'.format( 33 | fname, old 34 | ) 35 | ) 36 | # if parameter has been deprecated and renamed 37 | else: 38 | raise ValueError( 39 | '{}() parameter "{}=" has been deprecated, please use "{}=".'.format( 40 | fname, old, new 41 | ) 42 | ) 43 | kwargs[new] = kwargs.pop(old) 44 | -------------------------------------------------------------------------------- /hatchet/util/dot.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import matplotlib.cm 7 | import matplotlib.colors 8 | 9 | 10 | def trees_to_dot(roots, dataframe, metric, name, rank, thread, threshold): 11 | """Calls to_dot in turn for each tree in the graph/forest.""" 12 | text = ( 13 | "strict digraph {\n" 14 | "graph [bgcolor=transparent];\n" 15 | "node [penwidth=4, shape=circle];\n" 16 | "edge [penwidth=2];\n\n" 17 | ) 18 | 19 | all_nodes = "" 20 | all_edges = "" 21 | 22 | # call to_dot for each root in the graph 23 | visited = [] 24 | for root in roots: 25 | (nodes, edges) = to_dot( 26 | root, dataframe, metric, name, rank, thread, threshold, visited 27 | ) 28 | all_nodes += nodes 29 | all_edges += edges 30 | 31 | text += all_nodes + "\n" + all_edges + "\n}\n" 32 | 33 | return text 34 | 35 | 36 | def to_dot(hnode, dataframe, metric, name, rank, thread, threshold, visited): 37 | """Write to graphviz dot format.""" 38 | colormap = matplotlib.cm.Reds 39 | min_time = dataframe[metric].min() 40 | max_time = dataframe[metric].max() 41 | 42 | def add_nodes_and_edges(hnode): 43 | 44 | # set dataframe index based on if rank is a part of the index 45 | if "rank" in dataframe.index.names and "thread" in dataframe.index.names: 46 | df_index = (hnode, rank, thread) 47 | elif "rank" in dataframe.index.names: 48 | df_index = (hnode, rank) 49 | elif "thread" in dataframe.index.names: 50 | df_index = (hnode, thread) 51 | else: 52 | df_index = hnode 53 | 54 | node_time = dataframe.loc[df_index, metric] 55 | node_name = dataframe.loc[df_index, name] 56 | node_id = hnode._hatchet_nid 57 | 58 | weight = (node_time - min_time) / (max_time - min_time) 59 | color = matplotlib.colors.rgb2hex(colormap(weight)) 60 | 61 | # only display nodes whose metric is greater than some threshold 62 | if (node_time >= threshold * max_time) and (hnode not in visited): 63 | node_string = '"{0}" [color="{1}", label="{2}" shape=oval];\n'.format( 64 | node_id, color, node_name 65 | ) 66 | edge_string = "" 67 | 68 | # only display those edges where child's metric is greater than 69 | # threshold 70 | children = [] 71 | for child in hnode.children: 72 | if ( 73 | "rank" in dataframe.index.names 74 | and "thread" in dataframe.index.names 75 | ): 76 | df_index = (child, rank, thread) 77 | elif "rank" in dataframe.index.names: 78 | df_index = (child, rank) 79 | elif "thread" in dataframe.index.names: 80 | df_index = (child, thread) 81 | else: 82 | df_index = child 83 | 84 | child_time = dataframe.loc[df_index, metric] 85 | if child_time >= threshold * max_time: 86 | children.append(child) 87 | 88 | visited.append(hnode) 89 | for child in children: 90 | # add edges 91 | child_id = child._hatchet_nid 92 | 93 | edge_string += '"{0}" -> "{1}";\n'.format(node_id, child_id) 94 | (nodes, edges) = add_nodes_and_edges(child) 95 | node_string += nodes 96 | edge_string += edges 97 | else: 98 | node_string = "" 99 | edge_string = "" 100 | 101 | return (node_string, edge_string) 102 | 103 | # call add_nodes_and_edges on the root 104 | return add_nodes_and_edges(hnode) 105 | -------------------------------------------------------------------------------- /hatchet/util/executable.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import os 7 | 8 | 9 | def which(executable): 10 | """Finds an `executable` in the user's PATH like command-line which. 11 | 12 | Args: 13 | executable (str): executable to search for 14 | """ 15 | path = os.environ.get("PATH", "/usr/sbin:/usr/bin:/sbin:/bin") 16 | path = path.split(os.pathsep) 17 | 18 | for directory in path: 19 | exe = os.path.join(directory, executable) 20 | if os.path.isfile(exe) and os.access(exe, os.X_OK): 21 | return exe 22 | 23 | return None 24 | -------------------------------------------------------------------------------- /hatchet/util/profiler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import cProfile 7 | import traceback 8 | import sys 9 | import os 10 | 11 | from datetime import datetime 12 | 13 | try: 14 | from StringIO import StringIO # python2 15 | except ImportError: 16 | from io import StringIO # python3 17 | import pstats 18 | 19 | 20 | def print_incomptable_msg(stats_file): 21 | """ 22 | Function which makes the syntax cleaner in Profiler.write_to_file(). 23 | """ 24 | errmsg = """ Incompatible pstats file: {}\n Please run your code in Python {} to read in this file. """ 25 | if sys.version_info[0] == 2: 26 | print(errmsg.format(stats_file, 3)) 27 | if sys.version_info[0] == 3: 28 | print(errmsg.format(stats_file, 2.7)) 29 | traceback.print_exc() 30 | 31 | 32 | # class for profiling 33 | class Profiler: 34 | """ 35 | Wrapper class around cProfile. 36 | Exports a pstats file to be read by the pstats reader. 37 | """ 38 | 39 | def __init__(self): 40 | self._prf = cProfile.Profile() 41 | self._output = "hatchet-profile" 42 | self._active = False 43 | 44 | def start(self): 45 | """ 46 | Description: Place before the block of code to be profiled. 47 | """ 48 | if self._active: 49 | print( 50 | "Start dissallowed in scope where profiler is running. Please add Profiler.stop() before attempting start." 51 | ) 52 | raise 53 | 54 | self._active = True 55 | self._prf.enable() 56 | 57 | def stop(self): 58 | """ 59 | Description: Place at the end of the block of code being profiled. 60 | """ 61 | 62 | self._active = False 63 | self._prf.disable() 64 | self.write_to_file() 65 | 66 | def reset(self): 67 | """ 68 | Description: Resets the profilier. 69 | """ 70 | if self._active: 71 | print( 72 | "Reset dissallowed in scope where profiler is running. Please add Profiler.stop() before attempting reset." 73 | ) 74 | raise 75 | 76 | self._prf = cProfile.Profile() 77 | 78 | def __str__(self): 79 | """ 80 | Description: Writes stats object out as a string. 81 | """ 82 | s = StringIO() 83 | pstats.Stats(self._prf, stream=s).print_stats() 84 | return s.getvalue() 85 | 86 | def write_to_file(self, filename="", add_pstats_files=[]): 87 | """ 88 | Description: Write the pstats object to a binary 89 | file to be read in by an appropriate source. 90 | """ 91 | sts = pstats.Stats(self._prf) 92 | 93 | if len(add_pstats_files) > 0: 94 | for stats_file in add_pstats_files: 95 | try: 96 | sts.add(stats_file) 97 | except ValueError: 98 | print_incomptable_msg(stats_file) 99 | raise 100 | 101 | if filename == "": 102 | if os.path.exists(self._output + ".pstats"): 103 | now = datetime.now().strftime("%H%M%S") 104 | self.write_to_file( 105 | "{}_{}.pstats".format(self._output, now), add_pstats_files 106 | ) 107 | else: 108 | sts.dump_stats(self._output + ".pstats") 109 | else: 110 | if os.path.exists(filename): 111 | now = datetime.now().strftime("%H%M%S") 112 | filename = "{}_{}.pstats".format(filename, now) 113 | sts.dump_stats(filename) 114 | -------------------------------------------------------------------------------- /hatchet/util/timer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from collections import OrderedDict 7 | from contextlib import contextmanager 8 | from datetime import datetime 9 | from io import StringIO 10 | 11 | 12 | class Timer(object): 13 | """Simple phase timer with a context manager.""" 14 | 15 | def __init__(self): 16 | self._phase = None 17 | self._start_time = None 18 | self._times = OrderedDict() 19 | 20 | def start_phase(self, phase): 21 | now = datetime.now() 22 | delta = None 23 | 24 | if self._phase: 25 | delta = now - self._start_time 26 | self._times[self._phase] = delta 27 | 28 | self._phase = phase 29 | self._start_time = now 30 | return delta 31 | 32 | def end_phase(self): 33 | assert self._phase and self._start_time 34 | 35 | now = datetime.now() 36 | delta = now - self._start_time 37 | if self._times.get(self._phase): 38 | self._times[self._phase] = self._times.get(self._phase) + delta 39 | else: 40 | self._times[self._phase] = delta 41 | 42 | self._phase = None 43 | self._start_time = None 44 | 45 | def __str__(self): 46 | out = StringIO() 47 | out.write("Times:\n") 48 | for phase, delta in self._times.items(): 49 | out.write(" %-20s %.2fs\n" % (phase + ":", delta.total_seconds())) 50 | return out.getvalue() 51 | 52 | @contextmanager 53 | def phase(self, name): 54 | self.start_phase(name) 55 | yield 56 | self.end_phase() 57 | -------------------------------------------------------------------------------- /hatchet/version.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | __version_info__ = ("2024", "1", "3") 7 | __version__ = ".".join(__version_info__) 8 | -------------------------------------------------------------------------------- /hatchet/vis/README.md: -------------------------------------------------------------------------------- 1 | # Hatchet VIS Subdirectory 2 | 3 | This folder contains all the code related to the interactive Jupyter visualizations deployed with the Hatchet Library. 4 | 5 | ## Development Requirements 6 | 7 | The software in this subdirectory requires [Node.js and Node Package Manager](https://nodejs.org/en/download/) for the development and building of javascript code. 8 | 9 | ## Installing node packages 10 | 11 | One you have Node and NPM installed on your system, you can install all necessary node packages by running the following line in your terminal from this directory: 12 | 13 | ``` 14 | npm install 15 | ``` 16 | 17 | ## Building out javascript code with webpack 18 | 19 | To build out javascript into the static bundles used by the Jupyter visualizations, run the following line from this directory in your terminal: 20 | 21 | ``` 22 | npm run build 23 | ``` 24 | 25 | Alterntinatively if you are developing a visualization, you can run the following line to force bundles to automatically update when you change the javascript source code: 26 | 27 | ``` 28 | npm run watch 29 | ``` -------------------------------------------------------------------------------- /hatchet/vis/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | # This code comes from the equivalent file in Thicket: 7 | # https://github.com/LLNL/thicket 8 | import os 9 | import sys 10 | import subprocess 11 | 12 | _filenotfound_errmsg = """ 13 | Cannot find NPM! 14 | This is required to use hatchet.vis! 15 | Please install NPM and try again to import hatchet.vis! 16 | """ 17 | 18 | 19 | def check_npm(): 20 | if sys.version_info[0] == 2: 21 | from distutils.spawn import find_executable 22 | 23 | return find_executable("npm") is not None 24 | from shutil import which 25 | 26 | return which("npm") is not None 27 | 28 | 29 | def npm_build(vis_directory): 30 | print("Building hatchet.vis using NPM!") 31 | subprocess.check_output(["npm", "install", "-y"], cwd=vis_directory) 32 | subprocess.check_output(["npm", "run", "build"], cwd=vis_directory) 33 | 34 | 35 | # Get the absolute path to this __init__.py file 36 | # Store in curr_dir to minimize the number of variables 37 | curr_dir = os.path.realpath(os.path.expanduser(__file__)) 38 | # Get the hatchet/vis directory from the path to this __init__.py file 39 | curr_dir = os.path.abspath(os.path.join(curr_dir, os.pardir)) 40 | # Get the path to hatchet/vis/package-lock.json 41 | pkg_lock_file = os.path.abspath(os.path.join(curr_dir, "package-lock.json")) 42 | 43 | if not os.path.isfile(pkg_lock_file): 44 | if not check_npm(): 45 | raise FileNotFoundError(_filenotfound_errmsg) 46 | else: 47 | npm_build(curr_dir) 48 | -------------------------------------------------------------------------------- /hatchet/vis/external/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | # flake8: noqa 7 | from .roundtrip.roundtrip.manager import Roundtrip 8 | -------------------------------------------------------------------------------- /hatchet/vis/loader.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from IPython.core.magic import Magics, magics_class, line_magic 7 | from hatchet.external import Roundtrip as RT 8 | from hatchet import GraphFrame 9 | from os import path 10 | from os.path import dirname 11 | 12 | 13 | vis_dir = dirname(path.abspath(__file__)) 14 | 15 | 16 | def _gf_to_json(data): 17 | import json 18 | from pandas import Series 19 | 20 | def serialize(obj): 21 | if isinstance(obj, Series): 22 | return obj.to_json 23 | return obj.__dict__ 24 | 25 | try: 26 | if isinstance(data, GraphFrame): 27 | return json.dumps(data.to_literal(), default=serialize) 28 | else: 29 | with open("check", "w") as f: 30 | f.write(json.dumps(data, default=serialize)) 31 | return json.dumps(data, default=serialize) 32 | except ValueError: 33 | raise "Input data is not of type graphframe or json serializable." 34 | 35 | 36 | def _query_to_dict(json_query): 37 | import json 38 | 39 | return json_query 40 | 41 | return json.loads(json_query) 42 | 43 | 44 | @magics_class 45 | class CCT(Magics): 46 | def __init__(self, shell): 47 | super(CCT, self).__init__(shell) 48 | self.vis_dist = path.join(vis_dir, "static") 49 | 50 | @line_magic 51 | def cct(self, line): 52 | args = line.split(" ") 53 | 54 | RT.load_webpack(path.join(self.vis_dist, "cct_bundle.html"), cache=False) 55 | RT.var_to_js( 56 | args[0], "hatchet_tree_def", watch=False, to_js_converter=_gf_to_json 57 | ) 58 | 59 | RT.initialize() 60 | 61 | @line_magic 62 | def cct_fetch_query(self, line): 63 | args = line.split(" ") 64 | 65 | RT.fetch_data("jsNodeSelected", args[0], converter=_query_to_dict) 66 | 67 | 68 | def load_ipython_extension(ipython): 69 | ipython.register_magics(CCT) 70 | -------------------------------------------------------------------------------- /hatchet/vis/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hatchet-vis", 3 | "version": "0.0.1", 4 | "description": "Notebook-based VIS for Hatchet", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "build": "npx webpack build", 9 | "watch": "npx webpack build --watch" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/hatchet/hatchet.git" 14 | }, 15 | "keywords": [ 16 | "Performance-analysis", 17 | "Jupyter-notebook", 18 | "Visualization", 19 | "HPC" 20 | ], 21 | "author": "Hatchet developers", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/hatchet/hatchet/issues" 25 | }, 26 | "homepage": "https://github.com/hatchet/hatchet#readme", 27 | "devDependencies": { 28 | "@babel/core": "^7.16.0", 29 | "@babel/preset-env": "^7.16.0", 30 | "@babel/preset-react": "^7.16.0", 31 | "babel-loader": "^8.2.3", 32 | "eslint": "^7.32.0", 33 | "eslint-plugin-import": "^2.25.3", 34 | "eslint-plugin-node": "^11.1.0", 35 | "eslint-plugin-promise": "^5.2.0", 36 | "html-webpack-plugin": "^5.3.2", 37 | "webpack": "^5.58.2", 38 | "webpack-cli": "^4.9.0" 39 | }, 40 | "dependencies": { 41 | "css-loader": "^6.5.0", 42 | "d3": "^7.0.4", 43 | "d3-flextree": "^2.1.2", 44 | "d3-quadtree": "^3.0.1", 45 | "d3v4": "^4.2.2", 46 | "style-loader": "^3.3.1" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /hatchet/vis/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/requirements.txt -------------------------------------------------------------------------------- /hatchet/vis/scripts/boxplot.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/scripts/boxplot.js -------------------------------------------------------------------------------- /hatchet/vis/scripts/cct.js: -------------------------------------------------------------------------------- 1 | //d3.v4 2 | import { RT, d3 } from './cct/cct_globals'; 3 | import Controller from './cct/cct_controller'; 4 | import Model from './cct/cct_model'; 5 | import MenuView from './cct/cct_menu_view'; 6 | import ChartView from './cct/cct_chart_view'; 7 | import TooltipView from './cct/cct_tooltip_view'; 8 | import ScentedSliderPopup from './cct/cct_scented_slider_popup'; 9 | 10 | d3.select(element).attr('width', '100%'); 11 | 12 | 13 | 14 | RT['jsNodeSelected'] = JSON.stringify(["*"]); 15 | 16 | // --------------------------------------------- 17 | // Main driver area 18 | // --------------------------------------------- 19 | 20 | //model 21 | var model = new Model(); 22 | //controller 23 | var cont = new Controller(model); 24 | //views 25 | var menu = new MenuView(element, model); 26 | var tooltip = new TooltipView(element, model); 27 | var chart = new ChartView(element, model); 28 | var popup = new ScentedSliderPopup(element, model); 29 | 30 | //register signallers 31 | menu.register(cont.dispatcher()); 32 | chart.register(cont.dispatcher()); 33 | popup.register(cont.dispatcher()); 34 | 35 | model.register(menu.renderer()); 36 | model.register(tooltip.renderer()); 37 | model.register(popup.renderer()); 38 | model.register(chart.renderer()); 39 | 40 | //render all views one time 41 | menu.render(); 42 | tooltip.render(); 43 | chart.render(); 44 | -------------------------------------------------------------------------------- /hatchet/vis/scripts/cct/cct_controller.js: -------------------------------------------------------------------------------- 1 | import { globals} from "./cct_globals"; 2 | 3 | class Controller{ 4 | /** 5 | * Handles interaction and events by taking requests from the view 6 | * and firing off functions in the model. 7 | */ 8 | constructor(model){ 9 | this.model = model; 10 | } 11 | 12 | dispatcher(){ 13 | return (evt) => { 14 | 15 | // All types of events run through a central dispatch 16 | // function. The dispatch function decides what to do. 17 | switch(evt.type) { 18 | case(globals.signals.UPDATESELECTED): 19 | this.model.updateSelected(evt.nodes); 20 | break; 21 | case (globals.signals.CLICK): 22 | this.model.updateSelected(evt.node); 23 | break; 24 | case (globals.signals.COLLAPSESUBTREE): 25 | this.model.handleDoubleClick(evt.node); 26 | break; 27 | case(globals.signals.COMPOSEINTERNAL): 28 | this.model.handleNodeComposition(evt.node); 29 | break; 30 | case(globals.signals.DECOMPOSENODE): 31 | // this.model.handleNodeDecomposition(evt.node); 32 | console.log("Decompose coming soon.") 33 | break; 34 | case(globals.signals.TOGGLEBRUSH): 35 | this.model.toggleBrush(); 36 | break; 37 | case (globals.signals.BRUSH): 38 | this.model.setBrushedPoints(evt.selection); 39 | break; 40 | case (globals.signals.METRICCHANGE): 41 | this.model.changeMetric(evt.newMetric, evt.source); 42 | break; 43 | case(globals.signals.COLORCLICK): 44 | this.model.changeColorScheme(evt.value); 45 | break; 46 | case(globals.signals.TREECHANGE): 47 | this.model.updateActiveTrees(evt.display); 48 | break; 49 | case(globals.signals.LEGENDCLICK): 50 | this.model.updateLegends(evt.value); 51 | break; 52 | case(globals.signals.ENABLEMASSPRUNE): 53 | this.model.enablePruneTree(evt.threshold); 54 | break; 55 | case(globals.signals.REQUESTMASSPRUNE): 56 | this.model.pruneTree(evt.threshold); 57 | break; 58 | case(globals.signals.RESETVIEW): 59 | this.model.resetView(); 60 | break; 61 | case(globals.signals.PRUNERANGEUPDATE): 62 | this.model.updatePruneRange(evt.low, evt.high); 63 | break; 64 | case(globals.signals.SNAPSHOT): 65 | this.model.storeSnapshotQuery(); 66 | break; 67 | case(globals.signals.TOGGLEMENU): 68 | this.model.toggleMenuActive(); 69 | break; 70 | default: 71 | console.warn('Unknown event type', evt.type); 72 | } 73 | } 74 | } 75 | } 76 | 77 | export default Controller; -------------------------------------------------------------------------------- /hatchet/vis/scripts/cct/cct_globals.js: -------------------------------------------------------------------------------- 1 | import * as d3 from 'd3v4'; 2 | 3 | const globals = Object.freeze({ 4 | UNIFIED: 0, 5 | DEFAULT: 0, 6 | SUM: "sum", 7 | AVG: "avg", 8 | signals: { 9 | CLICK: "CLICK", 10 | COLLAPSESUBTREE: "COLLAPSESUBTREE", 11 | COMPOSEINTERNAL: "COMPOSEINTERNAL", 12 | BRUSH: "BRUSH", 13 | TOGGLEBRUSH: "TOGGLEBRUSH", 14 | COLLAPSE: "COLLAPSE", 15 | METRICCHANGE: "METRICCHANGE", 16 | TREECHANGE: "TREECHANGE", 17 | COLORCLICK: "COLORCLICK", 18 | LEGENDCLICK: "LEGENDCLICK", 19 | ENABLEMASSPRUNE: "ENABLEMASSPRUNE", 20 | REQUESTMASSPRUNE: "REQUESTMASSPRUNE", 21 | RESETVIEW: "RESET", 22 | PRUNERANGEUPDATE: "PRUNERANGEUPDATE", 23 | UPDATESELECTED: "UPDATESELECTED", 24 | SNAPSHOT: "SNAPSHOT", 25 | DECOMPOSENODE: "DECOMPOSENODE", 26 | TOGGLEMENU: "TOGGLEMENU" 27 | }, 28 | layout: { 29 | margin: {top: 20, right: 20, bottom: 20, left: 20}, 30 | }, 31 | duration: 750 32 | }); 33 | 34 | var makeSignaller = function() { 35 | let _subscribers = []; // Private member 36 | 37 | // Return the object that's created 38 | return { 39 | // Register a function with the notification system 40 | add: function(handlerFunction) {_subscribers.push(handlerFunction); }, 41 | 42 | // Loop through all registered function and call them with passed 43 | // arguments 44 | notify: function(args) { 45 | for (var i = 0; i < _subscribers.length; i++) { 46 | try{ 47 | _subscribers[i](args); 48 | } catch(error){ 49 | console.error(error); 50 | } 51 | } 52 | } 53 | }; 54 | } 55 | 56 | 57 | const digitAbbrevScale = d3.scaleOrdinal().range(["K", "K", "K", "M", "M", "M", "B", "B", "B", "T", "T", "T",]).domain(new Array(12).fill(0).map((_,i)=>i+4)); 58 | 59 | function getSigFigString(num){ 60 | /** 61 | * Converts an integer or float to a string 62 | * with fixed number of figures before and after 63 | * the decimal point and an magnitude abbreviation. 64 | */ 65 | if(num.toFixed(2).length <= 6){ 66 | return num.toFixed(2); 67 | } 68 | else{ 69 | let numdig = parseInt(num).toString().length; 70 | for(let i = 4; i <= numdig; i +=3){ 71 | num = (parseInt(num)/1000); 72 | } 73 | let numstr = num.toFixed(2).toString(); 74 | 75 | let abbrev = digitAbbrevScale(numdig); 76 | 77 | return numstr + abbrev; 78 | } 79 | } 80 | 81 | function areaToRad(area){ 82 | /** 83 | * Converts area to radius. Used for accuracy in scaling 84 | * nodes. 85 | */ 86 | return Math.sqrt(area/Math.PI); 87 | } 88 | 89 | 90 | var RT = window.Roundtrip; 91 | 92 | export { makeSignaller, globals, RT, d3, getSigFigString, areaToRad}; -------------------------------------------------------------------------------- /hatchet/vis/scripts/cct/cct_stats.js: -------------------------------------------------------------------------------- 1 | class Stats{ 2 | 3 | _asc(arr, metric){ 4 | /** 5 | * Sorts an array in ascending order. 6 | */ 7 | 8 | return arr.sort((a,b) => a[metric]-b[metric]) 9 | } 10 | 11 | _quantile(arr, q, metric){ 12 | /** 13 | * Gets a particular quantile from an array of numbers 14 | * 15 | * @param {Array} arr - An array of floats 16 | * @param {Number} q - An float between [0-1] represening the quantile we want 17 | */ 18 | const sorted = this._asc(arr, metric); 19 | const pos = (sorted.length - 1) * q; 20 | const base = Math.floor(pos); 21 | const rest = pos - base; 22 | if (sorted[base + 1] !== undefined) { 23 | return sorted[base][metric] + rest * (sorted[base + 1][metric] - sorted[base][metric]); 24 | } else { 25 | return sorted[base][metric]; 26 | } 27 | } 28 | 29 | _getIQR(arr, metric){ 30 | /** 31 | * Returns the interquartile range for a an array of numbers 32 | */ 33 | if(arr.length != 0){ 34 | var q25 = this._quantile(arr, .25, metric); 35 | var q75 = this._quantile(arr, .75, metric); 36 | var IQR = q75 - q25; 37 | 38 | return IQR; 39 | } 40 | 41 | return NaN; 42 | } 43 | } 44 | 45 | export default Stats; -------------------------------------------------------------------------------- /hatchet/vis/scripts/cct/cct_tooltip_view.js: -------------------------------------------------------------------------------- 1 | import { d3 } from "./cct_globals"; 2 | import View from "../utils/view"; 3 | 4 | class TooltipView extends View { 5 | /** 6 | * Class that instantiates the view for the tooltip that appears with selected nodes. 7 | * 8 | * @param {DOM Element} elem - The current cell of the calling Jupyter notebook 9 | * @param {Model} model - The model object containg data for the view 10 | */ 11 | constructor(elem, model){ 12 | super(elem, model); 13 | 14 | this.tooltip = d3.select(elem).append("div") 15 | .attr('id', 'tooltip') 16 | .style('position', 'absolute') 17 | .style('top', '50px') 18 | .style('right', '15px') 19 | .style('padding', '5px') 20 | .style('border-radius', '5px') 21 | .style('background', '#ccc') 22 | .style('color', 'black') 23 | .style('font-size', '14px') 24 | .style('font-family', 'monospace') 25 | .style('max-width','800px') 26 | .style('max-height', '200px') 27 | .style('overflow', 'scroll') 28 | .html('

Click a node or "Select nodes" to see more info

'); 29 | 30 | d3.select('#site') 31 | .on(`scroll.${d3.select(elem).select('script').attr('id')}`, (e)=>{ 32 | // console.log(d3.select(elem).select('script').attr('id')); 33 | if(this.elem.getBoundingClientRect().top < 150 && !( -this.elem.getBoundingClientRect().top - this.elem.getBoundingClientRect().height > -250) ){ 34 | this.tooltip 35 | .style('position', 'fixed') 36 | .style('top','150px'); 37 | } 38 | else{ 39 | this.tooltip 40 | .style('position', 'absolute') 41 | .style('top', '50px') 42 | } 43 | }) 44 | 45 | } 46 | 47 | render(){ 48 | this.tooltip.html(this.model.data["tipText"]); 49 | } 50 | 51 | } 52 | 53 | export default TooltipView; -------------------------------------------------------------------------------- /hatchet/vis/scripts/utils/view.js: -------------------------------------------------------------------------------- 1 | import { makeSignaller } from "../cct/cct_globals"; 2 | 3 | class View{ 4 | constructor(elem, model){ 5 | this.elem = elem; 6 | this.model = model; 7 | this.observers = makeSignaller(); 8 | } 9 | 10 | register(s){ 11 | this.observers.add(s, this.constructor.name); 12 | } 13 | 14 | renderer(){ 15 | /** 16 | * Returns a function which can be used as a 17 | * callback and have 'this' available 18 | */ 19 | return () => { 20 | this.render(); 21 | } 22 | } 23 | } 24 | 25 | export default View; -------------------------------------------------------------------------------- /hatchet/vis/static/boxplot_bundle.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /hatchet/vis/static/cct_bundle.html: -------------------------------------------------------------------------------- 1 | Webpack App -------------------------------------------------------------------------------- /hatchet/vis/static_fixer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from bs4 import BeautifulSoup 7 | from os import walk, path 8 | 9 | static_filepath = path.abspath("static/") 10 | 11 | for pt, dirs, files in walk(static_filepath): 12 | for file in files: 13 | if ".html" in file: 14 | with open(path.join(static_filepath, file), "r") as f: 15 | html = f.read() 16 | soup = BeautifulSoup(html) 17 | soup.script["src"] = path.join(static_filepath, file[0:-5] + ".js") 18 | with open(path.join(static_filepath, file), "w") as writer: 19 | writer.write(soup.prettify()) 20 | -------------------------------------------------------------------------------- /hatchet/vis/styles/boxplot.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/styles/boxplot.css -------------------------------------------------------------------------------- /hatchet/vis/styles/cct.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/styles/cct.css -------------------------------------------------------------------------------- /hatchet/vis/templates/boxplot.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/templates/boxplot.html -------------------------------------------------------------------------------- /hatchet/vis/templates/cct.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/hatchet/vis/templates/cct.html -------------------------------------------------------------------------------- /hatchet/vis/webpack.config.js: -------------------------------------------------------------------------------- 1 | const HtmlWebpackPlugin = require('html-webpack-plugin'); 2 | const path = require('path'); 3 | 4 | module.exports = { 5 | module:{ 6 | rules:[ 7 | { 8 | test: /\.css$/i, 9 | use: ["style-loader", "css-loader"] 10 | }, 11 | { 12 | test: /\.(js|jsx)$/, 13 | exclude: /node_modules/, 14 | loader: 'babel-loader', 15 | options:{ 16 | cwd: path.resolve(__dirname), 17 | presets:["@babel/preset-env"] 18 | } 19 | } 20 | ] 21 | }, 22 | entry: { 23 | cct: [path.resolve(__dirname,'scripts/cct.js')], 24 | }, 25 | output: { 26 | publicPath: path.resolve(__dirname, 'static'), 27 | filename: '[name]_bundle.js', 28 | path: path.resolve(__dirname, 'static') 29 | }, 30 | optimization: { 31 | minimize: false 32 | }, 33 | plugins:[ 34 | new HtmlWebpackPlugin({ 35 | chunks: ['cct'], 36 | filename: 'cct_bundle.html' 37 | }) 38 | ], 39 | mode: 'production' 40 | } -------------------------------------------------------------------------------- /hatchet/writers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | -------------------------------------------------------------------------------- /hatchet/writers/dataframe_writer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from hatchet.node import Node 7 | 8 | from abc import abstractmethod 9 | 10 | # TODO The ABC class was introduced in Python 3.4. 11 | # When support for earlier versions is (eventually) dropped, 12 | # this entire "try-except" block can be reduced to: 13 | # from abc import ABC 14 | try: 15 | from abc import ABC 16 | except ImportError: 17 | from abc import ABCMeta 18 | 19 | ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) 20 | 21 | 22 | def _get_node_from_df_iloc(df, ind): 23 | node = None 24 | if isinstance(df.iloc[ind].name, tuple): 25 | node = df.iloc[ind].name[0] 26 | elif isinstance(df.iloc[ind].name, Node): 27 | node = df.iloc[ind].name 28 | else: 29 | raise InvalidDataFrameIndex( 30 | "DataFrame index elements must be either a tuple or a Node" 31 | ) 32 | return node 33 | 34 | 35 | def _fill_children_and_parents(dump_df): 36 | dump_df["children"] = [[] for _ in range(len(dump_df))] 37 | dump_df["parents"] = [[] for _ in range(len(dump_df))] 38 | for i in range(len(dump_df)): 39 | node = _get_node_from_df_iloc(dump_df, i) 40 | dump_df.iat[i, dump_df.columns.get_loc("children")] = [ 41 | c._hatchet_nid for c in node.children 42 | ] 43 | node.children = [] 44 | dump_df.iat[i, dump_df.columns.get_loc("parents")] = [ 45 | p._hatchet_nid for p in node.parents 46 | ] 47 | node.parents = [] 48 | return dump_df 49 | 50 | 51 | class DataframeWriter(ABC): 52 | def __init__(self, filename): 53 | self.filename = filename 54 | 55 | @abstractmethod 56 | def _write_dataframe_to_file(self, df, **kwargs): 57 | pass 58 | 59 | def write(self, gf, **kwargs): 60 | gf_cpy = gf.deepcopy() 61 | dump_df = _fill_children_and_parents(gf_cpy.dataframe) 62 | dump_df["exc_metrics"] = None 63 | dump_df.iat[0, dump_df.columns.get_loc("exc_metrics")] = gf_cpy.exc_metrics 64 | dump_df["inc_metrics"] = None 65 | dump_df.iat[0, dump_df.columns.get_loc("inc_metrics")] = gf_cpy.inc_metrics 66 | dump_df["default_metric"] = None 67 | dump_df.iat[0, dump_df.columns.get_loc("default_metric")] = ( 68 | gf_cpy.default_metric 69 | ) 70 | self._write_dataframe_to_file(dump_df, **kwargs) 71 | 72 | 73 | class InvalidDataFrameIndex(Exception): 74 | """Raised when the DataFrame index is of an invalid type.""" 75 | -------------------------------------------------------------------------------- /hatchet/writers/hdf5_writer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | import warnings 7 | import sys 8 | 9 | from .dataframe_writer import DataframeWriter 10 | 11 | 12 | class HDF5Writer(DataframeWriter): 13 | def __init__(self, filename): 14 | if sys.version_info[0] == 2: 15 | super(HDF5Writer, self).__init__(filename) 16 | else: 17 | super().__init__(filename) 18 | 19 | def _write_dataframe_to_file(self, df, **kwargs): 20 | if "key" not in kwargs: 21 | raise KeyError("Writing to HDF5 requires a user-supplied key") 22 | key = kwargs["key"] 23 | del kwargs["key"] 24 | with warnings.catch_warnings(): 25 | warnings.filterwarnings("ignore", category=Warning) 26 | df.to_hdf(self.filename, key, **kwargs) 27 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | case *"$PWD"* in 4 | "$PYTHONPATH") 5 | ;; 6 | 7 | *) 8 | PYTHONPATH=$PWD:$PYTHONPATH 9 | ;; 10 | esac 11 | python setup.py clean --all 12 | python setup.py build_ext --inplace 13 | python hatchet/vis/static_fixer.py 14 | -------------------------------------------------------------------------------- /logo-hex.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/logo-hex.png -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/logo.png -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel", "Cython"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.poetry] 6 | name = "llnl-hatchet" 7 | version = "2024.1.3" 8 | description = "A Python library for analyzing hierarchical performance data." 9 | authors = [ 10 | "Abhinav Bhatele ", 11 | "Stephanie Brink ", 12 | "Todd Gamblin ", 13 | ] 14 | license = "MIT" 15 | 16 | [tool.ruff] 17 | line-length = 88 18 | target-version = 'py39' 19 | include = ['\.pyi?$'] 20 | exclude = [ 21 | ".eggs", 22 | ".git", 23 | ".hg", 24 | ".mypy_cache", 25 | ".tox", 26 | ".venv", 27 | "_build", 28 | "buck-out", 29 | "build", 30 | "dist", 31 | ] 32 | 33 | [tool.black] 34 | line-length = 88 35 | target-version = ['py35', 'py36', 'py37', 'py38', 'py39', 'py310', 'py311'] 36 | include = '\.pyi?$' 37 | exclude = ''' 38 | /( 39 | \.eggs 40 | | \.git 41 | | \.hg 42 | | \.mypy_cache 43 | | \.tox 44 | | \.venv 45 | | _build 46 | | buck-out 47 | | build 48 | | dist 49 | )/ 50 | ''' 51 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | [pytest] 7 | addopts = --durations=20 -ra 8 | testpaths = hatchet/tests 9 | python_files = *.py 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib 2 | pandas 3 | pydot 4 | pytest 5 | codecov 6 | pep8-naming 7 | numpy < 2.0.0 8 | PyYAML 9 | cython 10 | multiprocess 11 | textX < 3.0.0; python_version < "3.6" 12 | textX >= 3.0.0; python_version >= "3.6" 13 | caliper-reader 14 | beautifulsoup4 15 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/hatchet/0cf106133b39e0ba0affe6a00cefc0345ba189c1/screenshot.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017-2023 Lawrence Livermore National Security, LLC and other 2 | # Hatchet Project Developers. See the top-level LICENSE file for details. 3 | # 4 | # SPDX-License-Identifier: MIT 5 | 6 | from setuptools import setup 7 | from setuptools import Extension 8 | from codecs import open 9 | from os import path 10 | 11 | here = path.abspath(path.dirname(__file__)) 12 | 13 | # Get the long description from the README file 14 | with open(path.join(here, "README.md"), encoding="utf-8") as f: 15 | long_description = f.read() 16 | 17 | # Get the version in a safe way which does not refrence hatchet `__init__` file 18 | # per python docs: https://packaging.python.org/guides/single-sourcing-package-version/ 19 | version = {} 20 | with open("./hatchet/version.py") as fp: 21 | exec(fp.read(), version) 22 | 23 | 24 | cmd_class = {} 25 | ext_modules = [] 26 | should_cythonize = False 27 | mod_import_path = "hatchet.cython_modules.libs" 28 | mod_file_path = "hatchet/cython_modules" 29 | mod_names = [ 30 | "reader_modules", 31 | "graphframe_modules", 32 | ] 33 | 34 | 35 | for mname in mod_names: 36 | c_file = path.join(mod_file_path, "{}.c".format(mname)) 37 | pyx_file = path.join(mod_file_path, "{}.pyx".format(mname)) 38 | if not path.isfile(pyx_file): 39 | raise FileNotFoundError( 40 | "Requested Cython extension not found: {}".format(pyx_file) 41 | ) 42 | if path.isfile(c_file): 43 | should_cythonize = False 44 | ext_modules.append( 45 | Extension( 46 | "{}.{}".format(mod_import_path, mname), 47 | [c_file], 48 | ) 49 | ) 50 | else: 51 | should_cythonize = True 52 | ext_modules.append( 53 | Extension( 54 | "{}.{}".format(mod_import_path, mname), 55 | [pyx_file], 56 | ) 57 | ) 58 | 59 | if should_cythonize: 60 | from Cython.Build import cythonize, build_ext 61 | 62 | ext_modules = cythonize(ext_modules) 63 | cmd_class.update({"build_ext": build_ext}) 64 | 65 | 66 | ext_modules = [ 67 | Extension( 68 | "hatchet.cython_modules.libs.reader_modules", 69 | ["hatchet/cython_modules/reader_modules.pyx"], 70 | ), 71 | Extension( 72 | "hatchet.cython_modules.libs.graphframe_modules", 73 | ["hatchet/cython_modules/graphframe_modules.pyx"], 74 | ), 75 | ] 76 | 77 | 78 | setup( 79 | name="llnl-hatchet", 80 | version=version["__version__"], 81 | license="MIT", 82 | description="A Python library for analyzing hierarchical performance data", 83 | long_description=long_description, 84 | long_description_content_type="text/markdown", 85 | url="https://github.com/llnl/hatchet", 86 | project_urls={ 87 | "Source Code": "https://github.com/llnl/hatchet", 88 | "Documentation": "https://llnl-hatchet.readthedocs.io/en/latest/", 89 | }, 90 | author="Stephanie Brink", 91 | author_email="brink2@llnl.gov", 92 | classifiers=[ 93 | "Development Status :: 5 - Production/Stable", 94 | "License :: OSI Approved :: MIT License", 95 | ], 96 | keywords="", 97 | python_requires=">=3.5", 98 | packages=[ 99 | "hatchet", 100 | "hatchet.readers", 101 | "hatchet.writers", 102 | "hatchet.query", 103 | "hatchet.vis", 104 | "hatchet.util", 105 | "hatchet.external", 106 | "hatchet.tests", 107 | "hatchet.cython_modules.libs", 108 | ], 109 | include_package_data=True, 110 | install_requires=[ 111 | "pydot", 112 | "PyYAML", 113 | "matplotlib", 114 | "numpy < 2.0.0", 115 | "pandas", 116 | "textX < 3.0.0; python_version < '3.6'", 117 | "textX >= 3.0.0; python_version >= '3.6'", 118 | "multiprocess", 119 | "caliper-reader", 120 | ], 121 | ext_modules=ext_modules, 122 | cmdclass=cmd_class, 123 | ) 124 | --------------------------------------------------------------------------------