├── .editorconfig ├── .github └── dependabot.yml ├── .gitignore ├── .prettierignore ├── .prettierrc ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── app.yml ├── bench.js ├── collector.js ├── criterion-files ├── .DS_Store ├── import block │ ├── .DS_Store │ ├── Native │ │ ├── base │ │ │ ├── benchmark.json │ │ │ ├── estimates.json │ │ │ ├── raw.csv │ │ │ ├── sample.json │ │ │ └── tukey.json │ │ ├── change │ │ │ └── estimates.json │ │ ├── new │ │ │ ├── benchmark.json │ │ │ ├── estimates.json │ │ │ ├── raw.csv │ │ │ ├── sample.json │ │ │ └── tukey.json │ │ └── report │ │ │ ├── MAD.svg │ │ │ ├── SD.svg │ │ │ ├── both │ │ │ ├── pdf.svg │ │ │ └── regression.svg │ │ │ ├── change │ │ │ ├── mean.svg │ │ │ ├── median.svg │ │ │ └── t-test.svg │ │ │ ├── index.html │ │ │ ├── mean.svg │ │ │ ├── median.svg │ │ │ ├── pdf.svg │ │ │ ├── pdf_small.svg │ │ │ ├── regression.svg │ │ │ ├── regression_small.svg │ │ │ ├── relative_pdf_small.svg │ │ │ ├── relative_regression_small.svg │ │ │ └── slope.svg │ ├── Wasm │ │ ├── .DS_Store │ │ ├── base │ │ │ ├── benchmark.json │ │ │ ├── estimates.json │ │ │ ├── raw.csv │ │ │ ├── sample.json │ │ │ └── tukey.json │ │ ├── change │ │ │ └── estimates.json │ │ ├── new │ │ │ ├── benchmark.json │ │ │ ├── estimates.json │ │ │ ├── raw.csv │ │ │ ├── sample.json │ │ │ └── tukey.json │ │ └── report │ │ │ ├── MAD.svg │ │ │ ├── SD.svg │ │ │ ├── both │ │ │ ├── pdf.svg │ │ │ └── regression.svg │ │ │ ├── change │ │ │ ├── mean.svg │ │ │ ├── median.svg │ │ │ └── t-test.svg │ │ │ ├── index.html │ │ │ ├── mean.svg │ │ │ ├── median.svg │ │ │ ├── pdf.svg │ │ │ ├── pdf_small.svg │ │ │ ├── regression.svg │ │ │ ├── regression_small.svg │ │ │ ├── relative_pdf_small.svg │ │ │ ├── relative_regression_small.svg │ │ │ └── slope.svg │ ├── base │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ ├── change │ │ └── estimates.json │ ├── new │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ └── report │ │ ├── MAD.svg │ │ ├── SD.svg │ │ ├── both │ │ ├── pdf.svg │ │ └── regression.svg │ │ ├── change │ │ ├── mean.svg │ │ ├── median.svg │ │ └── t-test.svg │ │ ├── index.html │ │ ├── mean.svg │ │ ├── median.svg │ │ ├── pdf.svg │ │ ├── pdf_small.svg │ │ ├── regression.svg │ │ ├── regression_small.svg │ │ ├── relative_pdf_small.svg │ │ ├── relative_regression_small.svg │ │ ├── slope.svg │ │ └── violin.svg └── import-block-with-ed25519-B-0003 │ ├── Native │ ├── .DS_Store │ ├── base │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ ├── new │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ └── report │ │ ├── MAD.svg │ │ ├── SD.svg │ │ ├── index.html │ │ ├── mean.svg │ │ ├── median.svg │ │ ├── pdf.svg │ │ ├── pdf_small.svg │ │ ├── regression.svg │ │ ├── regression_small.svg │ │ └── slope.svg │ ├── Wasm │ ├── base │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ ├── new │ │ ├── benchmark.json │ │ ├── estimates.json │ │ ├── raw.csv │ │ ├── sample.json │ │ └── tukey.json │ └── report │ │ ├── MAD.svg │ │ ├── SD.svg │ │ ├── index.html │ │ ├── mean.svg │ │ ├── median.svg │ │ ├── pdf.svg │ │ ├── pdf_small.svg │ │ ├── regression.svg │ │ ├── regression_small.svg │ │ └── slope.svg │ └── report │ ├── index.html │ └── violin.svg ├── index.js ├── package.json ├── run ├── test-run.js ├── test.js └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = lf 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for npm 4 | - package-ecosystem: "npm" 5 | directory: "/" 6 | schedule: 7 | interval: "daily" 8 | ignore: 9 | - dependency-name: "*" 10 | # Ignore minor/patch upgrades; only bother with opening the upgrade PR 11 | # when a new major release comes out; security updates are nevertheless 12 | # unaffected by this setting and will continue to work. 13 | update-types: 14 | ["version-update:semver-patch", "version-update:semver-minor"] 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.pem 3 | .env 4 | git 5 | log.txt 6 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | git 3 | criterion-files 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "trailingComma": "all", 4 | "arrowParens": "always" 5 | } 6 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | education, socio-economic status, nationality, personal appearance, race, 10 | religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at nikvolf@gmail.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | [fork]: /fork 4 | [pr]: /compare 5 | [style]: https://standardjs.com/ 6 | [code-of-conduct]: CODE_OF_CONDUCT.md 7 | 8 | Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. 9 | 10 | Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms. 11 | 12 | ## Issues and PRs 13 | 14 | If you have suggestions for how this project could be improved, or want to report a bug, open an issue! We'd love all and any contributions. If you have questions, too, we'd love to hear them. 15 | 16 | We'd also love PRs. If you're thinking of a large PR, we advise opening up an issue first to talk about it, though! Look at the links below if you're not sure how to open a PR. 17 | 18 | ## Submitting a pull request 19 | 20 | 1. [Fork][fork] and clone the repository. 21 | 1. Configure and install the dependencies: `npm install`. 22 | 1. Make sure the tests pass on your machine: `npm test`, note: these tests also apply the linter, so there's no need to lint separately. 23 | 1. Create a new branch: `git checkout -b my-branch-name`. 24 | 1. Make your change, add tests, and make sure the tests still pass. 25 | 1. Push to your fork and [submit a pull request][pr]. 26 | 1. Pat your self on the back and wait for your pull request to be reviewed and merged. 27 | 28 | Here are a few things you can do that will increase the likelihood of your pull request being accepted: 29 | 30 | - Follow the [style guide][style] which is using standard. Any linting errors should be shown when running `npm test`. 31 | - Write and update tests. 32 | - Keep your changes as focused as possible. If there are multiple changes you would like to make that are not dependent upon each other, consider submitting them as separate pull requests. 33 | - Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). 34 | 35 | Work in Progress pull requests are also welcome to get feedback early on, or if there is something blocked you. 36 | 37 | ## Resources 38 | 39 | - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) 40 | - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) 41 | - [GitHub Help](https://help.github.com) 42 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14 2 | 3 | COPY . /app 4 | WORKDIR /app 5 | 6 | RUN curl https://getsubstrate.io -sSf | /bin/bash -s -- --fast 7 | ENV PATH="/root/.cargo/bin:${PATH}" 8 | 9 | RUN git config --global user.name "Parity Bot" 10 | RUN git config --global user.email admin@parity.io 11 | 12 | RUN yarn 13 | CMD yarn start 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2020, NikVolf 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bench-bot 2 | 3 | :warning: bench-bot has been deprecated in favor of https://github.com/paritytech/command-bot, which has a different execution model. A detailed comparison is provided in https://github.com/paritytech/pipeline-scripts/issues/54. 4 | 5 | Although development will stop in this repository, the open-source community might continue in forks such as https://github.com/AcalaNetwork/bench-bot and https://github.com/PureStake/bench-bot. 6 | 7 | # How to use 8 | 9 | The bot runs commands in response to pull request comments 10 | ([example](https://github.com/paritytech/polkadot/pull/2541)). The form is: 11 | 12 | `/bench [action] [...args]` 13 | 14 | [Environment variables](#configuration) and 15 | [Github settings](#required-github-settings) have to properly configured 16 | upfront for this interaction to work. 17 | 18 | # Configuration 19 | 20 | Create a `.env` file in the root with the following: 21 | 22 | ``` 23 | APP_ID= 24 | BASE_BRANCH= 25 | CLIENT_ID= 26 | CLIENT_SECRET= 27 | PRIVATE_KEY_PATH= 28 | WEBHOOK_SECRET= 29 | WEBHOOK_PROXY_URL= 30 | ``` 31 | 32 | During development it's recommended to use [smee](https://smee.io) for 33 | `WEBHOOK_PROXY_URL` because it enables testing your bot's functionality 34 | locally, without having to SSH into the dedicated machine. 35 | 36 | # Linting and formatting 37 | 38 | The commands `yarn run format` and `yarn run lint` are available for ensuring 39 | style consistency on this project's code. 40 | 41 | # Running 42 | 43 | ## Locally 44 | 45 | `yarn && yarn start` 46 | 47 | ## Dedicated machine 48 | 49 | _Note: Before disrupting the production deployment, it's first recommended to 50 | check if some benchmark is running through_ `pgrep -a cargo` _._ 51 | 52 | The [run script](./run) is used to manage the application. Use `run help` for 53 | documentation about its options. 54 | 55 | `run bootstrap` will take care of creating and installing everything from 56 | scratch. After install, you'll also need to set up 57 | [environment variables](#configuration) which optionally can be done through 58 | a `.env` file in the bot's installation path. 59 | 60 | ### Bot commands 61 | 62 | - `run {start,stop,restart}`: execute the relevant action for the bot. 63 | - `run update [ref]`: restart the bot with the branch or PR 64 | - For branch: `ssh user@remote '/home/benchbot/bench-bot/run update master'` 65 | - For PR: `ssh user@remote '/home/benchbot/bench-bot/run update pull/number/head:branch'` 66 | e.g. `pull/1/head:master` 67 | 68 | ### Logs 69 | 70 | See 71 | 72 | # Required Github settings 73 | 74 | ## Permissions 75 | 76 | * Metadata: Read Only 77 | * Issues: Read/Write 78 | * Pull Requests: Read/Write 79 | * Contents: Read/Write 80 | 81 | ## Event subscriptions 82 | 83 | * Issue comments 84 | -------------------------------------------------------------------------------- /app.yml: -------------------------------------------------------------------------------- 1 | # This is a GitHub App Manifest. These settings will be used by default when 2 | # initially configuring your GitHub App. 3 | # 4 | # NOTE: changing this file will not update your GitHub App settings. 5 | # You must visit github.com/settings/apps/your-app-name to edit them. 6 | # 7 | # Read more about configuring your GitHub App: 8 | # https://probot.github.io/docs/development/#configuring-a-github-app 9 | # 10 | # Read more about GitHub App Manifests: 11 | # https://developer.github.com/apps/building-github-apps/creating-github-apps-from-a-manifest/ 12 | 13 | # The list of events the GitHub App subscribes to. 14 | # Uncomment the event names below to enable them. 15 | default_events: 16 | # - check_run 17 | # - check_suite 18 | # - commit_comment 19 | # - create 20 | # - delete 21 | # - deployment 22 | # - deployment_status 23 | # - fork 24 | # - gollum 25 | - issue_comment 26 | - issues 27 | # - label 28 | # - milestone 29 | # - member 30 | # - membership 31 | # - org_block 32 | # - organization 33 | # - page_build 34 | # - project 35 | # - project_card 36 | # - project_column 37 | # - public 38 | # - pull_request 39 | # - pull_request_review 40 | # - pull_request_review_comment 41 | # - push 42 | # - release 43 | # - repository 44 | # - repository_import 45 | # - status 46 | # - team 47 | # - team_add 48 | # - watch 49 | 50 | # The set of permissions needed by the GitHub App. The format of the object uses 51 | # the permission name for the key (for example, issues) and the access type for 52 | # the value (for example, write). 53 | # Valid values are `read`, `write`, and `none` 54 | default_permissions: 55 | # Repository creation, deletion, settings, teams, and collaborators. 56 | # https://developer.github.com/v3/apps/permissions/#permission-on-administration 57 | # administration: read 58 | 59 | # Checks on code. 60 | # https://developer.github.com/v3/apps/permissions/#permission-on-checks 61 | # checks: read 62 | 63 | # Repository contents, commits, branches, downloads, releases, and merges. 64 | # https://developer.github.com/v3/apps/permissions/#permission-on-contents 65 | # contents: read 66 | 67 | # Deployments and deployment statuses. 68 | # https://developer.github.com/v3/apps/permissions/#permission-on-deployments 69 | # deployments: read 70 | 71 | # Issues and related comments, assignees, labels, and milestones. 72 | # https://developer.github.com/v3/apps/permissions/#permission-on-issues 73 | issues: write 74 | 75 | # Search repositories, list collaborators, and access repository metadata. 76 | # https://developer.github.com/v3/apps/permissions/#metadata-permissions 77 | metadata: read 78 | 79 | # Retrieve Pages statuses, configuration, and builds, as well as create new builds. 80 | # https://developer.github.com/v3/apps/permissions/#permission-on-pages 81 | # pages: read 82 | 83 | # Pull requests and related comments, assignees, labels, milestones, and merges. 84 | # https://developer.github.com/v3/apps/permissions/#permission-on-pull-requests 85 | # pull_requests: read 86 | 87 | # Manage the post-receive hooks for a repository. 88 | # https://developer.github.com/v3/apps/permissions/#permission-on-repository-hooks 89 | # repository_hooks: read 90 | 91 | # Manage repository projects, columns, and cards. 92 | # https://developer.github.com/v3/apps/permissions/#permission-on-repository-projects 93 | # repository_projects: read 94 | 95 | # Retrieve security vulnerability alerts. 96 | # https://developer.github.com/v4/object/repositoryvulnerabilityalert/ 97 | # vulnerability_alerts: read 98 | 99 | # Commit statuses. 100 | # https://developer.github.com/v3/apps/permissions/#permission-on-statuses 101 | # statuses: read 102 | 103 | # Organization members and teams. 104 | # https://developer.github.com/v3/apps/permissions/#permission-on-members 105 | # members: read 106 | 107 | # View and manage users blocked by the organization. 108 | # https://developer.github.com/v3/apps/permissions/#permission-on-organization-user-blocking 109 | # organization_user_blocking: read 110 | 111 | # Manage organization projects, columns, and cards. 112 | # https://developer.github.com/v3/apps/permissions/#permission-on-organization-projects 113 | # organization_projects: read 114 | 115 | # Manage team discussions and related comments. 116 | # https://developer.github.com/v3/apps/permissions/#permission-on-team-discussions 117 | # team_discussions: read 118 | 119 | # Manage the post-receive hooks for an organization. 120 | # https://developer.github.com/v3/apps/permissions/#permission-on-organization-hooks 121 | # organization_hooks: read 122 | 123 | # Get notified of, and update, content references. 124 | # https://developer.github.com/v3/apps/permissions/ 125 | # organization_administration: read 126 | # The name of the GitHub App. Defaults to the name specified in package.json 127 | # name: My Probot App 128 | 129 | # The homepage of your GitHub App. 130 | # url: https://example.com/ 131 | 132 | # A description of the GitHub App. 133 | # description: A description of my awesome app 134 | 135 | # Set to true when your GitHub App is available to the public or false when it is only accessible to the owner of the app. 136 | # Default: true 137 | # public: false 138 | -------------------------------------------------------------------------------- /collector.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | 3 | var Collector = function () { 4 | var self = this 5 | self.metrics = {} 6 | self.baseNative = 0.0 7 | self.baseWasm = 0.0 8 | 9 | self.FormatMs = function (fl) { 10 | return (fl / 1000000.0).toFixed(2) 11 | } 12 | 13 | self.FormatPc = function (fl) { 14 | return (fl > 0 ? "+" : "") + (fl * 100).toFixed(2) + "%" 15 | } 16 | 17 | self.CollectBaseCriterionWasmNative = async function (baseDir) { 18 | var masterNativeStats = await self.loadStats( 19 | baseDir + "/Native/base/estimates.json", 20 | ) 21 | var masterWasmStats = await self.loadStats( 22 | baseDir + "/Wasm/base/estimates.json", 23 | ) 24 | 25 | self.baseWasm = masterWasmStats.Median.point_estimate 26 | self.baseNative = masterNativeStats.Median.point_estimate 27 | 28 | self.metrics["Master: Wasm"] = `${self.FormatMs( 29 | self.baseWasm, 30 | )} ms (+/- ${self.FormatMs(masterWasmStats.Median.standard_error)} ms)` 31 | self.metrics["Master: Native"] = `${self.FormatMs( 32 | self.baseNative, 33 | )} ms (+/- ${self.FormatMs(masterNativeStats.Median.standard_error)} ms)` 34 | } 35 | 36 | self.CollectBranchCriterionWasmNative = async function (baseDir) { 37 | var branchNativeStats = await self.loadStats( 38 | baseDir + "/Native/new/estimates.json", 39 | ) 40 | var branchWasmStats = await self.loadStats( 41 | baseDir + "/Wasm/new/estimates.json", 42 | ) 43 | 44 | var branchWasm = branchWasmStats.Median.point_estimate 45 | var branchNative = branchNativeStats.Median.point_estimate 46 | 47 | self.metrics["Branch: Wasm"] = `${self.FormatMs( 48 | branchWasm, 49 | )} ms (+/- ${self.FormatMs(branchWasmStats.Median.standard_error)} ms)` 50 | self.metrics["Branch: Native"] = `${self.FormatMs( 51 | branchNative, 52 | )} ms (+/- ${self.FormatMs(branchNativeStats.Median.standard_error)} ms)` 53 | 54 | var changeWasm = (branchWasm - self.baseWasm) / self.baseWasm 55 | var changeNative = (branchNative - self.baseNative) / self.baseNative 56 | 57 | self.metrics["Change: Wasm"] = self.FormatPc(changeWasm) 58 | self.metrics["Change: Native"] = self.FormatPc(changeNative) 59 | } 60 | 61 | self.CollectBaseCustomRunner = async function (stdout) { 62 | self.baseNative = JSON.parse(stdout)[0]["average"] 63 | self.metrics["Master"] = `${self.FormatMs(self.baseNative)} ms` 64 | } 65 | 66 | self.CollectBranchCustomRunner = async function (stdout) { 67 | self.branchNative = JSON.parse(stdout)[0]["average"] 68 | var change = (self.branchNative - self.baseNative) / self.baseNative 69 | self.metrics["Branch"] = `${self.FormatMs(self.branchNative)} ms` 70 | self.metrics["Change"] = self.FormatPc(change) 71 | } 72 | 73 | self.loadStats = async function (path) { 74 | var buffer = fs.readFileSync(path) 75 | return JSON.parse(buffer) 76 | } 77 | 78 | self.Report = async function () { 79 | var report = "" 80 | for (metric in self.metrics) { 81 | report = report + "\n" + `${metric}: ${self.metrics[metric]}` 82 | } 83 | return report 84 | } 85 | } 86 | 87 | module.exports.Collector = Collector 88 | -------------------------------------------------------------------------------- /criterion-files/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paritytech/bench-bot/b5dbd8635cbe0038d0167fcdeea0b51f12ea5e92/criterion-files/.DS_Store -------------------------------------------------------------------------------- /criterion-files/import block/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paritytech/bench-bot/b5dbd8635cbe0038d0167fcdeea0b51f12ea5e92/criterion-files/import block/.DS_Store -------------------------------------------------------------------------------- /criterion-files/import block/Native/base/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":"Native","throughput":null,"full_id":"import block/Native","directory_name":"import block/Native","title":"import block/Native"} -------------------------------------------------------------------------------- /criterion-files/import block/Native/base/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26433197.003593426,"upper_bound":26573054.584651843},"point_estimate":26503027.4824144,"standard_error":35648.14188941792},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26377389.57843137,"upper_bound":26598406.223076925},"point_estimate":26526195.64375,"standard_error":56832.370398992185},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":76919.49599911271,"upper_bound":247826.9934132563},"point_estimate":194100.02326664852,"standard_error":46355.155306983674},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26433512.840581454,"upper_bound":26558878.051336408},"point_estimate":26495444.498954702,"standard_error":32109.366216624832},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":118002.93484315016,"upper_bound":198946.63329583596},"point_estimate":163873.30332484088,"standard_error":20787.28370009099}} -------------------------------------------------------------------------------- /criterion-files/import block/Native/base/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,Native,,,26317403.0,ns,1 3 | import block,,Native,,,53300752.0,ns,2 4 | import block,,Native,,,79123024.0,ns,3 5 | import block,,Native,,,107367307.0,ns,4 6 | import block,,Native,,,132537568.0,ns,5 7 | import block,,Native,,,159022929.0,ns,6 8 | import block,,Native,,,183784076.0,ns,7 9 | import block,,Native,,,213579057.0,ns,8 10 | import block,,Native,,,236030513.0,ns,9 11 | import block,,Native,,,263452148.0,ns,10 12 | import block,,Native,,,293302379.0,ns,11 13 | import block,,Native,,,319688715.0,ns,12 14 | import block,,Native,,,345728901.0,ns,13 15 | import block,,Native,,,371933323.0,ns,14 16 | import block,,Native,,,399034224.0,ns,15 17 | import block,,Native,,,424718043.0,ns,16 18 | import block,,Native,,,448467443.0,ns,17 19 | import block,,Native,,,473914014.0,ns,18 20 | import block,,Native,,,504821309.0,ns,19 21 | import block,,Native,,,529014366.0,ns,20 22 | -------------------------------------------------------------------------------- /criterion-files/import block/Native/base/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0],[26317403.0,53300752.0,79123024.0,107367307.0,132537568.0,159022929.0,183784076.0,213579057.0,236030513.0,263452148.0,293302379.0,319688715.0,345728901.0,371933323.0,399034224.0,424718043.0,448467443.0,473914014.0,504821309.0,529014366.0]] -------------------------------------------------------------------------------- /criterion-files/import block/Native/base/tukey.json: -------------------------------------------------------------------------------- 1 | [25632560.512499988,25999810.106249996,26979142.35625001,27346391.950000014] -------------------------------------------------------------------------------- /criterion-files/import block/Native/change/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.20671892448976975,"upper_bound":-0.1507135284745651},"point_estimate":-0.1776917957831714,"standard_error":0.014345995289728313},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.18115612275403026,"upper_bound":-0.1466782065813068},"point_estimate":-0.16348670921103647,"standard_error":0.009902736795326737}} -------------------------------------------------------------------------------- /criterion-files/import block/Native/new/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":"Native","throughput":null,"full_id":"import block/Native","directory_name":"import block/Native","title":"import block/Native"} -------------------------------------------------------------------------------- /criterion-files/import block/Native/new/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26433197.003593426,"upper_bound":26573054.584651843},"point_estimate":26503027.4824144,"standard_error":35648.14188941792},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26377389.57843137,"upper_bound":26598406.223076925},"point_estimate":26526195.64375,"standard_error":56832.370398992185},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":76919.49599911271,"upper_bound":247826.9934132563},"point_estimate":194100.02326664852,"standard_error":46355.155306983674},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":26433512.840581454,"upper_bound":26558878.051336408},"point_estimate":26495444.498954702,"standard_error":32109.366216624832},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":118002.93484315016,"upper_bound":198946.63329583596},"point_estimate":163873.30332484088,"standard_error":20787.28370009099}} -------------------------------------------------------------------------------- /criterion-files/import block/Native/new/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,Native,,,26317403.0,ns,1 3 | import block,,Native,,,53300752.0,ns,2 4 | import block,,Native,,,79123024.0,ns,3 5 | import block,,Native,,,107367307.0,ns,4 6 | import block,,Native,,,132537568.0,ns,5 7 | import block,,Native,,,159022929.0,ns,6 8 | import block,,Native,,,183784076.0,ns,7 9 | import block,,Native,,,213579057.0,ns,8 10 | import block,,Native,,,236030513.0,ns,9 11 | import block,,Native,,,263452148.0,ns,10 12 | import block,,Native,,,293302379.0,ns,11 13 | import block,,Native,,,319688715.0,ns,12 14 | import block,,Native,,,345728901.0,ns,13 15 | import block,,Native,,,371933323.0,ns,14 16 | import block,,Native,,,399034224.0,ns,15 17 | import block,,Native,,,424718043.0,ns,16 18 | import block,,Native,,,448467443.0,ns,17 19 | import block,,Native,,,473914014.0,ns,18 20 | import block,,Native,,,504821309.0,ns,19 21 | import block,,Native,,,529014366.0,ns,20 22 | -------------------------------------------------------------------------------- /criterion-files/import block/Native/new/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0],[26317403.0,53300752.0,79123024.0,107367307.0,132537568.0,159022929.0,183784076.0,213579057.0,236030513.0,263452148.0,293302379.0,319688715.0,345728901.0,371933323.0,399034224.0,424718043.0,448467443.0,473914014.0,504821309.0,529014366.0]] -------------------------------------------------------------------------------- /criterion-files/import block/Native/new/tukey.json: -------------------------------------------------------------------------------- 1 | [25632560.512499988,25999810.106249996,26979142.35625001,27346391.950000014] -------------------------------------------------------------------------------- /criterion-files/import block/Native/report/both/regression.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block/Native 4 | 5 | 6 | Total sample time (ms) 7 | 8 | 9 | Iterations 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 100.0 30 | 31 | 32 | 33 | 200.0 34 | 35 | 36 | 37 | 300.0 38 | 39 | 40 | 41 | 400.0 42 | 43 | 44 | 45 | 500.0 46 | 47 | 48 | 49 | 600.0 50 | 51 | 52 | 53 | 54 | 2 55 | 56 | 57 | 58 | 4 59 | 60 | 61 | 62 | 6 63 | 64 | 65 | 66 | 8 67 | 68 | 69 | 70 | 10 71 | 72 | 73 | 74 | 12 75 | 76 | 77 | 78 | 14 79 | 80 | 81 | 82 | 16 83 | 84 | 85 | 86 | 18 87 | 88 | 89 | 90 | 20 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | Base Sample 99 | 100 | 101 | New Sample 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /criterion-files/import block/Native/report/change/t-test.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block/Native: Welch t test 4 | 5 | 6 | Density 7 | 8 | 9 | t score 10 | 11 | 12 | 13 | 0.05 14 | 15 | 16 | 17 | 0.1 18 | 19 | 20 | 21 | 0.15 22 | 23 | 24 | 25 | 0.2 26 | 27 | 28 | 29 | 0.25 30 | 31 | 32 | 33 | 0.3 34 | 35 | 36 | 37 | 0.35 38 | 39 | 40 | 41 | 0.4 42 | 43 | 44 | 45 | 46 | -4.0 47 | 48 | 49 | 50 | -2.0 51 | 52 | 53 | 54 | 0.0 55 | 56 | 57 | 58 | 2.0 59 | 60 | 61 | 62 | 4.0 63 | 64 | 65 | 66 | 67 | 68 | t distribution 69 | 70 | 71 | t statistic 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /criterion-files/import block/Native/report/pdf_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Density (a.u.) 4 | 5 | 6 | Average Time (ms) 7 | 8 | 9 | 10 | 0.5 11 | 12 | 13 | 14 | 1 15 | 16 | 17 | 18 | 1.5 19 | 20 | 21 | 22 | 2 23 | 24 | 25 | 26 | 27 | 26 28 | 29 | 30 | 31 | 26.5 32 | 33 | 34 | 35 | 27 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /criterion-files/import block/Native/report/relative_regression_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Total sample time (ms) 4 | 5 | 6 | Iterations 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 100.0 27 | 28 | 29 | 30 | 200.0 31 | 32 | 33 | 34 | 300.0 35 | 36 | 37 | 38 | 400.0 39 | 40 | 41 | 42 | 500.0 43 | 44 | 45 | 46 | 600.0 47 | 48 | 49 | 50 | 51 | 2 52 | 53 | 54 | 55 | 4 56 | 57 | 58 | 59 | 6 60 | 61 | 62 | 63 | 8 64 | 65 | 66 | 67 | 10 68 | 69 | 70 | 71 | 12 72 | 73 | 74 | 75 | 14 76 | 77 | 78 | 79 | 16 80 | 81 | 82 | 83 | 18 84 | 85 | 86 | 87 | 20 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paritytech/bench-bot/b5dbd8635cbe0038d0167fcdeea0b51f12ea5e92/criterion-files/import block/Wasm/.DS_Store -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/base/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":"Wasm","throughput":null,"full_id":"import block/Wasm","directory_name":"import block/Wasm","title":"import block/Wasm"} -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/base/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":87896182.57519853,"upper_bound":89569577.35295603},"point_estimate":88780630.5827,"standard_error":426202.64730461966},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":88670435.78333333,"upper_bound":89962603.83333334},"point_estimate":89333585.30000001,"standard_error":386365.6048296586},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":617146.2784417995,"upper_bound":2553638.7191293403},"point_estimate":1381595.6142167754,"standard_error":547397.4323779449},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":87028193.27940299,"upper_bound":89447401.50386965},"point_estimate":88335137.14076655,"standard_error":620016.4696778981},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1175926.1471010144,"upper_bound":2457817.688689552},"point_estimate":1960933.0958133002,"standard_error":325181.06749720604}} -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/base/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,Wasm,,,91643037.0,ns,1 3 | import block,,Wasm,,,177686471.0,ns,2 4 | import block,,Wasm,,,269846396.0,ns,3 5 | import block,,Wasm,,,361857092.0,ns,4 6 | import block,,Wasm,,,446704337.0,ns,5 7 | import block,,Wasm,,,533563542.0,ns,6 8 | import block,,Wasm,,,622811274.0,ns,7 9 | import block,,Wasm,,,723976134.0,ns,8 10 | import block,,Wasm,,,807220789.0,ns,9 11 | import block,,Wasm,,,893263032.0,ns,10 12 | import block,,Wasm,,,989740499.0,ns,11 13 | import block,,Wasm,,,1038518514.0,ns,12 14 | import block,,Wasm,,,1123103257.0,ns,13 15 | import block,,Wasm,,,1180047005.0,ns,14 16 | import block,,Wasm,,,1327464541.0,ns,15 17 | import block,,Wasm,,,1358153010.0,ns,16 18 | import block,,Wasm,,,1480275925.0,ns,17 19 | import block,,Wasm,,,1627631170.0,ns,18 20 | import block,,Wasm,,,1700761177.0,ns,19 21 | import block,,Wasm,,,1807227661.0,ns,20 22 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/base/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0],[91643037.0,177686471.0,269846396.0,361857092.0,446704337.0,533563542.0,622811274.0,723976134.0,807220789.0,893263032.0,989740499.0,1038518514.0,1123103257.0,1180047005.0,1327464541.0,1358153010.0,1480275925.0,1627631170.0,1700761177.0,1807227661.0]] -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/base/tukey.json: -------------------------------------------------------------------------------- 1 | [82350005.07426466,85245997.8636029,92968645.30183825,95864638.09117651] -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/change/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.03389982181127734,"upper_bound":-0.012114125052028128},"point_estimate":-0.022653130215785078,"standard_error":0.005563709796007804},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.024764075311808464,"upper_bound":-0.006742103622063511},"point_estimate":-0.016939230078931722,"standard_error":0.004896084505610261}} -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/new/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":"Wasm","throughput":null,"full_id":"import block/Wasm","directory_name":"import block/Wasm","title":"import block/Wasm"} -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/new/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":87896182.57519853,"upper_bound":89569577.35295603},"point_estimate":88780630.5827,"standard_error":426202.64730461966},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":88670435.78333333,"upper_bound":89962603.83333334},"point_estimate":89333585.30000001,"standard_error":386365.6048296586},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":617146.2784417995,"upper_bound":2553638.7191293403},"point_estimate":1381595.6142167754,"standard_error":547397.4323779449},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":87028193.27940299,"upper_bound":89447401.50386965},"point_estimate":88335137.14076655,"standard_error":620016.4696778981},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":1175926.1471010144,"upper_bound":2457817.688689552},"point_estimate":1960933.0958133002,"standard_error":325181.06749720604}} -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/new/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,Wasm,,,91643037.0,ns,1 3 | import block,,Wasm,,,177686471.0,ns,2 4 | import block,,Wasm,,,269846396.0,ns,3 5 | import block,,Wasm,,,361857092.0,ns,4 6 | import block,,Wasm,,,446704337.0,ns,5 7 | import block,,Wasm,,,533563542.0,ns,6 8 | import block,,Wasm,,,622811274.0,ns,7 9 | import block,,Wasm,,,723976134.0,ns,8 10 | import block,,Wasm,,,807220789.0,ns,9 11 | import block,,Wasm,,,893263032.0,ns,10 12 | import block,,Wasm,,,989740499.0,ns,11 13 | import block,,Wasm,,,1038518514.0,ns,12 14 | import block,,Wasm,,,1123103257.0,ns,13 15 | import block,,Wasm,,,1180047005.0,ns,14 16 | import block,,Wasm,,,1327464541.0,ns,15 17 | import block,,Wasm,,,1358153010.0,ns,16 18 | import block,,Wasm,,,1480275925.0,ns,17 19 | import block,,Wasm,,,1627631170.0,ns,18 20 | import block,,Wasm,,,1700761177.0,ns,19 21 | import block,,Wasm,,,1807227661.0,ns,20 22 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/new/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0],[91643037.0,177686471.0,269846396.0,361857092.0,446704337.0,533563542.0,622811274.0,723976134.0,807220789.0,893263032.0,989740499.0,1038518514.0,1123103257.0,1180047005.0,1327464541.0,1358153010.0,1480275925.0,1627631170.0,1700761177.0,1807227661.0]] -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/new/tukey.json: -------------------------------------------------------------------------------- 1 | [82350005.07426466,85245997.8636029,92968645.30183825,95864638.09117651] -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/report/both/regression.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block/Wasm 4 | 5 | 6 | Total sample time (s) 7 | 8 | 9 | Iterations 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 0.2 33 | 34 | 35 | 36 | 0.4 37 | 38 | 39 | 40 | 0.6 41 | 42 | 43 | 44 | 0.8 45 | 46 | 47 | 48 | 1.0 49 | 50 | 51 | 52 | 1.2 53 | 54 | 55 | 56 | 1.4 57 | 58 | 59 | 60 | 1.6 61 | 62 | 63 | 64 | 65 | 2 66 | 67 | 68 | 69 | 4 70 | 71 | 72 | 73 | 6 74 | 75 | 76 | 77 | 8 78 | 79 | 80 | 81 | 10 82 | 83 | 84 | 85 | 12 86 | 87 | 88 | 89 | 14 90 | 91 | 92 | 93 | 16 94 | 95 | 96 | 97 | 18 98 | 99 | 100 | 101 | 20 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | Base Sample 110 | 111 | 112 | New Sample 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/report/pdf_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Density (a.u.) 4 | 5 | 6 | Average Time (ms) 7 | 8 | 9 | 10 | 0.05 11 | 12 | 13 | 14 | 0.1 15 | 16 | 17 | 18 | 0.15 19 | 20 | 21 | 22 | 0.2 23 | 24 | 25 | 26 | 27 | 85 28 | 29 | 30 | 31 | 90 32 | 33 | 34 | 35 | 95 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/report/regression_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Total sample time (s) 4 | 5 | 6 | Iterations 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 0.2 30 | 31 | 32 | 33 | 0.4 34 | 35 | 36 | 37 | 0.6 38 | 39 | 40 | 41 | 0.8 42 | 43 | 44 | 45 | 1.0 46 | 47 | 48 | 49 | 1.2 50 | 51 | 52 | 53 | 1.4 54 | 55 | 56 | 57 | 1.6 58 | 59 | 60 | 61 | 62 | 2 63 | 64 | 65 | 66 | 4 67 | 68 | 69 | 70 | 6 71 | 72 | 73 | 74 | 8 75 | 76 | 77 | 78 | 10 79 | 80 | 81 | 82 | 12 83 | 84 | 85 | 86 | 14 87 | 88 | 89 | 90 | 16 91 | 92 | 93 | 94 | 18 95 | 96 | 97 | 98 | 20 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /criterion-files/import block/Wasm/report/relative_regression_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Total sample time (s) 4 | 5 | 6 | Iterations 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 0.2 30 | 31 | 32 | 33 | 0.4 34 | 35 | 36 | 37 | 0.6 38 | 39 | 40 | 41 | 0.8 42 | 43 | 44 | 45 | 1.0 46 | 47 | 48 | 49 | 1.2 50 | 51 | 52 | 53 | 1.4 54 | 55 | 56 | 57 | 1.6 58 | 59 | 60 | 61 | 62 | 2 63 | 64 | 65 | 66 | 4 67 | 68 | 69 | 70 | 6 71 | 72 | 73 | 74 | 8 75 | 76 | 77 | 78 | 10 79 | 80 | 81 | 82 | 12 83 | 84 | 85 | 86 | 14 87 | 88 | 89 | 90 | 16 91 | 92 | 93 | 94 | 18 95 | 96 | 97 | 98 | 20 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /criterion-files/import block/base/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":null,"throughput":null,"full_id":"import block","directory_name":"import block","title":"import block"} -------------------------------------------------------------------------------- /criterion-files/import block/base/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30392205.156428568,"upper_bound":31505830.373333335},"point_estimate":30911080.594047617,"standard_error":285796.04614960734},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30215201.028571427,"upper_bound":31565421.0},"point_estimate":30574424.25,"standard_error":360141.97486123693},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":124063.90402242633,"upper_bound":1593315.6594879935},"point_estimate":901420.5192465859,"standard_error":379747.5279951375},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30364880.338235293,"upper_bound":32020276.19117647},"point_estimate":31036220.355844155,"standard_error":461351.33154414425},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":458687.7522258452,"upper_bound":1266219.5563831013},"point_estimate":951588.7522659358,"standard_error":222671.01099164112}} -------------------------------------------------------------------------------- /criterion-files/import block/base/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,,,,29726856.0,ns,1 3 | import block,,,,,63130842.0,ns,2 4 | import block,,,,,91689218.0,ns,3 5 | import block,,,,,124921854.0,ns,4 6 | import block,,,,,158408584.0,ns,5 7 | import block,,,,,183514655.0,ns,6 8 | import block,,,,,210101247.0,ns,7 9 | import block,,,,,243165702.0,ns,8 10 | import block,,,,,296382468.0,ns,9 11 | import block,,,,,304159382.0,ns,10 12 | -------------------------------------------------------------------------------- /criterion-files/import block/base/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0],[29726856.0,63130842.0,91689218.0,124921854.0,158408584.0,183514655.0,210101247.0,243165702.0,296382468.0,304159382.0]] -------------------------------------------------------------------------------- /criterion-files/import block/base/tukey.json: -------------------------------------------------------------------------------- 1 | [27158031.575000003,28779400.34375,33103050.393749997,34724419.162499994] -------------------------------------------------------------------------------- /criterion-files/import block/change/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.8313243536044368,"upper_bound":-0.8248280086046458},"point_estimate":-0.8282723667491974,"standard_error":0.0017065700908944449},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":-0.831502054923259,"upper_bound":-0.8241087299803227},"point_estimate":-0.8297879414372005,"standard_error":0.0020997033328263057}} -------------------------------------------------------------------------------- /criterion-files/import block/new/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block","function_id":null,"value_str":null,"throughput":null,"full_id":"import block","directory_name":"import block","title":"import block"} -------------------------------------------------------------------------------- /criterion-files/import block/new/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30392205.156428568,"upper_bound":31505830.373333335},"point_estimate":30911080.594047617,"standard_error":285796.04614960734},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30215201.028571427,"upper_bound":31565421.0},"point_estimate":30574424.25,"standard_error":360141.97486123693},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":124063.90402242633,"upper_bound":1593315.6594879935},"point_estimate":901420.5192465859,"standard_error":379747.5279951375},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":30364880.338235293,"upper_bound":32020276.19117647},"point_estimate":31036220.355844155,"standard_error":461351.33154414425},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":458687.7522258452,"upper_bound":1266219.5563831013},"point_estimate":951588.7522659358,"standard_error":222671.01099164112}} -------------------------------------------------------------------------------- /criterion-files/import block/new/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block,,,,,29726856.0,ns,1 3 | import block,,,,,63130842.0,ns,2 4 | import block,,,,,91689218.0,ns,3 5 | import block,,,,,124921854.0,ns,4 6 | import block,,,,,158408584.0,ns,5 7 | import block,,,,,183514655.0,ns,6 8 | import block,,,,,210101247.0,ns,7 9 | import block,,,,,243165702.0,ns,8 10 | import block,,,,,296382468.0,ns,9 11 | import block,,,,,304159382.0,ns,10 12 | -------------------------------------------------------------------------------- /criterion-files/import block/new/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0],[29726856.0,63130842.0,91689218.0,124921854.0,158408584.0,183514655.0,210101247.0,243165702.0,296382468.0,304159382.0]] -------------------------------------------------------------------------------- /criterion-files/import block/new/tukey.json: -------------------------------------------------------------------------------- 1 | [27158031.575000003,28779400.34375,33103050.393749997,34724419.162499994] -------------------------------------------------------------------------------- /criterion-files/import block/report/both/regression.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block 4 | 5 | 6 | Total sample time (s) 7 | 8 | 9 | Iterations 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 0.2 32 | 33 | 34 | 35 | 0.4 36 | 37 | 38 | 39 | 0.6 40 | 41 | 42 | 43 | 0.8 44 | 45 | 46 | 47 | 1.0 48 | 49 | 50 | 51 | 1.2 52 | 53 | 54 | 55 | 1.4 56 | 57 | 58 | 59 | 1.6 60 | 61 | 62 | 63 | 64 | 1 65 | 66 | 67 | 68 | 2 69 | 70 | 71 | 72 | 3 73 | 74 | 75 | 76 | 4 77 | 78 | 79 | 80 | 5 81 | 82 | 83 | 84 | 6 85 | 86 | 87 | 88 | 7 89 | 90 | 91 | 92 | 8 93 | 94 | 95 | 96 | 9 97 | 98 | 99 | 100 | 10 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | Base Sample 109 | 110 | 111 | New Sample 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /criterion-files/import block/report/change/t-test.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block: Welch t test 4 | 5 | 6 | Density 7 | 8 | 9 | t score 10 | 11 | 12 | 13 | 0.1 14 | 15 | 16 | 17 | 0.2 18 | 19 | 20 | 21 | 0.3 22 | 23 | 24 | 25 | 0.4 26 | 27 | 28 | 29 | 0.5 30 | 31 | 32 | 33 | 0.6 34 | 35 | 36 | 37 | 38 | -8.0 39 | 40 | 41 | 42 | -6.0 43 | 44 | 45 | 46 | -4.0 47 | 48 | 49 | 50 | -2.0 51 | 52 | 53 | 54 | 0.0 55 | 56 | 57 | 58 | 2.0 59 | 60 | 61 | 62 | 4.0 63 | 64 | 65 | 66 | 6.0 67 | 68 | 69 | 70 | 8.0 71 | 72 | 73 | 74 | 75 | 76 | t distribution 77 | 78 | 79 | t statistic 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /criterion-files/import block/report/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | import block Summary - Criterion.rs 7 | 50 | 51 | 52 | 53 |
54 |

import block

55 |

Violin Plot

56 | 57 | Violin Plot 58 | 59 |

This chart shows the relationship between function/parameter and iteration time. The thickness of the shaded 60 | region indicates the probability that a measurement of the given function/parameter would take a particular 61 | length of time.

62 |
63 | 64 |

import block/Native

65 | 66 | 67 | 68 | 69 | 75 | 81 | 82 | 83 |
70 | 71 | PDF of Slope 73 | 74 | 76 | 77 | Regression 79 | 80 |
84 |
85 |
86 | 87 |

import block/Wasm

88 | 89 | 90 | 91 | 92 | 98 | 104 | 105 | 106 |
93 | 94 | PDF of Slope 96 | 97 | 99 | 100 | Regression 102 | 103 |
107 |
108 |
109 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /criterion-files/import block/report/pdf_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Density (a.u.) 4 | 5 | 6 | Average Time (ms) 7 | 8 | 9 | 10 | 0.05 11 | 12 | 13 | 14 | 0.1 15 | 16 | 17 | 18 | 0.15 19 | 20 | 21 | 22 | 0.2 23 | 24 | 25 | 26 | 0.25 27 | 28 | 29 | 30 | 0.3 31 | 32 | 33 | 34 | 0.35 35 | 36 | 37 | 38 | 0.4 39 | 40 | 41 | 42 | 43 | 28 44 | 45 | 46 | 47 | 30 48 | 49 | 50 | 51 | 32 52 | 53 | 54 | 55 | 34 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /criterion-files/import block/report/regression.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | import block 4 | 5 | 6 | Total sample time (ms) 7 | 8 | 9 | Iterations 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 50.0 29 | 30 | 31 | 32 | 100.0 33 | 34 | 35 | 36 | 150.0 37 | 38 | 39 | 40 | 200.0 41 | 42 | 43 | 44 | 250.0 45 | 46 | 47 | 48 | 300.0 49 | 50 | 51 | 52 | 53 | 2 54 | 55 | 56 | 57 | 3 58 | 59 | 60 | 61 | 4 62 | 63 | 64 | 65 | 5 66 | 67 | 68 | 69 | 6 70 | 71 | 72 | 73 | 7 74 | 75 | 76 | 77 | 8 78 | 79 | 80 | 81 | 9 82 | 83 | 84 | 85 | 10 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | Sample 102 | 103 | 104 | Linear regression 105 | 106 | 107 | Confidence interval 108 | 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /criterion-files/import block/report/regression_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Total sample time (ms) 4 | 5 | 6 | Iterations 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 50.0 26 | 27 | 28 | 29 | 100.0 30 | 31 | 32 | 33 | 150.0 34 | 35 | 36 | 37 | 200.0 38 | 39 | 40 | 41 | 250.0 42 | 43 | 44 | 45 | 300.0 46 | 47 | 48 | 49 | 50 | 2 51 | 52 | 53 | 54 | 3 55 | 56 | 57 | 58 | 4 59 | 60 | 61 | 62 | 5 63 | 64 | 65 | 66 | 6 67 | 68 | 69 | 70 | 7 71 | 72 | 73 | 74 | 8 75 | 76 | 77 | 78 | 9 79 | 80 | 81 | 82 | 10 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /criterion-files/import block/report/relative_regression_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Total sample time (s) 4 | 5 | 6 | Iterations 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 0.2 29 | 30 | 31 | 32 | 0.4 33 | 34 | 35 | 36 | 0.6 37 | 38 | 39 | 40 | 0.8 41 | 42 | 43 | 44 | 1.0 45 | 46 | 47 | 48 | 1.2 49 | 50 | 51 | 52 | 1.4 53 | 54 | 55 | 56 | 1.6 57 | 58 | 59 | 60 | 61 | 1 62 | 63 | 64 | 65 | 2 66 | 67 | 68 | 69 | 3 70 | 71 | 72 | 73 | 4 74 | 75 | 76 | 77 | 5 78 | 79 | 80 | 81 | 6 82 | 83 | 84 | 85 | 7 86 | 87 | 88 | 89 | 8 90 | 91 | 92 | 93 | 9 94 | 95 | 96 | 97 | 10 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/paritytech/bench-bot/b5dbd8635cbe0038d0167fcdeea0b51f12ea5e92/criterion-files/import-block-with-ed25519-B-0003/Native/.DS_Store -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/base/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block with ed25519 B-0003","function_id":null,"value_str":"Native","throughput":null,"full_id":"import block with ed25519 B-0003/Native","directory_name":"import block with ed25519 B-0003/Native","title":"import block with ed25519 B-0003/Native"} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/base/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":41346770.654568866,"upper_bound":43839819.816103265},"point_estimate":42588766.662221216,"standard_error":635929.6541829065},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":40191126.578947365,"upper_bound":44053252.92490843},"point_estimate":42364376.248817965,"standard_error":1267471.5625723551},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":3578766.309425605,"upper_bound":6774826.637913696},"point_estimate":4994584.17108956,"standard_error":830457.6844299687},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":41705999.80552586,"upper_bound":44893428.341629975},"point_estimate":43297591.53052999,"standard_error":817902.9074605453},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":3816820.4634299353,"upper_bound":5165122.70084337},"point_estimate":4553233.6798023125,"standard_error":344081.8397460011}} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/base/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block with ed25519 B-0003,,Native,,,32878742.0,ns,1 3 | import block with ed25519 B-0003,,Native,,,75457681.0,ns,2 4 | import block with ed25519 B-0003,,Native,,,110613476.0,ns,3 5 | import block with ed25519 B-0003,,Native,,,183911778.0,ns,4 6 | import block with ed25519 B-0003,,Native,,,182452308.0,ns,5 7 | import block with ed25519 B-0003,,Native,,,211542920.0,ns,6 8 | import block with ed25519 B-0003,,Native,,,285992950.0,ns,7 9 | import block with ed25519 B-0003,,Native,,,360983760.0,ns,8 10 | import block with ed25519 B-0003,,Native,,,445762198.0,ns,9 11 | import block with ed25519 B-0003,,Native,,,383260475.0,ns,10 12 | import block with ed25519 B-0003,,Native,,,427672582.0,ns,11 13 | import block with ed25519 B-0003,,Native,,,456004583.0,ns,12 14 | import block with ed25519 B-0003,,Native,,,511210363.0,ns,13 15 | import block with ed25519 B-0003,,Native,,,557005430.0,ns,14 16 | import block with ed25519 B-0003,,Native,,,653910143.0,ns,15 17 | import block with ed25519 B-0003,,Native,,,815979413.0,ns,16 18 | import block with ed25519 B-0003,,Native,,,696372016.0,ns,17 19 | import block with ed25519 B-0003,,Native,,,756267029.0,ns,18 20 | import block with ed25519 B-0003,,Native,,,775849715.0,ns,19 21 | import block with ed25519 B-0003,,Native,,,1026912084.0,ns,20 22 | import block with ed25519 B-0003,,Native,,,921450704.0,ns,21 23 | import block with ed25519 B-0003,,Native,,,966393548.0,ns,22 24 | import block with ed25519 B-0003,,Native,,,1130612098.0,ns,23 25 | import block with ed25519 B-0003,,Native,,,1160634279.0,ns,24 26 | import block with ed25519 B-0003,,Native,,,1110317616.0,ns,25 27 | import block with ed25519 B-0003,,Native,,,1023494612.0,ns,26 28 | import block with ed25519 B-0003,,Native,,,1282223423.0,ns,27 29 | import block with ed25519 B-0003,,Native,,,1121035777.0,ns,28 30 | import block with ed25519 B-0003,,Native,,,1152517988.0,ns,29 31 | import block with ed25519 B-0003,,Native,,,1344029191.0,ns,30 32 | import block with ed25519 B-0003,,Native,,,1121179692.0,ns,31 33 | import block with ed25519 B-0003,,Native,,,1399811451.0,ns,32 34 | import block with ed25519 B-0003,,Native,,,1351639591.0,ns,33 35 | import block with ed25519 B-0003,,Native,,,1373686409.0,ns,34 36 | import block with ed25519 B-0003,,Native,,,1636330359.0,ns,35 37 | import block with ed25519 B-0003,,Native,,,1437594657.0,ns,36 38 | import block with ed25519 B-0003,,Native,,,1717433819.0,ns,37 39 | import block with ed25519 B-0003,,Native,,,1527262810.0,ns,38 40 | import block with ed25519 B-0003,,Native,,,1724888135.0,ns,39 41 | import block with ed25519 B-0003,,Native,,,1733617494.0,ns,40 42 | import block with ed25519 B-0003,,Native,,,1919337552.0,ns,41 43 | import block with ed25519 B-0003,,Native,,,2064086101.0,ns,42 44 | import block with ed25519 B-0003,,Native,,,2076660045.0,ns,43 45 | import block with ed25519 B-0003,,Native,,,1921980854.0,ns,44 46 | import block with ed25519 B-0003,,Native,,,2194479545.0,ns,45 47 | import block with ed25519 B-0003,,Native,,,2240305490.0,ns,46 48 | import block with ed25519 B-0003,,Native,,,2007554125.0,ns,47 49 | import block with ed25519 B-0003,,Native,,,1748620073.0,ns,48 50 | import block with ed25519 B-0003,,Native,,,1916479425.0,ns,49 51 | import block with ed25519 B-0003,,Native,,,1884870352.0,ns,50 52 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/base/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0],[32878742.0,75457681.0,110613476.0,183911778.0,182452308.0,211542920.0,285992950.0,360983760.0,445762198.0,383260475.0,427672582.0,456004583.0,511210363.0,557005430.0,653910143.0,815979413.0,696372016.0,756267029.0,775849715.0,1026912084.0,921450704.0,966393548.0,1130612098.0,1160634279.0,1110317616.0,1023494612.0,1282223423.0,1121035777.0,1152517988.0,1344029191.0,1121179692.0,1399811451.0,1351639591.0,1373686409.0,1636330359.0,1437594657.0,1717433819.0,1527262810.0,1724888135.0,1733617494.0,1919337552.0,2064086101.0,2076660045.0,1921980854.0,2194479545.0,2240305490.0,2007554125.0,1748620073.0,1916479425.0,1884870352.0]] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/base/tukey.json: -------------------------------------------------------------------------------- 1 | [18414798.193087354,28874499.048466757,56767034.66281183,67226735.51819123] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/new/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block with ed25519 B-0003","function_id":null,"value_str":"Native","throughput":null,"full_id":"import block with ed25519 B-0003/Native","directory_name":"import block with ed25519 B-0003/Native","title":"import block with ed25519 B-0003/Native"} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/new/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":41346770.654568866,"upper_bound":43839819.816103265},"point_estimate":42588766.662221216,"standard_error":635929.6541829065},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":40191126.578947365,"upper_bound":44053252.92490843},"point_estimate":42364376.248817965,"standard_error":1267471.5625723551},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":3578766.309425605,"upper_bound":6774826.637913696},"point_estimate":4994584.17108956,"standard_error":830457.6844299687},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":41705999.80552586,"upper_bound":44893428.341629975},"point_estimate":43297591.53052999,"standard_error":817902.9074605453},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":3816820.4634299353,"upper_bound":5165122.70084337},"point_estimate":4553233.6798023125,"standard_error":344081.8397460011}} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/new/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block with ed25519 B-0003,,Native,,,32878742.0,ns,1 3 | import block with ed25519 B-0003,,Native,,,75457681.0,ns,2 4 | import block with ed25519 B-0003,,Native,,,110613476.0,ns,3 5 | import block with ed25519 B-0003,,Native,,,183911778.0,ns,4 6 | import block with ed25519 B-0003,,Native,,,182452308.0,ns,5 7 | import block with ed25519 B-0003,,Native,,,211542920.0,ns,6 8 | import block with ed25519 B-0003,,Native,,,285992950.0,ns,7 9 | import block with ed25519 B-0003,,Native,,,360983760.0,ns,8 10 | import block with ed25519 B-0003,,Native,,,445762198.0,ns,9 11 | import block with ed25519 B-0003,,Native,,,383260475.0,ns,10 12 | import block with ed25519 B-0003,,Native,,,427672582.0,ns,11 13 | import block with ed25519 B-0003,,Native,,,456004583.0,ns,12 14 | import block with ed25519 B-0003,,Native,,,511210363.0,ns,13 15 | import block with ed25519 B-0003,,Native,,,557005430.0,ns,14 16 | import block with ed25519 B-0003,,Native,,,653910143.0,ns,15 17 | import block with ed25519 B-0003,,Native,,,815979413.0,ns,16 18 | import block with ed25519 B-0003,,Native,,,696372016.0,ns,17 19 | import block with ed25519 B-0003,,Native,,,756267029.0,ns,18 20 | import block with ed25519 B-0003,,Native,,,775849715.0,ns,19 21 | import block with ed25519 B-0003,,Native,,,1026912084.0,ns,20 22 | import block with ed25519 B-0003,,Native,,,921450704.0,ns,21 23 | import block with ed25519 B-0003,,Native,,,966393548.0,ns,22 24 | import block with ed25519 B-0003,,Native,,,1130612098.0,ns,23 25 | import block with ed25519 B-0003,,Native,,,1160634279.0,ns,24 26 | import block with ed25519 B-0003,,Native,,,1110317616.0,ns,25 27 | import block with ed25519 B-0003,,Native,,,1023494612.0,ns,26 28 | import block with ed25519 B-0003,,Native,,,1282223423.0,ns,27 29 | import block with ed25519 B-0003,,Native,,,1121035777.0,ns,28 30 | import block with ed25519 B-0003,,Native,,,1152517988.0,ns,29 31 | import block with ed25519 B-0003,,Native,,,1344029191.0,ns,30 32 | import block with ed25519 B-0003,,Native,,,1121179692.0,ns,31 33 | import block with ed25519 B-0003,,Native,,,1399811451.0,ns,32 34 | import block with ed25519 B-0003,,Native,,,1351639591.0,ns,33 35 | import block with ed25519 B-0003,,Native,,,1373686409.0,ns,34 36 | import block with ed25519 B-0003,,Native,,,1636330359.0,ns,35 37 | import block with ed25519 B-0003,,Native,,,1437594657.0,ns,36 38 | import block with ed25519 B-0003,,Native,,,1717433819.0,ns,37 39 | import block with ed25519 B-0003,,Native,,,1527262810.0,ns,38 40 | import block with ed25519 B-0003,,Native,,,1724888135.0,ns,39 41 | import block with ed25519 B-0003,,Native,,,1733617494.0,ns,40 42 | import block with ed25519 B-0003,,Native,,,1919337552.0,ns,41 43 | import block with ed25519 B-0003,,Native,,,2064086101.0,ns,42 44 | import block with ed25519 B-0003,,Native,,,2076660045.0,ns,43 45 | import block with ed25519 B-0003,,Native,,,1921980854.0,ns,44 46 | import block with ed25519 B-0003,,Native,,,2194479545.0,ns,45 47 | import block with ed25519 B-0003,,Native,,,2240305490.0,ns,46 48 | import block with ed25519 B-0003,,Native,,,2007554125.0,ns,47 49 | import block with ed25519 B-0003,,Native,,,1748620073.0,ns,48 50 | import block with ed25519 B-0003,,Native,,,1916479425.0,ns,49 51 | import block with ed25519 B-0003,,Native,,,1884870352.0,ns,50 52 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/new/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0],[32878742.0,75457681.0,110613476.0,183911778.0,182452308.0,211542920.0,285992950.0,360983760.0,445762198.0,383260475.0,427672582.0,456004583.0,511210363.0,557005430.0,653910143.0,815979413.0,696372016.0,756267029.0,775849715.0,1026912084.0,921450704.0,966393548.0,1130612098.0,1160634279.0,1110317616.0,1023494612.0,1282223423.0,1121035777.0,1152517988.0,1344029191.0,1121179692.0,1399811451.0,1351639591.0,1373686409.0,1636330359.0,1437594657.0,1717433819.0,1527262810.0,1724888135.0,1733617494.0,1919337552.0,2064086101.0,2076660045.0,1921980854.0,2194479545.0,2240305490.0,2007554125.0,1748620073.0,1916479425.0,1884870352.0]] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/new/tukey.json: -------------------------------------------------------------------------------- 1 | [18414798.193087354,28874499.048466757,56767034.66281183,67226735.51819123] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/report/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | import block with ed25519 B-0003/Native - Criterion.rs 7 | 82 | 83 | 84 | 85 |
86 |

import block with ed25519 B-0003/Native

87 |
88 |
89 | 90 | 91 | 92 | 97 | 102 | 103 | 104 |
93 | 94 | PDF of Slope 95 | 96 | 98 | 99 | Regression 100 | 101 |
105 |
106 |
107 |
108 |

Additional Statistics:

109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 |
Lower boundEstimateUpper bound
Slope41.706 ms43.298 ms44.893 ms
R²0.41630580.44961730.4161411
Mean41.347 ms42.589 ms43.840 ms
Std. Dev.3.8168 ms4.5532 ms5.1651 ms
Median40.191 ms42.364 ms44.053 ms
MAD3.5788 ms4.9946 ms6.7748 ms
157 |
158 |
159 |

Additional Plots:

160 |
    161 | 162 |
  • 163 | Slope 164 |
    • 165 |
  • 166 | Mean 167 |
    • 168 |
  • 169 | Std. Dev. 170 |
    • 171 |
  • 172 | Median 173 |
    • 174 |
  • 175 | MAD 176 |
    • 177 |
178 |
179 |
180 |
181 |

Understanding this report:

182 |

The plot on the left displays the average time per iteration for this benchmark. The shaded region 183 | shows the estimated probabilty of an iteration taking a certain amount of time, while the line 184 | shows the mean. Click on the plot for a larger view showing the outliers.

185 |

The plot on the right shows the linear regression calculated from the measurements. Each point 186 | represents a sample, though here it shows the total time for the sample rather than time per 187 | iteration. The line is the line of best fit for these measurements.

188 |

See the 189 | documentation for more details on the additional statistics.

190 |
191 |
192 |
193 | 198 | 199 | 200 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Native/report/pdf_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Density (a.u.) 4 | 5 | 6 | Average Time (ms) 7 | 8 | 9 | 10 | 0.01 11 | 12 | 13 | 14 | 0.02 15 | 16 | 17 | 18 | 0.03 19 | 20 | 21 | 22 | 0.04 23 | 24 | 25 | 26 | 0.05 27 | 28 | 29 | 30 | 0.06 31 | 32 | 33 | 34 | 0.07 35 | 36 | 37 | 38 | 0.08 39 | 40 | 41 | 42 | 43 | 30 44 | 45 | 46 | 47 | 40 48 | 49 | 50 | 51 | 50 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/base/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block with ed25519 B-0003","function_id":null,"value_str":"Wasm","throughput":null,"full_id":"import block with ed25519 B-0003/Wasm","directory_name":"import block with ed25519 B-0003/Wasm","title":"import block with ed25519 B-0003/Wasm"} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/base/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":133450486.81126772,"upper_bound":140494450.45037556},"point_estimate":136818782.5403393,"standard_error":1793208.1993405146},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":130837492.33333333,"upper_bound":138783712.3043478},"point_estimate":134940914.15454546,"standard_error":1639749.4575453778},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":7810542.315144707,"upper_bound":14721699.11905745},"point_estimate":10942484.63753205,"standard_error":1753295.0483952558},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":131120878.02272655,"upper_bound":139175872.39424548},"point_estimate":134874650.2255562,"standard_error":2047675.7105017537},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":9300983.786294702,"upper_bound":16044445.40030727},"point_estimate":12810704.743468622,"standard_error":1743970.1579782623}} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/base/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block with ed25519 B-0003,,Wasm,,,124083655.0,ns,1 3 | import block with ed25519 B-0003,,Wasm,,,325268347.0,ns,2 4 | import block with ed25519 B-0003,,Wasm,,,386000982.0,ns,3 5 | import block with ed25519 B-0003,,Wasm,,,503648674.0,ns,4 6 | import block with ed25519 B-0003,,Wasm,,,674172002.0,ns,5 7 | import block with ed25519 B-0003,,Wasm,,,805486035.0,ns,6 8 | import block with ed25519 B-0003,,Wasm,,,987357129.0,ns,7 9 | import block with ed25519 B-0003,,Wasm,,,1455688925.0,ns,8 10 | import block with ed25519 B-0003,,Wasm,,,1177537431.0,ns,9 11 | import block with ed25519 B-0003,,Wasm,,,1348011738.0,ns,10 12 | import block with ed25519 B-0003,,Wasm,,,1485521707.0,ns,11 13 | import block with ed25519 B-0003,,Wasm,,,1744604897.0,ns,12 14 | import block with ed25519 B-0003,,Wasm,,,1872098143.0,ns,13 15 | import block with ed25519 B-0003,,Wasm,,,1755508838.0,ns,14 16 | import block with ed25519 B-0003,,Wasm,,,1857581238.0,ns,15 17 | import block with ed25519 B-0003,,Wasm,,,2506922903.0,ns,16 18 | import block with ed25519 B-0003,,Wasm,,,2343381785.0,ns,17 19 | import block with ed25519 B-0003,,Wasm,,,2412655621.0,ns,18 20 | import block with ed25519 B-0003,,Wasm,,,2593551916.0,ns,19 21 | import block with ed25519 B-0003,,Wasm,,,2629492331.0,ns,20 22 | import block with ed25519 B-0003,,Wasm,,,3285800001.0,ns,21 23 | import block with ed25519 B-0003,,Wasm,,,3116219833.0,ns,22 24 | import block with ed25519 B-0003,,Wasm,,,3228480548.0,ns,23 25 | import block with ed25519 B-0003,,Wasm,,,2896076356.0,ns,24 26 | import block with ed25519 B-0003,,Wasm,,,3682732882.0,ns,25 27 | import block with ed25519 B-0003,,Wasm,,,3524625765.0,ns,26 28 | import block with ed25519 B-0003,,Wasm,,,3869073282.0,ns,27 29 | import block with ed25519 B-0003,,Wasm,,,3574384952.0,ns,28 30 | import block with ed25519 B-0003,,Wasm,,,4049069640.0,ns,29 31 | import block with ed25519 B-0003,,Wasm,,,4053017423.0,ns,30 32 | import block with ed25519 B-0003,,Wasm,,,3593986285.0,ns,31 33 | import block with ed25519 B-0003,,Wasm,,,4633774693.0,ns,32 34 | import block with ed25519 B-0003,,Wasm,,,4683726372.0,ns,33 35 | import block with ed25519 B-0003,,Wasm,,,4364577670.0,ns,34 36 | import block with ed25519 B-0003,,Wasm,,,4197541317.0,ns,35 37 | import block with ed25519 B-0003,,Wasm,,,4424417351.0,ns,36 38 | import block with ed25519 B-0003,,Wasm,,,5003633521.0,ns,37 39 | import block with ed25519 B-0003,,Wasm,,,4830830240.0,ns,38 40 | import block with ed25519 B-0003,,Wasm,,,6319423712.0,ns,39 41 | import block with ed25519 B-0003,,Wasm,,,6053271044.0,ns,40 42 | import block with ed25519 B-0003,,Wasm,,,6462157316.0,ns,41 43 | import block with ed25519 B-0003,,Wasm,,,5981548240.0,ns,42 44 | import block with ed25519 B-0003,,Wasm,,,5338374710.0,ns,43 45 | import block with ed25519 B-0003,,Wasm,,,5666004630.0,ns,44 46 | import block with ed25519 B-0003,,Wasm,,,5536073619.0,ns,45 47 | import block with ed25519 B-0003,,Wasm,,,6384050766.0,ns,46 48 | import block with ed25519 B-0003,,Wasm,,,6264274995.0,ns,47 49 | import block with ed25519 B-0003,,Wasm,,,5996686321.0,ns,48 50 | import block with ed25519 B-0003,,Wasm,,,6407644266.0,ns,49 51 | import block with ed25519 B-0003,,Wasm,,,6532546024.0,ns,50 52 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/base/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0],[124083655.0,325268347.0,386000982.0,503648674.0,674172002.0,805486035.0,987357129.0,1455688925.0,1177537431.0,1348011738.0,1485521707.0,1744604897.0,1872098143.0,1755508838.0,1857581238.0,2506922903.0,2343381785.0,2412655621.0,2593551916.0,2629492331.0,3285800001.0,3116219833.0,3228480548.0,2896076356.0,3682732882.0,3524625765.0,3869073282.0,3574384952.0,4049069640.0,4053017423.0,3593986285.0,4633774693.0,4683726372.0,4364577670.0,4197541317.0,4424417351.0,5003633521.0,4830830240.0,6319423712.0,6053271044.0,6462157316.0,5981548240.0,5338374710.0,5666004630.0,5536073619.0,6384050766.0,6264274995.0,5996686321.0,6407644266.0,6532546024.0]] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/base/tukey.json: -------------------------------------------------------------------------------- 1 | [84451336.83422464,106143136.88665015,163987937.02645147,185679737.07887697] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/new/benchmark.json: -------------------------------------------------------------------------------- 1 | {"group_id":"import block with ed25519 B-0003","function_id":null,"value_str":"Wasm","throughput":null,"full_id":"import block with ed25519 B-0003/Wasm","directory_name":"import block with ed25519 B-0003/Wasm","title":"import block with ed25519 B-0003/Wasm"} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/new/estimates.json: -------------------------------------------------------------------------------- 1 | {"Mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":133450486.81126772,"upper_bound":140494450.45037556},"point_estimate":136818782.5403393,"standard_error":1793208.1993405146},"Median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":130837492.33333333,"upper_bound":138783712.3043478},"point_estimate":134940914.15454546,"standard_error":1639749.4575453778},"MedianAbsDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":7810542.315144707,"upper_bound":14721699.11905745},"point_estimate":10942484.63753205,"standard_error":1753295.0483952558},"Slope":{"confidence_interval":{"confidence_level":0.95,"lower_bound":131120878.02272655,"upper_bound":139175872.39424548},"point_estimate":134874650.2255562,"standard_error":2047675.7105017537},"StdDev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":9300983.786294702,"upper_bound":16044445.40030727},"point_estimate":12810704.743468622,"standard_error":1743970.1579782623}} -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/new/raw.csv: -------------------------------------------------------------------------------- 1 | group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count 2 | import block with ed25519 B-0003,,Wasm,,,124083655.0,ns,1 3 | import block with ed25519 B-0003,,Wasm,,,325268347.0,ns,2 4 | import block with ed25519 B-0003,,Wasm,,,386000982.0,ns,3 5 | import block with ed25519 B-0003,,Wasm,,,503648674.0,ns,4 6 | import block with ed25519 B-0003,,Wasm,,,674172002.0,ns,5 7 | import block with ed25519 B-0003,,Wasm,,,805486035.0,ns,6 8 | import block with ed25519 B-0003,,Wasm,,,987357129.0,ns,7 9 | import block with ed25519 B-0003,,Wasm,,,1455688925.0,ns,8 10 | import block with ed25519 B-0003,,Wasm,,,1177537431.0,ns,9 11 | import block with ed25519 B-0003,,Wasm,,,1348011738.0,ns,10 12 | import block with ed25519 B-0003,,Wasm,,,1485521707.0,ns,11 13 | import block with ed25519 B-0003,,Wasm,,,1744604897.0,ns,12 14 | import block with ed25519 B-0003,,Wasm,,,1872098143.0,ns,13 15 | import block with ed25519 B-0003,,Wasm,,,1755508838.0,ns,14 16 | import block with ed25519 B-0003,,Wasm,,,1857581238.0,ns,15 17 | import block with ed25519 B-0003,,Wasm,,,2506922903.0,ns,16 18 | import block with ed25519 B-0003,,Wasm,,,2343381785.0,ns,17 19 | import block with ed25519 B-0003,,Wasm,,,2412655621.0,ns,18 20 | import block with ed25519 B-0003,,Wasm,,,2593551916.0,ns,19 21 | import block with ed25519 B-0003,,Wasm,,,2629492331.0,ns,20 22 | import block with ed25519 B-0003,,Wasm,,,3285800001.0,ns,21 23 | import block with ed25519 B-0003,,Wasm,,,3116219833.0,ns,22 24 | import block with ed25519 B-0003,,Wasm,,,3228480548.0,ns,23 25 | import block with ed25519 B-0003,,Wasm,,,2896076356.0,ns,24 26 | import block with ed25519 B-0003,,Wasm,,,3682732882.0,ns,25 27 | import block with ed25519 B-0003,,Wasm,,,3524625765.0,ns,26 28 | import block with ed25519 B-0003,,Wasm,,,3869073282.0,ns,27 29 | import block with ed25519 B-0003,,Wasm,,,3574384952.0,ns,28 30 | import block with ed25519 B-0003,,Wasm,,,4049069640.0,ns,29 31 | import block with ed25519 B-0003,,Wasm,,,4053017423.0,ns,30 32 | import block with ed25519 B-0003,,Wasm,,,3593986285.0,ns,31 33 | import block with ed25519 B-0003,,Wasm,,,4633774693.0,ns,32 34 | import block with ed25519 B-0003,,Wasm,,,4683726372.0,ns,33 35 | import block with ed25519 B-0003,,Wasm,,,4364577670.0,ns,34 36 | import block with ed25519 B-0003,,Wasm,,,4197541317.0,ns,35 37 | import block with ed25519 B-0003,,Wasm,,,4424417351.0,ns,36 38 | import block with ed25519 B-0003,,Wasm,,,5003633521.0,ns,37 39 | import block with ed25519 B-0003,,Wasm,,,4830830240.0,ns,38 40 | import block with ed25519 B-0003,,Wasm,,,6319423712.0,ns,39 41 | import block with ed25519 B-0003,,Wasm,,,6053271044.0,ns,40 42 | import block with ed25519 B-0003,,Wasm,,,6462157316.0,ns,41 43 | import block with ed25519 B-0003,,Wasm,,,5981548240.0,ns,42 44 | import block with ed25519 B-0003,,Wasm,,,5338374710.0,ns,43 45 | import block with ed25519 B-0003,,Wasm,,,5666004630.0,ns,44 46 | import block with ed25519 B-0003,,Wasm,,,5536073619.0,ns,45 47 | import block with ed25519 B-0003,,Wasm,,,6384050766.0,ns,46 48 | import block with ed25519 B-0003,,Wasm,,,6264274995.0,ns,47 49 | import block with ed25519 B-0003,,Wasm,,,5996686321.0,ns,48 50 | import block with ed25519 B-0003,,Wasm,,,6407644266.0,ns,49 51 | import block with ed25519 B-0003,,Wasm,,,6532546024.0,ns,50 52 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/new/sample.json: -------------------------------------------------------------------------------- 1 | [[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0,17.0,18.0,19.0,20.0,21.0,22.0,23.0,24.0,25.0,26.0,27.0,28.0,29.0,30.0,31.0,32.0,33.0,34.0,35.0,36.0,37.0,38.0,39.0,40.0,41.0,42.0,43.0,44.0,45.0,46.0,47.0,48.0,49.0,50.0],[124083655.0,325268347.0,386000982.0,503648674.0,674172002.0,805486035.0,987357129.0,1455688925.0,1177537431.0,1348011738.0,1485521707.0,1744604897.0,1872098143.0,1755508838.0,1857581238.0,2506922903.0,2343381785.0,2412655621.0,2593551916.0,2629492331.0,3285800001.0,3116219833.0,3228480548.0,2896076356.0,3682732882.0,3524625765.0,3869073282.0,3574384952.0,4049069640.0,4053017423.0,3593986285.0,4633774693.0,4683726372.0,4364577670.0,4197541317.0,4424417351.0,5003633521.0,4830830240.0,6319423712.0,6053271044.0,6462157316.0,5981548240.0,5338374710.0,5666004630.0,5536073619.0,6384050766.0,6264274995.0,5996686321.0,6407644266.0,6532546024.0]] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/new/tukey.json: -------------------------------------------------------------------------------- 1 | [84451336.83422464,106143136.88665015,163987937.02645147,185679737.07887697] -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/report/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | import block with ed25519 B-0003/Wasm - Criterion.rs 7 | 82 | 83 | 84 | 85 |
86 |

import block with ed25519 B-0003/Wasm

87 |
88 |
89 | 90 | 91 | 92 | 97 | 102 | 103 | 104 |
93 | 94 | PDF of Slope 95 | 96 | 98 | 99 | Regression 100 | 101 |
105 |
106 |
107 |
108 |

Additional Statistics:

109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 |
Lower boundEstimateUpper bound
Slope131.12 ms134.87 ms139.18 ms
R²0.60888420.63358220.6015458
Mean133.45 ms136.82 ms140.49 ms
Std. Dev.9.3010 ms12.811 ms16.044 ms
Median130.84 ms134.94 ms138.78 ms
MAD7.8105 ms10.942 ms14.722 ms
157 |
158 |
159 |

Additional Plots:

160 |
    161 | 162 |
  • 163 | Slope 164 |
    • 165 |
  • 166 | Mean 167 |
    • 168 |
  • 169 | Std. Dev. 170 |
    • 171 |
  • 172 | Median 173 |
    • 174 |
  • 175 | MAD 176 |
    • 177 |
178 |
179 |
180 |
181 |

Understanding this report:

182 |

The plot on the left displays the average time per iteration for this benchmark. The shaded region 183 | shows the estimated probabilty of an iteration taking a certain amount of time, while the line 184 | shows the mean. Click on the plot for a larger view showing the outliers.

185 |

The plot on the right shows the linear regression calculated from the measurements. Each point 186 | represents a sample, though here it shows the total time for the sample rather than time per 187 | iteration. The line is the line of best fit for these measurements.

188 |

See the 189 | documentation for more details on the additional statistics.

190 |
191 |
192 |
193 | 198 | 199 | 200 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/Wasm/report/pdf_small.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | Density (a.u.) 4 | 5 | 6 | Average Time (ms) 7 | 8 | 9 | 10 | 0.005 11 | 12 | 13 | 14 | 0.01 15 | 16 | 17 | 18 | 0.015 19 | 20 | 21 | 22 | 0.02 23 | 24 | 25 | 26 | 0.025 27 | 28 | 29 | 30 | 0.03 31 | 32 | 33 | 34 | 0.035 35 | 36 | 37 | 38 | 39 | 100 40 | 41 | 42 | 43 | 150 44 | 45 | 46 | 47 | 200 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /criterion-files/import-block-with-ed25519-B-0003/report/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | import block with ed25519 B-0003 Summary - Criterion.rs 7 | 50 | 51 | 52 | 53 |
54 |

import block with ed25519 B-0003

55 |

Violin Plot

56 | 57 | Violin Plot 58 | 59 |

This chart shows the relationship between function/parameter and iteration time. The thickness of the shaded 60 | region indicates the probability that a measurement of the given function/parameter would take a particular 61 | length of time.

62 |
63 | 64 |

import block with ed25519 B-0003/Native

65 | 66 | 67 | 68 | 69 | 75 | 81 | 82 | 83 |
70 | 71 | PDF of Slope 73 | 74 | 76 | 77 | Regression 79 | 80 |
84 |
85 |
86 | 87 |

import block with ed25519 B-0003/Wasm

88 | 89 | 90 | 91 | 92 | 98 | 104 | 105 | 106 |
93 | 94 | PDF of Slope 96 | 97 | 99 | 100 | Regression 102 | 103 |
107 |
108 |
109 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { createAppAuth } = require("@octokit/auth-app") 2 | const assert = require("assert") 3 | const fs = require("fs") 4 | const shell = require("shelljs") 5 | const Mutex = require("async-mutex").Mutex 6 | 7 | var { benchBranch, benchmarkRuntime, benchRustup } = require("./bench") 8 | 9 | const githubCommentLimitLength = 65536 10 | const githubCommentLimitTruncateMessage = "..." 11 | 12 | let isTerminating = false 13 | let logFatal = undefined 14 | 15 | let pendingPayloadCount = 0 16 | 17 | for (const event of ["uncaughtException", "unhandledRejection"]) { 18 | process.on(event, function (error, origin) { 19 | if (isTerminating) { 20 | return 21 | } 22 | isTerminating = true 23 | 24 | try { 25 | if (logFatal) { 26 | logFatal({ event, error, origin }) 27 | } 28 | } catch (error) { 29 | console.error({ level: "error", event, error, origin, exception }) 30 | } 31 | 32 | process.exit(1) 33 | }) 34 | } 35 | 36 | const mutex = new Mutex() 37 | module.exports = (app) => { 38 | if (process.env.DEBUG) { 39 | app.log("Running in debug mode") 40 | } 41 | 42 | // Crash the server on Probot failures or errors 43 | // We retain the original error handlers on logError and logFatal so that the 44 | // application can still report errors on the expected channels 45 | // This is necessary to work around problems in reconnection issues from our 46 | // event source 47 | // (https://github.com/paritytech/bench-bot/issues/83#issuecomment-1024283664) 48 | // FIXME: This is suboptimal and we should not have to stop the application in 49 | // case of errors 50 | // The server will automatically restarted upon exit in ./run 51 | let isWaitingForExitingWhenFree = false 52 | const exitWhenFree = async function() { 53 | const isFree = await mutex.runExclusive(function () { 54 | return pendingPayloadCount === 0 55 | }) 56 | if (isFree) { 57 | process.exit(1) 58 | } else { 59 | await exitWhenFree() 60 | } 61 | } 62 | const logThenExit = function(log) { 63 | return async function(...args) { 64 | log(...args) 65 | // only exit the application after the current events have been processed 66 | // we only need to register this action once since exitWhenFree calls itself 67 | if (isWaitingForExitingWhenFree) { 68 | return 69 | } 70 | isWaitingForExitingWhenFree = true 71 | await exitWhenFree() 72 | } 73 | } 74 | const logError = app.log.error 75 | app.log.error = logThenExit(logError) 76 | logFatal = app.log.fatal 77 | app.log.fatal = logThenExit(logFatal) 78 | 79 | const baseBranch = process.env.BASE_BRANCH || "master" 80 | app.log.debug(`base branch: ${baseBranch}`) 81 | 82 | const appId = parseInt(process.env.APP_ID) 83 | assert(appId) 84 | 85 | const clientId = process.env.CLIENT_ID 86 | assert(clientId) 87 | const clientSecret = process.env.CLIENT_SECRET 88 | assert(clientSecret) 89 | 90 | const privateKeyPath = process.env.PRIVATE_KEY_PATH 91 | assert(privateKeyPath) 92 | const privateKey = fs.readFileSync(privateKeyPath).toString() 93 | assert(privateKey) 94 | 95 | const authInstallation = createAppAuth({ 96 | appId, 97 | privateKey, 98 | clientId, 99 | clientSecret, 100 | }) 101 | 102 | app.on("issue_comment", async (context) => { 103 | pendingPayloadCount++ 104 | await mutex.runExclusive(async function() { 105 | let commentText = context.payload.comment.body 106 | if ( 107 | !context.payload.issue.hasOwnProperty("pull_request") || 108 | context.payload.action !== "created" || 109 | !commentText.startsWith("/bench") 110 | ) { 111 | return 112 | } 113 | 114 | try { 115 | const installationId = (context.payload.installation || {}).id 116 | if (!installationId) { 117 | await context.octokit.issues.createComment( 118 | context.issue({ 119 | body: `Error: Installation id was missing from webhook payload`, 120 | }), 121 | ) 122 | return 123 | } 124 | 125 | const getPushDomain = async function () { 126 | const token = ( 127 | await authInstallation({ type: "installation", installationId }) 128 | ).token 129 | 130 | const url = `https://x-access-token:${token}@github.com` 131 | return { url, token } 132 | } 133 | 134 | const repo = context.payload.repository.name 135 | const owner = context.payload.repository.owner.login 136 | const pull_number = context.payload.issue.number 137 | 138 | // Capture `` in `/bench ` 139 | let action = commentText.split(" ").splice(1, 1).join(" ").trim() 140 | // Capture all `` text in `/bench ` 141 | let extra = commentText.split(" ").splice(2).join(" ").trim() 142 | 143 | let pr = await context.octokit.pulls.get({ owner, repo, pull_number }) 144 | const contributor = pr.data.head.user.login 145 | const branch = pr.data.head.ref 146 | app.log.debug(`branch: ${branch}`) 147 | 148 | var { stdout: toolchain, code: toolchainError } = shell.exec( 149 | "rustup show active-toolchain --verbose", 150 | { silent: false }, 151 | ) 152 | if (toolchainError) { 153 | await context.octokit.issues.createComment( 154 | context.issue({ 155 | body: "ERROR: Failed to query the currently active Rust toolchain", 156 | }), 157 | ) 158 | return 159 | } else { 160 | toolchain = toolchain.trim() 161 | } 162 | 163 | const initialInfo = `Starting benchmark for branch: ${branch} (vs ${baseBranch})\n\nToolchain: \n${toolchain}\n\n Comment will be updated.` 164 | let comment_id = undefined 165 | if (process.env.DEBUG) { 166 | app.log(initialInfo) 167 | } else { 168 | const issueComment = context.issue({ body: initialInfo }) 169 | const issue_comment = await context.octokit.issues.createComment( 170 | issueComment, 171 | ) 172 | comment_id = issue_comment.data.id 173 | } 174 | 175 | let config = { 176 | owner, 177 | contributor, 178 | repo, 179 | branch, 180 | baseBranch, 181 | id: action, 182 | extra, 183 | getPushDomain, 184 | logFatal 185 | } 186 | 187 | let report 188 | if (action == "runtime" || action == "xcm") { 189 | report = await benchmarkRuntime(app, config) 190 | } else if (action == "rustup") { 191 | report = await benchRustup(app, config) 192 | } else { 193 | report = await benchBranch(app, config) 194 | } 195 | if (process.env.DEBUG) { 196 | console.log(report) 197 | return 198 | } 199 | 200 | if (report.isError) { 201 | logError(report.message) 202 | 203 | if (report.error) { 204 | logError(report.error) 205 | } 206 | 207 | const output = `${report.message}${report.error ? `: ${report.error.toString()}` : "" 208 | }` 209 | 210 | await context.octokit.issues.updateComment({ 211 | owner, 212 | repo, 213 | comment_id, 214 | body: `Error running benchmark: **${branch}**\n\n
stdout${output}
`, 215 | }) 216 | 217 | return 218 | } 219 | 220 | let { title, output, extraInfo, benchCommand } = report 221 | 222 | const bodyPrefix = ` 223 | Benchmark **${title}** for branch "${branch}" with command ${benchCommand} 224 | 225 | Toolchain: ${toolchain} 226 | 227 |
228 | Results 229 | 230 | \`\`\` 231 | `.trim() 232 | 233 | const bodySuffix = ` 234 | \`\`\` 235 | 236 |
237 | `.trim() 238 | 239 | const padding = 16 240 | const formattingLength = 241 | bodyPrefix.length + bodySuffix.length + extraInfo.length + padding 242 | const length = formattingLength + output.length 243 | if (length >= githubCommentLimitLength) { 244 | output = `${output.slice( 245 | 0, 246 | githubCommentLimitLength - 247 | (githubCommentLimitTruncateMessage.length + formattingLength), 248 | )}${githubCommentLimitTruncateMessage}` 249 | } 250 | 251 | const body = ` 252 | ${bodyPrefix} 253 | ${output} 254 | ${bodySuffix} 255 | 256 | ${extraInfo} 257 | `.trim() 258 | 259 | await context.octokit.issues.updateComment({ 260 | owner, 261 | repo, 262 | comment_id, 263 | body, 264 | }) 265 | } catch (error) { 266 | logFatal({ 267 | error, 268 | repo, 269 | owner, 270 | pull_number, 271 | msg: "Caught exception in issue_comment's handler", 272 | }) 273 | await context.octokit.issues.createComment( 274 | context.issue({ 275 | body: `Exception caught: \`${error.message}\`\n${error.stack}`, 276 | }), 277 | ) 278 | } 279 | }) 280 | pendingPayloadCount-- 281 | }) 282 | } 283 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bench-bot", 3 | "version": "0.0.0", 4 | "description": "bench-bot", 5 | "author": "Parity ", 6 | "license": "ISC", 7 | "repository": "https://github.com/paritytech/bench-bot", 8 | "homepage": "https://github.com/paritytech/bench-bot", 9 | "bugs": "https://github.com/paritytech/bench-bot/issues", 10 | "scripts": { 11 | "dev": "nodemon", 12 | "start": "probot run ./index.js", 13 | "test": "mocha", 14 | "test:watch": "jest --watch --notify --notifyMode=change --coverage", 15 | "prettier": "prettier '{*,**/*}.{js,json,yaml,html}'", 16 | "format": "yarn run prettier --write", 17 | "lint": "yarn run prettier --check" 18 | }, 19 | "dependencies": { 20 | "@octokit/auth-app": "^3.5.3", 21 | "@octokit/webhooks": "^9.17.0", 22 | "async-mutex": "^0.3.1", 23 | "probot": "^12.1.0", 24 | "shelljs": "^0.8.4", 25 | "smee-client": "^1.2.2" 26 | }, 27 | "devDependencies": { 28 | "jest": "^27.0.6", 29 | "nodemon": "^2.0.9", 30 | "prettier": "^2.4.0", 31 | "prettier-plugin-compactify": "^0.1.4" 32 | }, 33 | "nodemonConfig": { 34 | "exec": "npm start", 35 | "watch": [ 36 | ".env", 37 | "." 38 | ] 39 | }, 40 | "jest": { 41 | "testEnvironment": "node" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /test-run.js: -------------------------------------------------------------------------------- 1 | var { benchBranch } = require("./bench") 2 | require("dotenv").config() 3 | 4 | console.log(`BASE_BRANCH=${process.env.BASE_BRANCH}`) 5 | 6 | let config = { 7 | owner: "paritytech", 8 | repo: "substrate", 9 | branch: "nv-dynamic-extensions", 10 | baseBranch: process.env.BASE_BRANCH, 11 | id: "ed25519", 12 | pushToken: null, 13 | extra: null, 14 | } 15 | 16 | benchBranch(console, config).then((report) => { 17 | console.log("Report: ") 18 | console.log(report) 19 | }) 20 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var assert = require("assert") 2 | var libCollector = require("./collector") 3 | 4 | describe("Collector", function () { 5 | it("Should collect import run results", async function () { 6 | var collector = new libCollector.Collector() 7 | await collector.CollectBaseCriterionWasmNative( 8 | "./criterion-files/import-block-with-ed25519-B-0003", 9 | ) 10 | await collector.CollectBranchCriterionWasmNative( 11 | "./criterion-files/import-block-with-ed25519-B-0003", 12 | ) 13 | 14 | assert.equal(collector.metrics["Master: Wasm"], "134.94 ms (+/- 1.64 ms)") 15 | assert.equal(collector.metrics["Master: Native"], "42.36 ms (+/- 1.27 ms)") 16 | assert.equal(collector.metrics["Branch: Wasm"], "134.94 ms (+/- 1.64 ms)") 17 | assert.equal(collector.metrics["Branch: Native"], "42.36 ms (+/- 1.27 ms)") 18 | assert.equal(collector.metrics["Change: Native"], "0.00%") 19 | assert.equal(collector.metrics["Change: Wasm"], "0.00%") 20 | }) 21 | 22 | it("Should iterate through metrics", async function () { 23 | var collector = new libCollector.Collector() 24 | await collector.CollectBaseCriterionWasmNative( 25 | "./criterion-files/import-block-with-ed25519-B-0003", 26 | ) 27 | await collector.CollectBranchCriterionWasmNative( 28 | "./criterion-files/import-block-with-ed25519-B-0003", 29 | ) 30 | 31 | assert((await collector.Report()).length > 100) 32 | }) 33 | 34 | it("Shoud parse json stdout", async function () { 35 | var collector = new libCollector.Collector() 36 | 37 | await collector.CollectBaseCustomRunner( 38 | `[{"name":"Import benchmark (random transfers, wasm)","raw_average":73361870,"average":70916310}]`, 39 | ) 40 | await collector.CollectBranchCustomRunner( 41 | `[{"name":"Import benchmark (random transfers, wasm)","raw_average":73361870,"average":60916310}]`, 42 | ) 43 | 44 | assert.equal(collector.metrics["Change"], "-14.10%") 45 | assert.equal(collector.metrics["Branch"], "60.92 ms") 46 | assert.equal(collector.metrics["Master"], "70.92 ms") 47 | }) 48 | }) 49 | --------------------------------------------------------------------------------