├── .codecov.yml ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug.yml │ ├── cli_feature.yml │ └── documentation.yml ├── dependabot.yml ├── stale.yml └── workflows │ ├── build_container.yml │ ├── cli.sh │ ├── cli_tutorial_check.yml │ ├── installation.yml │ ├── nightly_regression.yml │ ├── regression.yml │ ├── regressiontest_container.yml │ ├── shellcheck.yml │ ├── spack_regression_test.yml │ ├── style.yml │ └── urlchecker.yml ├── .gitignore ├── .gitlab └── ascent.yml ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── .shellcheckrc ├── .yamllint.yml ├── .zenodo.json ├── CHANGELOG.rst ├── CODE_OF_CONDUCT.md ├── LEGAL.txt ├── LICENSE ├── README.rst ├── SECURITY.md ├── aws_oddc └── sleep.yml ├── aws_tutorial ├── hello_world │ ├── hello.c │ ├── hello.cpp │ ├── hello.f90 │ ├── hello.yml │ └── multi_compiler_hello.yml ├── mpi_job_submission.yml ├── mpiproc.yml ├── openmp_example_custom_compiler.yml ├── openmp_hello.c ├── osu_bandwidth_test.yml ├── tensorflow.yml └── tensorflow_model.py ├── bash_completion.sh ├── bin └── buildtest ├── buildtest ├── __init__.py ├── builders │ ├── __init__.py │ ├── base.py │ ├── script.py │ └── spack.py ├── buildsystem │ ├── __init__.py │ ├── builders.py │ ├── checks.py │ └── parser.py ├── cli │ ├── __init__.py │ ├── build.py │ ├── buildspec.py │ ├── cd.py │ ├── cdash.py │ ├── clean.py │ ├── commands.py │ ├── compilers.py │ ├── config.py │ ├── debugreport.py │ ├── helpcolor.py │ ├── history.py │ ├── info.py │ ├── inspect.py │ ├── path.py │ ├── report.py │ ├── show.py │ └── stats.py ├── config.py ├── defaults.py ├── exceptions.py ├── executors │ ├── __init__.py │ ├── base.py │ ├── container.py │ ├── local.py │ ├── lsf.py │ ├── pbs.py │ ├── setup.py │ └── slurm.py ├── log.py ├── main.py ├── scheduler │ ├── __init__.py │ ├── detection.py │ ├── job.py │ ├── lsf.py │ ├── pbs.py │ └── slurm.py ├── schemas │ ├── __init__.py │ ├── defaults.py │ ├── definitions.schema.json │ ├── examples │ │ ├── global.schema.json │ │ │ ├── invalid │ │ │ │ ├── exceed_testname_length.yml │ │ │ │ ├── invalid_pattern.yml │ │ │ │ ├── maintainers_type_mismatch.yml │ │ │ │ └── unique_maintainers.yml │ │ │ └── valid │ │ │ │ └── examples.yml │ │ ├── script.schema.json │ │ │ ├── invalid │ │ │ │ ├── additionalProperties │ │ │ │ │ ├── executors.yml │ │ │ │ │ ├── file_count.yml │ │ │ │ │ ├── file_regex.yml │ │ │ │ │ ├── metrics.yml │ │ │ │ │ ├── regex.yml │ │ │ │ │ ├── status.yml │ │ │ │ │ └── test.yml │ │ │ │ ├── description_type.yml │ │ │ │ ├── empty_returncode.yml │ │ │ │ ├── empty_tags.yml │ │ │ │ ├── env_type.yml │ │ │ │ ├── executors_vars_type.yml │ │ │ │ ├── file_count │ │ │ │ │ ├── count_property_failure.yml │ │ │ │ │ ├── count_required.yml │ │ │ │ │ ├── depth_property_failure.yml │ │ │ │ │ ├── filetype_value.yml │ │ │ │ │ ├── minimum_file_traverse_limit.yml │ │ │ │ │ └── minimum_value_on_count.yml │ │ │ │ ├── file_exists_failure.yml │ │ │ │ ├── file_linecount_invalid.yml │ │ │ │ ├── file_regex_filename_property.yml │ │ │ │ ├── invalid_container_platform.yml │ │ │ │ ├── invalid_re_value.yml │ │ │ │ ├── invalid_value_mode.yml │ │ │ │ ├── linecount_stream_mismatch.yml │ │ │ │ ├── metrics │ │ │ │ │ ├── metrics_name.yml │ │ │ │ │ ├── metrics_type.yml │ │ │ │ │ ├── metrics_type_property.yml │ │ │ │ │ ├── missing_regex_in_metrics.yml │ │ │ │ │ └── regex_and_file_regex_in_metrics.yml │ │ │ │ ├── missing_regex_exp.yml │ │ │ │ ├── missing_run.yml │ │ │ │ ├── non_int_returncodes.yml │ │ │ │ ├── non_int_returncodes_list.yml │ │ │ │ ├── regex_stream_value.yml │ │ │ │ ├── returncode_type.yml │ │ │ │ ├── runtime_min.yml │ │ │ │ ├── runtime_min_greater_than_0.yml │ │ │ │ ├── shebang_type.yml │ │ │ │ ├── shell_type.yml │ │ │ │ ├── skip_value.yml │ │ │ │ ├── slurm_job_state.yml │ │ │ │ ├── summary_type.yml │ │ │ │ ├── tags_value.yml │ │ │ │ └── test_name.yml │ │ │ └── valid │ │ │ │ ├── assert_eq.yml │ │ │ │ ├── assert_ge.yml │ │ │ │ ├── assert_gt.yml │ │ │ │ ├── assert_le.yml │ │ │ │ ├── assert_lt.yml │ │ │ │ ├── assert_ne.yml │ │ │ │ ├── assert_range.yml │ │ │ │ ├── bind_mounts.yml │ │ │ │ ├── contains.yml │ │ │ │ ├── environment.yml │ │ │ │ ├── executor_scheduler.yml │ │ │ │ ├── executors_metrics.yml │ │ │ │ ├── executors_var_env_declaration.yml │ │ │ │ ├── exists.yml │ │ │ │ ├── explicit_state.yml │ │ │ │ ├── file_and_dir_check.yml │ │ │ │ ├── file_count.yml │ │ │ │ ├── file_count_file_traverse_limit.yml │ │ │ │ ├── file_count_filetype.yml │ │ │ │ ├── file_count_pattern.yml │ │ │ │ ├── file_linecount.yml │ │ │ │ ├── hello_world.yml │ │ │ │ ├── is_symlink.yml │ │ │ │ ├── job_dep_ex1.yml │ │ │ │ ├── job_dep_ex2.yml │ │ │ │ ├── job_dep_ex3.yml │ │ │ │ ├── job_dep_ex4.yml │ │ │ │ ├── job_submission.yml │ │ │ │ ├── linecount.yml │ │ │ │ ├── metrics_regex.yml │ │ │ │ ├── metrics_with_regex_type.yml │ │ │ │ ├── mode.yml │ │ │ │ ├── pass_returncode.yml │ │ │ │ ├── post_run.yml │ │ │ │ ├── python-shell.yml │ │ │ │ ├── regex_on_filename.yml │ │ │ │ ├── run_commands.yml │ │ │ │ ├── run_example.yml │ │ │ │ ├── runtime_status_test.yml │ │ │ │ ├── shebang.yml │ │ │ │ ├── shell_examples.yml │ │ │ │ ├── skip_tests.yml │ │ │ │ ├── specify_regex_type.yml │ │ │ │ ├── status_by_executors.yml │ │ │ │ ├── summary_example.yml │ │ │ │ ├── tag_example.yml │ │ │ │ ├── variable_declaration.yml │ │ │ │ └── zsh_tcsh_and_csh_shell.yml │ │ ├── settings.schema.json │ │ │ ├── invalid │ │ │ │ └── invalid_executor_names.yml │ │ │ └── valid │ │ │ │ ├── combined_executor.yml │ │ │ │ ├── local-executor.yml │ │ │ │ ├── lsf-example.yml │ │ │ │ ├── pbs-example.yml │ │ │ │ ├── slurm-example.yml │ │ │ │ └── torque-example.yml │ │ ├── spack.schema.json │ │ │ ├── invalid │ │ │ │ ├── additionalProperties │ │ │ │ │ ├── spack.yml │ │ │ │ │ └── spack_test.yml │ │ │ │ ├── mirror_type.yml │ │ │ │ ├── spack_env_deactivate.yml │ │ │ │ ├── spack_load_invalid_example.yml │ │ │ │ ├── spack_test_run_invalid_spec.yml │ │ │ │ └── specs_list_check_string.yml │ │ │ └── valid │ │ │ │ ├── env_activate.yml │ │ │ │ ├── env_concretize.yml │ │ │ │ ├── env_create_directory.yml │ │ │ │ ├── env_create_manifest.yml │ │ │ │ ├── env_create_name.yml │ │ │ │ ├── env_mirror.yml │ │ │ │ ├── install_without_env.yml │ │ │ │ ├── pre_post.yml │ │ │ │ ├── remove_spack_env_automatically.yml │ │ │ │ ├── remove_spack_environment.yml │ │ │ │ ├── sbatch_multi_executors.yml │ │ │ │ ├── sbatch_with_spack.yml │ │ │ │ ├── skip_test.yml │ │ │ │ ├── spack_env_deactivate.yml │ │ │ │ ├── spack_load_valid_example.yml │ │ │ │ ├── var_and_env_declaration.yml │ │ │ │ └── vars_multi_executors.yml │ │ └── special_invalid_buildspecs │ │ │ └── missing_type.yml │ ├── global.schema.json │ ├── script.schema.json │ ├── settings.schema.json │ ├── spack.schema.json │ └── utils.py ├── settings │ ├── aws.yml │ ├── aws_oddc_pbs.yml │ ├── config.yml │ ├── container_executor.yml │ └── spack_container.yml ├── system.py ├── tools │ ├── __init__.py │ ├── cpu.py │ ├── editor.py │ ├── modules.py │ ├── stylecheck.py │ ├── tutorialexamples.py │ └── unittests.py └── utils │ ├── __init__.py │ ├── command.py │ ├── file.py │ ├── print.py │ ├── shell.py │ ├── table.py │ ├── timer.py │ └── tools.py ├── docs ├── Makefile ├── __init__.py ├── _static │ ├── CDASH.png │ ├── DiscoverBuildspecs.jpg │ ├── GeneralPipeline.png │ ├── ParserDiagram.png │ └── buildspec-structure.png ├── _templates │ └── autoapi │ │ └── index.rst ├── api.rst ├── aws_examples │ ├── compiler_list_yaml.txt │ ├── container_executor_build.txt │ ├── container_executor_inspect.txt │ ├── container_executor_list.txt │ ├── docker_helloworld_build.txt │ ├── docker_helloworld_inspect.txt │ ├── hello_build.txt │ ├── hello_inspect.txt │ ├── mpi_job_submission_build.txt │ ├── mpiproc_build.txt │ ├── mpiproc_inspect.txt │ ├── multi_compiler_hello_build.txt │ ├── multi_compiler_hello_inspect.txt │ ├── openmp_example_build.txt │ ├── openmp_example_inspect.txt │ ├── osu_bandwidth_test_build.txt │ ├── osu_bandwidth_test_inspect.txt │ ├── singularity_helloworld_build.txt │ └── singularity_helloworld_inspect.txt ├── batch_support.rst ├── builder.rst ├── buildspec_tutorial.rst ├── buildspecs │ ├── aws.rst │ ├── buildspec_overview.rst │ ├── e4s_testsuite.rst │ └── spack.rst ├── buildtest_perlmutter.rst ├── buildtest_site.rst ├── buildtest_tutorial_examples │ └── spack │ │ ├── build │ │ ├── clone_spack.txt │ │ ├── e4s_testsuite_mpich.txt │ │ ├── env_create_directory.txt │ │ ├── env_create_manifest.txt │ │ ├── env_install.txt │ │ ├── install_specs.txt │ │ ├── mirror_example.txt │ │ ├── pre_post_cmds.txt │ │ ├── remove_environment_example.txt │ │ ├── spack_env_deactivate.txt │ │ ├── spack_load.txt │ │ ├── spack_sbatch.txt │ │ ├── spack_test.txt │ │ └── spack_test_specs.txt │ │ └── inspect │ │ ├── clone_spack.txt │ │ ├── e4s_testsuite_mpich.txt │ │ ├── env_create_directory.txt │ │ ├── env_create_manifest.txt │ │ ├── env_install.txt │ │ ├── install_specs.txt │ │ ├── mirror_example.txt │ │ ├── pre_post_cmds.txt │ │ ├── remove_environment_example.txt │ │ ├── spack_env_deactivate.txt │ │ ├── spack_load.txt │ │ ├── spack_sbatch.txt │ │ ├── spack_test.txt │ │ └── spack_test_specs.txt ├── command.rst ├── command_line_tutorial.rst ├── conf.py ├── conferences.rst ├── configuring_buildtest.rst ├── configuring_buildtest │ ├── cli_to_buildtest_configuration.rst │ ├── compilers.rst │ ├── executors.rst │ ├── overview.rst │ └── site_examples.rst ├── contributing.rst ├── contributing │ ├── build_documentation.rst │ ├── buildtest_branch_settings.png │ ├── buildtest_merge_options.png │ ├── code_contribution_guide.rst │ ├── coverage_locally.png │ ├── github_integration.rst │ ├── maintainer_guide.rst │ ├── new_maintainer_checklist.rst │ ├── regression_testing.rst │ └── schema.rst ├── features.rst ├── general_tests ├── getting_started.rst ├── gettingstarted │ ├── buildingtest.rst │ ├── buildspecs_interface.rst │ └── query_test_report.rst ├── index.rst ├── installing_buildtest.rst ├── make.bat ├── quick_start.rst ├── requirements.txt ├── scripting_examples │ ├── ex1.py │ ├── ex1.py.out │ ├── ex2.py │ └── ex2.py.out ├── tests ├── troubleshooting.rst ├── tutorials ├── what_is_buildtest.rst ├── writing_buildspecs.rst └── writing_buildspecs │ ├── comparison_operators.rst │ ├── compilation.rst │ ├── containers.rst │ ├── customize_shell.rst │ ├── global.rst │ ├── metrics.rst │ ├── multi_executor.rst │ ├── status_check.rst │ └── test_dependency.rst ├── examples └── spack │ ├── clone_spack.yml │ ├── e4s_testsuite_mpich.yml │ ├── env_create_directory.yml │ ├── env_create_manifest.yml │ ├── env_install.yml │ ├── example │ └── spack.yaml │ ├── install_specs.yml │ ├── mirror_example.yml │ ├── pre_post_cmds.yml │ ├── remove_environment_example.yml │ ├── spack_env_deactivate.yml │ ├── spack_load.yml │ ├── spack_multiple_executor_sbatch.yml │ ├── spack_sbatch.yml │ ├── spack_test.yml │ └── spack_test_specs.yml ├── general_tests ├── configuration │ ├── disk_usage.yml │ ├── kernel_state.yml │ ├── systemd-default-target.yml │ └── ulimits.yml ├── containers │ └── singularity │ │ ├── build.yml │ │ ├── inspect.yml │ │ ├── pull.yml │ │ └── run.yml └── sched │ ├── lsf │ ├── bhosts.yml │ ├── bmgroups.yml │ ├── bqueues.yml │ ├── bugroup.yml │ └── lsinfo.yml │ ├── pbs │ └── hostname.yml │ └── slurm │ ├── sacctmgr.yml │ ├── scontrol.yml │ ├── sinfo.yml │ └── squeue.yml ├── logos ├── BuildTest_Primary_Center_3x1.png ├── BuildTest_Primary_Center_4x3.png ├── BuildTest_Primary_Left_3x1.png ├── BuildTest_Primary_Right_3x1.png └── buildtest_secondary.jpg ├── perlmutter_tutorial ├── ex1 │ ├── .solution.txt │ └── module_version.yml ├── ex2 │ └── .solution.sh ├── ex3 │ └── .solution.sh ├── ex4 │ ├── .solution.txt │ └── stream.yml └── ex5 │ ├── .solution.txt │ └── hostname.yml ├── pyproject.toml ├── requirements.txt ├── scripts ├── pbs │ ├── Dockerfile │ └── setup.sh ├── run_scripts.sh └── spack_container │ ├── Dockerfile │ ├── compilers.yaml │ ├── doc-examples.py │ ├── modules.yaml │ ├── regtest-tutorial.sh │ ├── run-tutorial-examples.sh │ ├── setup.sh │ └── spack_setup.sh ├── setup.csh ├── setup.sh ├── tests ├── __init__.py ├── builders │ ├── __init__.py │ ├── assert_eq.yml │ ├── assert_ge.yml │ ├── assert_gt.yml │ ├── assert_le.yml │ ├── assert_lt.yml │ ├── assert_ne.yml │ ├── assert_range.yml │ ├── contains.yml │ ├── exists.yml │ ├── file_and_dir_check.yml │ ├── file_count.yml │ ├── file_count_file_traverse_limit.yml │ ├── file_count_filetype.yml │ ├── file_count_pattern.yml │ ├── file_linecount.yml │ ├── file_linecount_failure.yml │ ├── file_linecount_invalid.yml │ ├── is_symlink.yml │ ├── linecount.yml │ ├── metrics_file_regex_with_invalid_linenum.yml │ ├── metrics_file_regex_with_linenum.yml │ ├── metrics_regex_with_invalid_linenum.yml │ ├── metrics_regex_with_linenum.yml │ ├── metrics_with_regex_type.yml │ ├── post_run.yml │ ├── regex_on_filename.yml │ ├── regex_on_invalids.yml │ ├── runtime_status_test.yml │ ├── specify_regex_type.yml │ ├── status_regex.yml │ ├── stream_example.yml │ ├── test_builders.py │ └── test_containers.py ├── buildsystem │ ├── __init__.py │ ├── invalid_buildspecs │ │ ├── invalid_executor.yml │ │ ├── invalid_type.yml │ │ ├── missing_executor.yml │ │ └── missing_type.yml │ ├── test_base.py │ ├── test_spack.py │ ├── valid_builds │ │ └── sched_directives.yml │ └── valid_buildspecs │ │ ├── environment.yml │ │ ├── python-shell.yml │ │ ├── shell_examples.yml │ │ └── slurm.yml ├── cli │ ├── __init__.py │ ├── cdash_examples │ │ ├── invalid_project.yml │ │ └── invalid_url.yml │ ├── configuration │ │ ├── container_executors.yml │ │ ├── file_traversal_example.yml │ │ └── invalid_executors.yml │ ├── test_argparse.py │ ├── test_build.py │ ├── test_buildspec.py │ ├── test_cdash.py │ ├── test_commands.py │ ├── test_compilers.py │ ├── test_compilers │ │ ├── invalid_moduletool.yml │ │ └── missing_compiler_find.yml │ ├── test_config.py │ ├── test_debugreport.py │ ├── test_helpcolor.py │ ├── test_history.py │ ├── test_info.py │ ├── test_inspect.py │ ├── test_path.py │ ├── test_report.py │ └── test_show.py ├── examples │ ├── pbs │ │ ├── hold.yml │ │ ├── job_dependency.yml │ │ ├── multiple_tests.yml │ │ ├── pbs_job_state.yml │ │ └── sleep.yml │ ├── perlmutter │ │ ├── hold_job.yml │ │ └── hostname.yml │ ├── summit │ │ ├── hold_job.yml │ │ ├── hostname.yml │ │ └── lsf_job_state.yml │ └── torque │ │ ├── sleep.yml │ │ └── sleep_cancel.yml ├── executors │ ├── __init__.py │ └── test_base.py ├── schema_tests │ ├── __init__.py │ ├── test_global.py │ ├── test_schemas.py │ └── test_settings.py ├── schemas │ ├── __init__.py │ └── test_utils.py ├── settings │ ├── invalid │ │ └── torque_invalid_executor.yml │ ├── nersc.yml │ ├── pbs.yml │ ├── summit.yml │ └── torque.yml ├── sitecustomize.py ├── test_exceptions.py ├── test_log.py ├── test_pbs.py ├── test_stats.py ├── test_summit.py ├── test_torque.py ├── tools │ ├── __init__.py │ ├── test_editor.py │ └── test_stylecheck.py └── utils │ ├── __init__.py │ ├── test_command.py │ ├── test_file.py │ └── test_shell.py ├── tox.ini └── tutorials ├── add_numbers.yml ├── burstbuffer_datawarp_executors.yml ├── compilation ├── compiler_exclude.yml ├── hello.c ├── hello.cpp ├── hello.f ├── hello_world_compilation.yml └── stream.yml ├── containers ├── bind_mounts.yml ├── container_executor │ ├── python_container.yml │ └── ubuntu.yml ├── hello_world.yml ├── hello_world_singularity.yml ├── run_commands.yml ├── run_script.yml └── script.py ├── csh_shell_examples.yml ├── environment.yml ├── gcc_version.yml ├── hello.py ├── hello_world.yml ├── invalid_buildspec_section.yml ├── invalid_executor.yml ├── invalid_tags.yml ├── job_dependency ├── ex1.yml ├── ex2.yml ├── ex3.yml └── ex4.yml ├── maintainers_example.yml ├── metrics ├── invalid_metric_name.yml ├── invalid_metrics.yml ├── metrics_file_regex_invalid_file.yml ├── metrics_file_regex_with_invalid_linenum.yml ├── metrics_file_regex_with_linenum.yml ├── metrics_regex.yml ├── metrics_regex_with_invalid_linenum.yml ├── metrics_regex_with_linenum.yml ├── metrics_with_regex_type.yml └── missing_required_in_metrics.yml ├── multi_executors ├── executor_regex_script.yml ├── executor_scheduler.yml ├── executors_var_env_declaration.yml └── status_by_executors.yml ├── perf_checks ├── assert_eq.yml ├── assert_eq_exceptions.yml ├── assert_ge.yml ├── assert_gt.yml ├── assert_le.yml ├── assert_lt.yml ├── assert_ne.yml ├── assert_range.yml └── contains.yml ├── post_run.yml ├── python-hello.yml ├── python-shell.yml ├── shebang.yml ├── shell_examples.yml ├── skip_buildspec.yml ├── skip_tests.yml ├── sleep.yml ├── strict_example.yml ├── summary_example.yml ├── tags_example.yml ├── test_status ├── exists.yml ├── explicit_state.yml ├── file_and_dir_check.yml ├── file_count.yml ├── file_count_file_traverse_limit.yml ├── file_count_filetype.yml ├── file_count_pattern.yml ├── file_exists_exception.yml ├── file_exists_with_number.yml ├── file_linecount.yml ├── file_linecount_failure.yml ├── file_linecount_invalid.yml ├── is_symlink.yml ├── linecount.yml ├── mode.yml ├── pass_returncode.yml ├── regex_on_filename.yml ├── runtime_status_test.yml ├── specify_regex_type.yml └── status_regex.yml └── vars.yml /.codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: yes 3 | notify: 4 | after_n_builds: 2 5 | wait_for_ci: yes 6 | comment: 7 | layout: "reach, diff, flags, files" 8 | behavior: default 9 | require_changes: true 10 | github_checks: 11 | annotations: false 12 | coverage: 13 | precision: 3 14 | round: nearest 15 | range: "10...100" 16 | status: 17 | patch: off 18 | project: 19 | default: yes 20 | target: auto 21 | threshold: null 22 | base: pr 23 | flags: null 24 | if_ci_failed: error 25 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: shahzebsiddiqui 4 | open_collective: buildtest 5 | # patreon: # Replace with a single Patreon username 6 | # open_collective: # Replace with a single Open Collective username 7 | # ko_fi: # Replace with a single Ko-fi username 8 | # tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 9 | # community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 10 | # liberapay: # Replace with a single Liberapay username 11 | # issuehunt: # Replace with a single IssueHunt username 12 | # otechie: # Replace with a single Otechie username 13 | # lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.yml: -------------------------------------------------------------------------------- 1 | name: Issue with Documentation 2 | description: Issue with buildtest documentation 3 | title: "[DOCS]: " 4 | labels: [documentation, triage] 5 | assignees: 6 | - shahzebsiddiqui 7 | body: 8 | - type: input 9 | id: link 10 | attributes: 11 | label: Documentation Link 12 | description: Specify link to url that needs attention 13 | placeholder: ex. https://buildtest.readthedocs.io/ 14 | validations: 15 | required: true 16 | - type: textarea 17 | id: description 18 | attributes: 19 | label: Issue Description 20 | description: Please provide text and explanation of the issue 21 | validations: 22 | required: true 23 | - type: checkboxes 24 | id: slackterms 25 | attributes: 26 | label: Post question in Slack 27 | options: 28 | - label: I agree that I posted my question in [slack](https://hpcbuildtest.slack.com/) before creating this issue 29 | required: true 30 | - type: checkboxes 31 | id: issueterms 32 | attributes: 33 | label: Is there an existing issue 34 | options: 35 | - label: I confirm there is no existing [issue](https://github.com/buildtesters/buildtest/issues) for this issue 36 | required: true 37 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | # check weekly on Sunday at 1am UTC 7 | schedule: 8 | interval: "weekly" 9 | day: "sunday" 10 | time: "01:00" 11 | labels: 12 | - "github-action dependencies" 13 | 14 | # Maintain dependencies for pip 15 | - package-ecosystem: "pip" 16 | directory: "/" 17 | # check weekly at 2am UTC 18 | schedule: 19 | interval: "weekly" 20 | day: "sunday" 21 | time: "02:00" 22 | labels: 23 | - "pip dependencies" 24 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 90 3 | # Label to use when marking an issue as stale 4 | staleLabel: wontfix 5 | # Comment to post when marking an issue as stale. Set to `false` to disable 6 | markComment: > 7 | This issue has been automatically marked as stale because it has not had 8 | recent activity. It will be closed if no further activity occurs. Thank you 9 | for your contributions. 10 | # Comment to post when closing a stale issue. Set to `false` to disable 11 | closeComment: false 12 | -------------------------------------------------------------------------------- /.github/workflows/build_container.yml: -------------------------------------------------------------------------------- 1 | name: Build Container 2 | 3 | on: 4 | pull_request: 5 | branches: [devel] 6 | 7 | jobs: 8 | build_container: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | contents: read 12 | packages: write 13 | 14 | steps: 15 | - name: Check out the repo 16 | uses: actions/checkout@v4 17 | 18 | - name: Set up QEMU 19 | uses: docker/setup-qemu-action@v3 20 | 21 | - name: Set up Docker Buildx 22 | uses: docker/setup-buildx-action@v3 23 | 24 | - name: Log in to GitHub Container Registry 25 | uses: docker/login-action@v3 26 | with: 27 | registry: ghcr.io 28 | username: ${{ github.actor }} 29 | password: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | - name: Build and push Docker image 32 | uses: docker/build-push-action@v6 33 | with: 34 | tags: ghcr.io/buildtesters/buildtest_spack:spack-sc23 35 | context: scripts/spack_container 36 | push: true 37 | -------------------------------------------------------------------------------- /.github/workflows/cli_tutorial_check.yml: -------------------------------------------------------------------------------- 1 | name: regressiontest 2 | 3 | on: 4 | pull_request: 5 | branches: [devel] 6 | 7 | jobs: 8 | 9 | commandline_tutorial_check: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Setup Python 16 | uses: actions/setup-python@v5.0.0 17 | with: 18 | python-version: 3.9 19 | 20 | - name: Buildtest CLI Check 21 | shell: bash 22 | run: | 23 | sudo apt-get install -y csh tcsh zsh 24 | source setup.sh 25 | # tutorial check 26 | bash .github/workflows/cli.sh 27 | -------------------------------------------------------------------------------- /.github/workflows/shellcheck.yml: -------------------------------------------------------------------------------- 1 | name: Shell Check 2 | 3 | on: 4 | push: 5 | branches: 6 | - devel 7 | pull_request: 8 | branches: 9 | - devel 10 | 11 | jobs: 12 | style_checks: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout Repository 16 | uses: actions/checkout@v4 17 | - name: Run Shell Check 18 | run: | 19 | sudo apt-get install -y shellcheck 20 | shellcheck --version 21 | shellcheck -x setup.sh scripts/spack_container/*.sh 22 | -------------------------------------------------------------------------------- /.github/workflows/spack_regression_test.yml: -------------------------------------------------------------------------------- 1 | name: Regression Test with Spack Support 2 | 3 | on: 4 | pull_request: 5 | branches: [devel] 6 | paths: 7 | - 'buildtest/**' 8 | - 'tests/**' 9 | - '.github/workflows/spack_regression_test.yml' 10 | - 'requirements.txt' 11 | - 'pyproject.toml' 12 | 13 | jobs: 14 | 15 | buildtest_regtest_with_spack: 16 | runs-on: ubuntu-22.04 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Setup Python 20 | uses: actions/setup-python@v5.0.0 21 | - name: Set up Spack 22 | uses: spack/setup-spack@v2 23 | - name: Spack Regression Test 24 | shell: spack-bash {0} 25 | run: | 26 | spack --version 27 | whoami 28 | spack find 29 | source setup.sh 30 | pip install pytest coverage 31 | python $BUILDTEST_ROOT/buildtest/tools/unittests.py -c -p "-m spack" 32 | returncode=$? 33 | if [ $returncode != 0 ]; then exit $returncode; fi 34 | - name: Upload coverage to Codecov 35 | uses: codecov/codecov-action@v4 36 | with: 37 | verbose: true 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # auto generated files in $BUILDTEST_ROOT/var 7 | var/ 8 | 9 | # generated docs from sphinx-autoapi 10 | docs/api 11 | 12 | # xml files generated for buildtest cdash 13 | *.xml 14 | 15 | # logfile 16 | *.log 17 | 18 | # C extensions 19 | *.so 20 | 21 | # Installer logs 22 | pip-log.txt 23 | pip-delete-this-directory.txt 24 | 25 | # Unit test / coverage reports 26 | htmlcov/ 27 | .coverage 28 | .coverage.* 29 | .cache 30 | coverage.xml 31 | *.cover 32 | .pytest_cache/ 33 | 34 | # Sphinx documentation 35 | docs/_build/ 36 | 37 | # Environments 38 | .env 39 | .venv 40 | env/ 41 | venv/ 42 | ENV/ 43 | env.bak/ 44 | 45 | # ignore .packages directory 46 | .packages/ 47 | Pipfile 48 | Pipfile.lock 49 | coverage.json 50 | 51 | # ignore .idea 52 | .idea/ -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | ci: 2 | autoupdate_branch: '' 3 | autoupdate_commit_msg: '[pre-commit.ci] pre-commit weekly autoupdate' 4 | autoupdate_schedule: weekly 5 | 6 | repos: 7 | - repo: https://github.com/pycqa/isort 8 | rev: 5.13.2 9 | hooks: 10 | - id: isort 11 | args: ["--profile", "black", "--filter-files", "buildtest", "tests", "docs"] 12 | 13 | - repo: https://github.com/psf/black-pre-commit-mirror 14 | rev: '24.3.0' 15 | hooks: 16 | - id: black 17 | language_version: python3 18 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | 3 | version: 2 4 | 5 | sphinx: 6 | builder: html 7 | configuration: docs/conf.py 8 | fail_on_warning: true 9 | 10 | build: 11 | os: "ubuntu-22.04" 12 | tools: 13 | python: "3.10" 14 | apt_packages: 15 | - iputils-ping 16 | - build-essential 17 | - curl 18 | - gfortran 19 | - csh 20 | - zsh 21 | - tcsh 22 | 23 | python: 24 | install: 25 | - method: pip 26 | path: . 27 | extra_requirements: 28 | - docs 29 | -------------------------------------------------------------------------------- /.shellcheckrc: -------------------------------------------------------------------------------- 1 | disable=SC1091,SC1130,SC2016,SC2296 2 | -------------------------------------------------------------------------------- /.yamllint.yml: -------------------------------------------------------------------------------- 1 | yaml-files: 2 | - '*.yaml' 3 | - '*.yml' 4 | - '.yamllint' 5 | 6 | rules: 7 | braces: enable 8 | brackets: enable 9 | colons: enable 10 | commas: enable 11 | comments: disable 12 | comments-indentation: disable 13 | document-end: disable 14 | document-start: disable 15 | empty-lines: enable 16 | empty-values: disable 17 | float-values: disable 18 | hyphens: enable 19 | indentation: 20 | indent-sequences: whatever 21 | key-duplicates: enable 22 | key-ordering: disable 23 | line-length: 24 | max: 80 25 | level: warning 26 | new-line-at-end-of-file: enable 27 | new-lines: enable 28 | octal-values: disable 29 | quoted-strings: disable 30 | trailing-spaces: disable 31 | truthy: disable 32 | 33 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "creators": [ 3 | { 4 | "name": "Siddiqui, Shahzeb", 5 | "orcid": "0000-0002-2342-6974" 6 | }, 7 | { 8 | "affiliation": "Stanford University Research Computing Center, Stanford University, Stanford CA USA", 9 | "name": "Sochat, Vanessa", 10 | "orcid": "0000-0002-4387-3819" 11 | }, 12 | ], 13 | "keywords": [ 14 | "build testing", 15 | "high performance computing", 16 | "testing", 17 | ], 18 | "access_right": "open", 19 | "license": "MIT", 20 | "upload_type": "software" 21 | } 22 | -------------------------------------------------------------------------------- /LEGAL.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021-2025, The Regents of the University of California, 2 | through Lawrence Berkeley National Laboratory (subject to receipt of 3 | any required approvals from the U.S. Dept. of Energy), Shahzeb Siddiqui, 4 | and Vanessa Sochat. All rights reserved. 5 | 6 | If you have questions about your rights to use or distribute this software, 7 | please contact Berkeley Lab's Intellectual Property Office at 8 | IPO@lbl.gov. 9 | 10 | NOTICE. This Software was developed under funding from the U.S. Department 11 | of Energy and the U.S. Government consequently retains certain rights. As 12 | such, the U.S. Government has been granted for itself and others acting on 13 | its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the 14 | Software to reproduce, distribute copies to the public, prepare derivative 15 | works, and perform publicly and display publicly, and to permit others to do so. 16 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | We provide security updates for the following versions. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | devel | :white_check_mark: | 10 | | 1.x | :white_check_mark: | 11 | | < 1.0 | :x: | 12 | 13 | ## Reporting a Vulnerability 14 | 15 | To report a vulnerability or other security issue, email [shahzebmsiddiqui@gmail.com](mailto:shahzebmsiddiqui@gmail.com). 16 | -------------------------------------------------------------------------------- /aws_oddc/sleep.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname_test: 3 | type: script 4 | executor: generic.torque.lbl 5 | description: run sleep for 5 seconds 6 | pbs: ["-l nodes=1"] 7 | run: | 8 | sleep 5 9 | -------------------------------------------------------------------------------- /aws_tutorial/hello_world/hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | printf("Hello, World in C\n"); 5 | return 0; 6 | } 7 | -------------------------------------------------------------------------------- /aws_tutorial/hello_world/hello.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | std::cout << "Hello, World in C++" << std::endl; 5 | return 0; 6 | } 7 | -------------------------------------------------------------------------------- /aws_tutorial/hello_world/hello.f90: -------------------------------------------------------------------------------- 1 | PROGRAM HelloWorld 2 | PRINT *, 'Hello, World in Fortran' 3 | END PROGRAM HelloWorld 4 | -------------------------------------------------------------------------------- /aws_tutorial/hello_world/hello.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Hello world compilation in C 6 | run: | 7 | gcc hello.c -o hello 8 | ./hello 9 | -------------------------------------------------------------------------------- /aws_tutorial/hello_world/multi_compiler_hello.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_multi_compiler: 3 | type: script 4 | executor: generic.local.bash 5 | description: Hello world compilation in C, C++ and Fortran with multiple compilers 6 | compilers: 7 | name: ['gcc'] 8 | run: | 9 | $BUILDTEST_CC hello.c -o hello_c 10 | $BUILDTEST_CXX hello.cpp -o hello_cpp 11 | $BUILDTEST_FC hello.f90 -o hello_f90 12 | ./hello_c 13 | ./hello_cpp 14 | ./hello_f90 15 | -------------------------------------------------------------------------------- /aws_tutorial/mpi_job_submission.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | mpi_job_submission: 3 | type: script 4 | executor: generic.torque.e4spro-cluster 5 | description: Run MPI Proc Name test 6 | pbs: ["-l nodes=1:ppn=2,walltime=1:00"] 7 | run: | 8 | cd $HOME/examples/mpi-procname 9 | mpicc -o mpiprocname mpiprocname.c 10 | mpirun -np 2 ./mpiprocname 11 | rm mpiprocname -------------------------------------------------------------------------------- /aws_tutorial/mpiproc.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | mpiprocname: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run MPI Proc Name test 6 | run: | 7 | cd $HOME/examples/mpi-procname 8 | mpicc -o mpiprocname mpiprocname.c 9 | mpirun -np 8 ./mpiprocname 10 | rm mpiprocname -------------------------------------------------------------------------------- /aws_tutorial/openmp_example_custom_compiler.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_openmp_custom_compiler: 3 | type: script 4 | executor: 'generic.local.bash' 5 | description: Hello World OpenMP example with custom compiler settings 6 | compilers: 7 | name: ["gcc"] 8 | config: 9 | gcc_12.3.0: 10 | env: 11 | OMP_NUM_THREADS: "2" 12 | cflags: "-O1 -fopenmp" 13 | gcc_11.4.0: 14 | env: 15 | OMP_NUM_THREADS: "6" 16 | cflags: "-O2 -fopenmp" 17 | run: | 18 | $BUILDTEST_CC $BUILDTEST_CFLAGS -o openmp_hello openmp_hello.c 19 | ./openmp_hello 20 | -------------------------------------------------------------------------------- /aws_tutorial/openmp_hello.c: -------------------------------------------------------------------------------- 1 | 2 | // OpenMP program to print Hello World 3 | // using C language 4 | 5 | // OpenMP header 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | int main(int argc, char* argv[]) 12 | { 13 | 14 | // Beginning of parallel region 15 | #pragma omp parallel 16 | { 17 | 18 | printf("Hello World... from thread = %d\n", 19 | omp_get_thread_num()); 20 | } 21 | // Ending of parallel region 22 | } 23 | -------------------------------------------------------------------------------- /aws_tutorial/osu_bandwidth_test.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | osu_bandwidth: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run OSU Bandwidth Test 6 | run: mpirun -np 2 osu_bw 7 | 8 | osu_bandwidth_perf: 9 | type: script 10 | executor: generic.local.bash 11 | description: Run OSU Bandwidth Performance Test 12 | run: mpirun -np 2 osu_bw 13 | metrics: 14 | osu_bw: 15 | type: float 16 | regex: 17 | exp: '^16384\s+([\d.]+)$' 18 | stream: stdout 19 | item: 1 20 | status: 21 | assert_ge: 22 | comparisons: 23 | - name: osu_bw 24 | ref: 10000 25 | -------------------------------------------------------------------------------- /aws_tutorial/tensorflow.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | run_tensorflow_model: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run TensorFlow Model 6 | run: python3 tensorflow_model.py -------------------------------------------------------------------------------- /bin/buildtest: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # This file is bilingual. The following shell code finds our preferred python. 4 | # Following line is a shell no-op, and starts a multi-line Python comment. 5 | # See https://stackoverflow.com/a/47886254 6 | 7 | """:" 8 | # preferred pythons for running buildtest 9 | PREFERRED_PYTHONS="python3 python" 10 | 11 | # Find a suitable python interpreter (adapt for your specific needs) 12 | for cmd in "${BUILDTEST_PYTHON:-}" ${PREFERRED_PYTHONS} ; do 13 | if command -v > /dev/null "$cmd"; then 14 | export BUILDTEST_PYTHON="$(command -v "$cmd")" 15 | exec "${BUILDTEST_PYTHON}" "$0" "$@" 16 | fi 17 | done 18 | 19 | echo "==> Error: buildtest could not find a python interpreter!" >&2 20 | exit 1 21 | ":""" 22 | 23 | import os 24 | import sys 25 | 26 | buildtest_file=os.path.realpath(os.path.expanduser(__file__)) 27 | prefix=os.path.dirname(os.path.dirname(buildtest_file)) 28 | 29 | sys.path.insert(0, prefix) 30 | 31 | if sys.version_info[:3] < (3, 9, 0): 32 | sys.exit("buildtest requires Python 3.9.0 or higher.") 33 | 34 | from buildtest.main import main 35 | 36 | if __name__ == "__main__": 37 | 38 | sys.exit(main()) 39 | -------------------------------------------------------------------------------- /buildtest/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "2.1" 2 | BUILDTEST_VERSION = __version__ 3 | BUILDTEST_COPYRIGHT = "Copyright (c) 2021-2024, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy), Shahzeb Siddiqui, and Vanessa Sochat. All rights reserved." 4 | -------------------------------------------------------------------------------- /buildtest/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/builders/__init__.py -------------------------------------------------------------------------------- /buildtest/buildsystem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/buildsystem/__init__.py -------------------------------------------------------------------------------- /buildtest/cli/commands.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli import BuildTestParser 2 | 3 | 4 | def list_buildtest_commands(with_aliases=None): 5 | """This method implements command ``buildtest commands`` which shows a list of buildtest commands 6 | 7 | Args: 8 | with_aliases (bool): Return a list of buildtest commands with aliases 9 | """ 10 | 11 | cmds = BuildTestParser() 12 | subcmds = sorted(cmds.get_subcommands()) 13 | 14 | # if --with-aliases we will show all available choices for subcommands including aliases 15 | if with_aliases: 16 | subparser = cmds.get_subparsers() 17 | subcmds = sorted(list(subparser.choices.keys())) 18 | 19 | for field in sorted(subcmds): 20 | print(field) 21 | -------------------------------------------------------------------------------- /buildtest/cli/helpcolor.py: -------------------------------------------------------------------------------- 1 | from rich.color import ANSI_COLOR_NAMES 2 | from rich.table import Column, Table 3 | 4 | from buildtest.defaults import console 5 | 6 | 7 | def print_available_colors(color_names=ANSI_COLOR_NAMES): 8 | """Print the available color options in a table format on background of the color option.""" 9 | table = Table( 10 | Column("Number", overflow="fold"), 11 | Column("Color", overflow="fold"), 12 | header_style="bold", 13 | title="Available Colors", 14 | ) 15 | for color, number in color_names.items(): 16 | table.add_row(f"[black]{number}", f"[black]{color}", style="on " + color) 17 | console.print(table) 18 | -------------------------------------------------------------------------------- /buildtest/executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/executors/__init__.py -------------------------------------------------------------------------------- /buildtest/executors/container.py: -------------------------------------------------------------------------------- 1 | from buildtest.executors.local import LocalExecutor 2 | 3 | 4 | class ContainerExecutor(LocalExecutor): 5 | type = "container" 6 | -------------------------------------------------------------------------------- /buildtest/scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/scheduler/__init__.py -------------------------------------------------------------------------------- /buildtest/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/schemas/__init__.py -------------------------------------------------------------------------------- /buildtest/schemas/examples/global.schema.json/invalid/exceed_testname_length.yml: -------------------------------------------------------------------------------- 1 | # this test fails because it exceeds 32 character length for test name 2 | buildspecs: 3 | _________this_test_exceeds_character_length______________: 4 | type: script 5 | run: hostname 6 | executor: generic.local.bash 7 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/global.schema.json/invalid/invalid_pattern.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | # invalid pattern for test. Must be matching regex "^[A-Za-z_.][A-Za-z0-9_]*$" when declaring a dict 3 | (badname: 4 | type: script 5 | run: "ping login 1" 6 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/global.schema.json/invalid/maintainers_type_mismatch.yml: -------------------------------------------------------------------------------- 1 | # wrong type for maintainers key, expects a string 2 | maintainers: 1 3 | buildspecs: 4 | hostname: 5 | type: script 6 | run: "hostname" 7 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/global.schema.json/invalid/unique_maintainers.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname: 3 | type: script 4 | run: "hostname" 5 | maintainers: [shahzebsiddiqui, shahzebsiddiqui] 6 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/global.schema.json/valid/examples.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | # testing all caps 3 | ABCDEFGHIJKLMNOPQRSTUVWXYZ: 4 | type: script 5 | run: "hostname" 6 | 7 | # testing all lowercase letters 8 | abcdefghijklmnopqrstuvwxyz: 9 | type: script 10 | run: "hostname" 11 | 12 | # testing '_' in beginning followed by all numbers 13 | _0123456789: 14 | type: script 15 | run: "hostname" 16 | 17 | # testing '_' in combination with caps, lowercase and numbers 18 | _ABCDEFabcdef0123456789: 19 | type: script 20 | run: "hostname" 21 | 22 | # testing '_' at end of word 23 | abcdefghijklmnopqrstuvwxyz_: 24 | type: script 25 | run: "hostname" 26 | 27 | # testing '.' in beginning of word 28 | .helloworld: 29 | type: script 30 | run: hostname 31 | 32 | # testing '.' in middle of word 33 | hello.world: 34 | type: script 35 | run: hostname 36 | 37 | # testing '.' at end of word 38 | helloworld.: 39 | type: script 40 | run: hostname 41 | 42 | # testing '-' in middle of word 43 | hello-world: 44 | type: script 45 | run: hostname 46 | 47 | # testing '-' at end of word 48 | helloworld-: 49 | type: script 50 | run: hostname 51 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/executors.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | executors_additionalProperties: 3 | type: script 4 | executor: "generic.local.(bash|sh|zsh)" 5 | description: Testing for additional properties in 'executors' 6 | tags: [tutorials] 7 | run: hostname 8 | sbatch: ["-N 4"] 9 | executors: 10 | generic.local.bash: 11 | sbatch: ["-n 4", "-N 1", "-t 30"] 12 | FOO: BAR 13 | generic.local.sh: 14 | sbatch: ["-n 8", "-N 1", "-t 60"] 15 | generic.local.zsh: 16 | sbatch: ["-n 16", "-N 2", "-t 120"] 17 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/file_count.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_additionalProperties_check: 3 | type: script 4 | executor: generic.local.bash 5 | description: Testing for additional properties in file_count 6 | run: hostname 7 | status: 8 | file_count: 9 | - dir: /tmp 10 | count: 0 11 | FOO: BAR 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/file_regex.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | testing_additionalProperties_file_regex: 3 | type: script 4 | executor: generic.local.bash 5 | description: Testing for additional properties in file_regex 6 | run: | 7 | echo "Hello" > hello.txt 8 | status: 9 | file_regex: 10 | - file: hello.txt 11 | exp: '^(Hello)$' 12 | FOO: BAR 13 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_metrics_additional_property: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test for additional property for metrics property 6 | vars: 7 | FOO: BAR 8 | run: echo $FOO 9 | metrics: 10 | foo: 11 | variable: FOO 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/regex.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | regex_additionalProperties_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: Testing for additional properties in regex field 6 | run: hostname 7 | status: 8 | regex: 9 | stream: stdout 10 | exp: "world$" 11 | X: 1 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/status.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | additionalProperties_status: 3 | type: script 4 | executor: generic.local.bash 5 | description: test additional properties in status object. This is not allowed 6 | sbatch: ["-n 2", "-q normal", "-t 10"] 7 | run: hostname 8 | status: 9 | slurm_job_state: "COMPLETED" 10 | FOO: BAR 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/additionalProperties/test.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | additionalProperties_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: additional properties are not allowed so any invalid key/value pair will result in error 6 | FOO: BAR 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/description_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_description: 3 | type: script 4 | executor: generic.local.bash 5 | description: 6 | - "Multi Line description" 7 | - "is not accepted" 8 | run: sleep 1 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/empty_returncode.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | empty_returncode_list: 3 | type: script 4 | executor: generic.local.bash 5 | description: An empty returncode list will cause an error 6 | run: hostname 7 | status: 8 | returncode: [] 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/empty_tags.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | empty_tags: 3 | type: script 4 | executor: generic.local.bash 5 | description: tag list can't be empty, requires one item. 6 | tags: [] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/env_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_env_type: 3 | type: script 4 | executor: generic.local.bash 5 | description: env key should be a dictionary 6 | env: 7 | - FOO=BAR 8 | run: echo $FOO 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/executors_vars_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | executors_invalid_var_type: 3 | type: script 4 | executor: "generic.local.(bash|sh|zsh)" 5 | description: Invalid type field for 'vars' 6 | tags: [tutorials] 7 | run: echo $FOO 8 | executors: 9 | generic.local.bash: 10 | vars: ["FOO=BAR"] 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/count_property_failure.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_property_count_failure: 3 | type: script 4 | executor: generic.local.bash 5 | description: The count property in file_count must be greater than 0 6 | run: hostname 7 | status: 8 | file_count: 9 | - dir: /tmp 10 | count: -1 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/count_required.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_required_property_check: 3 | type: script 4 | executor: generic.local.bash 5 | description: The dir property in file_count is required 6 | run: hostname 7 | status: 8 | file_count: 9 | - dir: /tmp 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/depth_property_failure.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_property_depth_failure: 3 | type: script 4 | executor: generic.local.bash 5 | description: The depth property in file_count must be greater than 0 6 | run: hostname 7 | status: 8 | file_count: 9 | - dir: /tmp 10 | count: 0 11 | depth: -1 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/filetype_value.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_filetype_invalid_value: 3 | type: script 4 | executor: generic.local.bash 5 | description: Invalid value for filetype, it must be 'file', 'dir' or 'symlink' 6 | run: mkdir foo 7 | status: 8 | file_count: 9 | - dir: foo 10 | count: -1 11 | filetype: 'filename' 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/minimum_file_traverse_limit.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | minimum_check_file_traverse_limit: 3 | type: script 4 | executor: generic.local.bash 5 | description: The minimum value for file_traverse_limit is 1 6 | run: mkdir foo 7 | status: 8 | file_count: 9 | - dir: foo 10 | count: 1 11 | file_traverse_limit: 0 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_count/minimum_value_on_count.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_minimum_check_on_count: 3 | type: script 4 | executor: generic.local.bash 5 | description: The count property must be 0 or greater 6 | run: mkdir foo 7 | status: 8 | file_count: 9 | - dir: foo 10 | count: -1 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_exists_failure.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_exists_failure: 3 | type: script 4 | executor: generic.local.bash 5 | description: this test will fail validation, because item must be a string 6 | run: mkdir -p 1 7 | status: 8 | exists: [1] 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_linecount_invalid.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_linecount_invalid.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/file_regex_filename_property.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_property_filename: 3 | type: script 4 | executor: generic.local.bash 5 | description: Invalid property name filename under file_regex 6 | run: echo "Hello" > hello.txt 7 | status: 8 | file_regex: 9 | - filename: hello.txt 10 | exp: '^(Hello)$' 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/invalid_container_platform.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_container_platform: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid container platform 6 | container: 7 | platform: "containerd" 8 | image: hello-world 9 | run: echo 'Test Complete!' 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/invalid_re_value.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_re_value: 3 | type: script 4 | executor: generic.local.bash 5 | description: The "re" value is invalid 6 | run: echo "world" 7 | status: 8 | regex: 9 | stream: stdout 10 | exp: "world$" 11 | re: "search" 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/invalid_value_mode.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_value_for_mode: 3 | type: script 4 | executor: generic.local.bash 5 | description: The status mode must be 'any' or 'all' 6 | run: exit 0 7 | status: 8 | returncode: 0 9 | mode: '1' 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/linecount_stream_mismatch.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | linecount_stream_check: 3 | type: script 4 | executor: generic.local.bash 5 | description: "This test will fail because stream is not valid. It must be (stdout, stderr)" 6 | run: | 7 | for i in {1..10}; do 8 | echo $i 9 | done 10 | status: 11 | linecount: 12 | stream: foo 13 | count: 10 14 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/metrics/metrics_name.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_metrics_name: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Metrics name does not follow pattern" 6 | run: echo "hello" > file.txt 7 | metrics: 8 | (foo-bar: 9 | type: str 10 | regex: 11 | stream: stdout 12 | exp: "BAR" 13 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/metrics/metrics_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_metrics_type: 3 | type: script 4 | executor: generic.local.bash 5 | description: metrics property is an object, testing for type 6 | vars: 7 | FOO: BAR 8 | run: echo $FOO 9 | metrics: FOO 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/metrics/metrics_type_property.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_type_property_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: metrics 'type' must be a 'str', 'float', 'int' 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: tuple 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/metrics/missing_regex_in_metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_regex_in_metrics: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test for missing regex in metrics property, which should result in error 6 | run: echo "hello" 7 | metrics: 8 | foo: 9 | type: str 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/metrics/regex_and_file_regex_in_metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | test_regex_and_file_regex_in_metrics: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test for regex and file_regex in metrics. Only one is allowed at a time 6 | run: echo "hello" > file.txt 7 | metrics: 8 | foo: 9 | type: str 10 | file_regex: 11 | file: file.txt 12 | exp: "hello" 13 | regex: 14 | stream: stdout 15 | exp: "hello" 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/missing_regex_exp.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_regex_exp: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test fails because of missing key 'exp' in regex 6 | run: hostname 7 | status: 8 | regex: 9 | stream: stdout 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/missing_run.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_run_key: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid key name roon, missing run key 6 | cmd: | 7 | systemctl is-active slurmd 8 | systemctl is-enabled slurmd | grep enabled 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/non_int_returncodes.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | non_int_returncodes: 3 | type: script 4 | executor: generic.local.bash 5 | description: The returncode must be an int and not a number 6 | run: exit 1 7 | status: 8 | returncode: 1.01 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/non_int_returncodes_list.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | non_int_returncodes_list: 3 | type: script 4 | executor: generic.local.bash 5 | description: The returncode must be a list of integers and no numbers 6 | run: exit 1 7 | status: 8 | returncode: [1, 2.230] 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/regex_stream_value.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_regex_stream: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test fails because of invalid regex stream 6 | run: hostname 7 | status: 8 | regex: 9 | stream: file 10 | exp: "world$" 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/returncode_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_returncode_type: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test fails because of invalid return code type 6 | run: hostname 7 | status: 8 | returncode: ["1"] 9 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/runtime_min.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_runtime_min: 3 | type: script 4 | executor: generic.local.sh 5 | description: "Invalid type for min property in runtime. Must be an integer or float not a string" 6 | run: sleep 2 7 | status: 8 | runtime: 9 | min: "1" 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/runtime_min_greater_than_0.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | runtime_min_must_exceed_0: 3 | type: script 4 | executor: generic.local.sh 5 | description: "The runtime must exceed 0" 6 | run: sleep 2 7 | status: 8 | runtime: 9 | min: -1 10 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/shebang_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_type_shell_shebang: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid type for shell shebang, must be a string 6 | shebang: ["#!/bin/bash"] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/shell_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_shell_type: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid shell type must be a string 6 | shell: ["/bin/bash"] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/skip_value.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_skip_value: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid value for skip, must be boolean 6 | skip: 1 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/slurm_job_state.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_slurm_job_state: 3 | type: script 4 | executor: generic.local.sh 5 | description: invalid value for slurm_job_state, should raise error with enum values. 6 | sbatch: 7 | - "-n 2" 8 | - "-q normal" 9 | - "-t 10" 10 | run: hostname 11 | status: 12 | slurm_job_state: "FINISH" 13 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/summary_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_summary: 3 | type: script 4 | executor: generic.local.bash 5 | description: summary field must be a string 6 | summary: 1 7 | run: sleep 1 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/tags_value.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_tags_value: 3 | type: script 4 | executor: generic.local.bash 5 | description: invalid tag value must be all string items 6 | tags: ["network", 400] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/invalid/test_name.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_test_name_&!@#$%: 3 | type: script 4 | executor: generic.local.bash 5 | description: "invalid test name" 6 | run: sleep 1 7 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_eq.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_eq.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_ge.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_ge.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_gt.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_gt.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_le.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_le.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_lt.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_lt.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_ne.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_ne.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/assert_range.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/assert_range.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/bind_mounts.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/containers/bind_mounts.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/contains.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/perf_checks/contains.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/environment.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/environment.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/executor_scheduler.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/multi_executors/executor_scheduler.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/executors_metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | executors_metrics_declaration: 3 | type: script 4 | executor: 'generic.local.(bash|sh)' 5 | description: Declaring metrics by executors section 6 | tags: [tutorials] 7 | run: echo "Hello World" 8 | executors: 9 | generic.local.bash: 10 | metrics: 11 | hello: 12 | type: str 13 | regex: 14 | stream: stdout 15 | exp: "(Hello)" 16 | generic.local.sh: 17 | metrics: 18 | world: 19 | type: str 20 | regex: 21 | stream: stdout 22 | exp: "(World)" 23 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/executors_var_env_declaration.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/multi_executors/executors_var_env_declaration.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/exists.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/exists.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/explicit_state.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/explicit_state.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_and_dir_check.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_and_dir_check.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_count.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_count.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_count_file_traverse_limit.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_count_file_traverse_limit.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_count_filetype.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_count_filetype.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_count_pattern.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_count_pattern.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/file_linecount.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/file_linecount.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/hello_world.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/containers/hello_world.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/is_symlink.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/is_symlink.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/job_dep_ex1.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/job_dependency/ex1.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/job_dep_ex2.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/job_dependency/ex2.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/job_dep_ex3.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/job_dependency/ex3.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/job_dep_ex4.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/job_dependency/ex4.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/job_submission.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | sbatch_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test runs hostname using sbatch directives 6 | sbatch: 7 | - "-t 10:00:00" 8 | - "-p normal" 9 | - "-N 1" 10 | - "-n 8" 11 | run: hostname 12 | 13 | bsub_example: 14 | type: script 15 | executor: generic.local.bash 16 | description: This test runs hostname using bsub directives 17 | bsub: 18 | - "-W 00:30" 19 | - "-N 1" 20 | run: hostname 21 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/linecount.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/linecount.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/metrics_regex.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/metrics/metrics_regex.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/metrics_with_regex_type.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/metrics/metrics_with_regex_type.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/mode.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/mode.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/pass_returncode.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/pass_returncode.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/post_run.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/post_run.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/python-shell.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/python-shell.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/regex_on_filename.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/regex_on_filename.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/run_commands.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/containers/run_commands.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/run_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | multiline_run: 3 | executor: generic.local.bash 4 | type: script 5 | description: multiline run command 6 | run: | 7 | echo "1" 8 | echo "2" 9 | 10 | single_command_run: 11 | executor: generic.local.bash 12 | type: script 13 | description: single command as a string for run command 14 | run: "hostname" 15 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/runtime_status_test.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/runtime_status_test.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/shebang.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/shebang.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/shell_examples.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/shell_examples.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/skip_tests.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/skip_tests.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/specify_regex_type.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/test_status/specify_regex_type.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/status_by_executors.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/multi_executors/status_by_executors.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/summary_example.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/summary_example.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/tag_example.yml: -------------------------------------------------------------------------------- 1 | ../../../../../tutorials/tags_example.yml -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/variable_declaration.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | declare_vars: 3 | executor: generic.local.bash 4 | type: script 5 | description: declaring variables 6 | vars: 7 | First: Bob 8 | Last: Bill 9 | run: | 10 | echo "First:" $First 11 | echo "Last:" $Last 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/script.schema.json/valid/zsh_tcsh_and_csh_shell.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | declare_shell_bin_zsh: 3 | executor: generic.local.zsh 4 | type: script 5 | description: declare shell zsh 6 | shell: "zsh" 7 | run: hostname 8 | 9 | declare_shell_zsh: 10 | executor: generic.local.zsh 11 | type: script 12 | description: declare shell /bin/zsh 13 | shell: "/bin/zsh" 14 | run: hostname 15 | 16 | declare_shell_bin_csh: 17 | executor: generic.local.csh 18 | type: script 19 | description: declare shell /bin/csh 20 | shell: "/bin/csh" 21 | run: hostname 22 | 23 | declare_shell_csh: 24 | executor: generic.local.csh 25 | type: script 26 | description: declare shell /bin/tcsh 27 | shell: "csh" 28 | run: hostname 29 | 30 | declare_shell_bin_tcsh: 31 | executor: generic.local.csh 32 | type: script 33 | description: declare shell /bin/tcsh 34 | shell: "/bin/tcsh" 35 | run: hostname 36 | 37 | declare_shell_tcsh: 38 | executor: generic.local.csh 39 | type: script 40 | description: declare shell tcsh 41 | shell: "tcsh" 42 | run: hostname 43 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/settings.schema.json/invalid/invalid_executor_names.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: ['.*'] 4 | moduletool: N/A 5 | executors: 6 | local: 7 | # the line below is invalid due to pattern property 8 | ==bash: 9 | description: submit jobs on local machine 10 | shell: bash -v 11 | slurm: 12 | # the line below is invalid due to pattern property 13 | $haswell: 14 | launcher: sbatch 15 | options: ["-p haswell", "-t 00:10"] 16 | lsf: 17 | <>batch: 18 | launcher: bsub 19 | queue: batch 20 | options: ["-q batch", "-t 00:10"] 21 | compilers: 22 | compiler: 23 | gcc: 24 | default: 25 | cc: /usr/bin/gcc 26 | cxx: /usr/bin/g++ 27 | fc: /usr/bin/gfortran 28 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/settings.schema.json/valid/pbs-example.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: ['.*'] 4 | moduletool: none 5 | poolsize: 1 6 | file_traversal_limit: 1000 7 | buildspecs: 8 | # whether to rebuild cache file automatically when running `buildtest buildspec find` 9 | rebuild: False 10 | # limit number of records to display when running `buildtest buildspec find` 11 | count: 15 12 | # format fields to display when running `buildtest buildspec find`, By default we will show name,description 13 | format: "name,description" 14 | # enable terse mode 15 | terse: False 16 | report: 17 | count: 25 18 | format: "name,id,state,runtime,returncode" 19 | paths: 20 | pbs: /usr/bin 21 | executors: 22 | defaults: 23 | pollinterval: 10 24 | maxpendtime: 30 25 | local: 26 | bash: 27 | description: submit jobs via bash shell 28 | shell: bash 29 | pbs: 30 | workq: 31 | queue: workq 32 | compilers: 33 | compiler: 34 | gcc: 35 | default: 36 | cc: /usr/bin/gcc 37 | cxx: /usr/bin/g++ 38 | fc: /usr/bin/gfortran 39 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/settings.schema.json/valid/torque-example.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: ['.*'] 4 | moduletool: none 5 | poolsize: 1 6 | file_traversal_limit: 1000 7 | buildspecs: 8 | # whether to rebuild cache file automatically when running `buildtest buildspec find` 9 | rebuild: False 10 | # limit number of records to display when running `buildtest buildspec find` 11 | count: 15 12 | # format fields to display when running `buildtest buildspec find`, By default we will show name,description 13 | format: "name,description" 14 | # enable terse mode 15 | terse: False 16 | report: 17 | count: 25 18 | format: "name,id,state,runtime,returncode" 19 | paths: 20 | torque: /usr/bin 21 | executors: 22 | defaults: 23 | pollinterval: 10 24 | maxpendtime: 30 25 | local: 26 | bash: 27 | description: submit jobs via bash shell 28 | shell: bash 29 | torque: 30 | lbl-cluster: 31 | queue: lbl-cluster 32 | compilers: 33 | compiler: 34 | gcc: 35 | default: 36 | cc: /usr/bin/gcc 37 | cxx: /usr/bin/g++ 38 | fc: /usr/bin/gfortran 39 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/additionalProperties/spack.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | additionalProperties_spack_field: 3 | type: spack 4 | executor: generic.local.sh 5 | description: additional Properties can't be specified in spack section 6 | spack: 7 | root: $HOME/spack 8 | FOO: BAR 9 | env: 10 | create: 11 | name: myproject 12 | specs: 13 | - zlib 14 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/additionalProperties/spack_test.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_test_additionalProperties: 3 | type: spack 4 | executor: generic.local.sh 5 | description: "Check for additionalProperties in test section. FOO key is not allowed" 6 | tags: [spack] 7 | pre_cmds: | 8 | cd /tmp 9 | git clone https://github.com/spack/spack 10 | spack: 11 | root: /tmp/spack 12 | verify_spack: false 13 | install: 14 | specs: ['m4', 'zlib'] 15 | test: 16 | FOO: BAR 17 | remove_tests: true 18 | run: 19 | specs: ['m4', 'zlib'] 20 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/mirror_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_type_mirror_field: 3 | type: spack 4 | executor: generic.local.sh 5 | description: The mirror field must be a key value pair 6 | spack: 7 | root: $HOME/spack 8 | mirror: https://caches.e4s.io 9 | env: 10 | create: 11 | name: myproject 12 | specs: 13 | - zlib 14 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/spack_env_deactivate.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_env_deactivate_first: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "deactivate a spack environment first prior to activating it" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | dir: $HOME/spack-envs/m4 12 | deactivate: 2.0 13 | activate: 14 | dir: $HOME/spack-envs/m4 15 | specs: 16 | - 'm4' 17 | install: 18 | option: '' 19 | post_cmds: | 20 | spack find 21 | rm -rf $HOME/spack-envs/m4 22 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/spack_load_invalid_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_load_invalid_example: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Run spack load for m4 package and run tests" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | load: 10 | options: '--only package' 11 | specs: [1.5] 12 | test: 13 | run: 14 | specs: ['m4'] 15 | results: 16 | option: "-l" 17 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/spack_test_run_invalid_spec.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_test_run_invalid_spec: 3 | type: spack 4 | executor: generic.local.sh 5 | description: "specs property requires a list of strings. " 6 | tags: [spack] 7 | pre_cmds: | 8 | cd /tmp 9 | git clone https://github.com/spack/spack 10 | spack: 11 | root: /tmp/spack 12 | verify_spack: false 13 | install: 14 | specs: ['m4', 'zlib'] 15 | test: 16 | remove_tests: true 17 | run: 18 | specs: ['m4', 1] 19 | results: 20 | option: '-f' 21 | post_cmds: | 22 | spack find 23 | rm -rf $SPACK_ROOT 24 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/invalid/specs_list_check_string.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | specs_must_be_list_of_strings: 3 | type: spack 4 | executor: generic.local.sh 5 | description: 'specs must be a list of strings' 6 | spack: 7 | root: $HOME/spack 8 | env: 9 | create: 10 | name: myproject 11 | specs: [1, zlib] 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_activate.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_activate: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Activate spack environment by name 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | activate: 10 | name: myproject 11 | specs: 12 | - m4 13 | - zlib 14 | install: 15 | options: '' 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_concretize.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_concretized_install: 3 | type: spack 4 | description: run 'spack concretize -f' in an environment and install specs 5 | executor: generic.local.sh 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | create: 10 | name: myproject 11 | manifest: $HOME/spack.yaml 12 | concretize: true 13 | install: 14 | options: '--cache-only' 15 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_create_directory.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_create_directory: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Create spack environment by directory 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | create: 10 | dir: $HOME/spack-env/myproject 11 | specs: 12 | - 'm4' 13 | - 'zlib@1.2.11' 14 | install: 15 | options: '--cache-only' 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_create_manifest.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_create_from_manifest: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Create spack enviromment from manifest file 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | create: 10 | name: myproject 11 | manifest: $HOME/spack.yaml 12 | install: 13 | options: '--cache-only' 14 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_create_name.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_create_name: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Create spack environment by name 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | create: 10 | name: myproject 11 | specs: 12 | - m4 13 | - zlib 14 | install: 15 | options: '' 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/env_mirror.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | env_mirror: 3 | type: spack 4 | executor: generic.local.sh 5 | description: declare spack mirror 'spack mirror add h5 /path/to/mirror' in environment 6 | spack: 7 | root: $HOME/spack/ 8 | env: 9 | mirror: 10 | h5: /path/to/mirror 11 | create: 12 | name: myproject 13 | manifest: $HOME/spack.yaml 14 | install: 15 | options: '--cache-only' 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/install_without_env.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | install_without_env: 3 | type: spack 4 | executor: generic.local.sh 5 | description: install specs without environment 6 | spack: 7 | root: $HOME/spack/ 8 | install: 9 | options: '--cache-only' 10 | specs: ['m4', 'bzip2'] 11 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/pre_post.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | pre_post_cmd_spack_install: 3 | type: spack 4 | executor: generic.local.sh 5 | description: run commands before and after spack using pre_cmds and post_cmds 6 | pre_cmds: | 7 | cd $HOME 8 | git clone https://github.com/spack/spack 9 | spack: 10 | root: $HOME/spack/ 11 | install: 12 | options: '--cache-only' 13 | specs: ['m4', 'bzip2'] 14 | post_cmds: | 15 | spack find 16 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/remove_spack_env_automatically.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | remove_spack_environment_automatically: 3 | type: spack 4 | executor: generic.local.sh 5 | description: remove spack environment automatically 6 | spack: 7 | root: $HOME/spack 8 | env: 9 | create: 10 | remove_environment: true 11 | name: m4 12 | activate: 13 | name: m4 14 | specs: 15 | - 'm4' 16 | 17 | post_cmds: | 18 | spack find 19 | rm -rf $SPACK_ROOT 20 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/remove_spack_environment.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | remove_spack_environment: 3 | type: spack 4 | executor: generic.local.sh 5 | description: remove spack environment explicitly before creating environment 6 | spack: 7 | root: $HOME/spack 8 | env: 9 | rm: 10 | name: m4 11 | create: 12 | name: m4 13 | activate: 14 | name: m4 15 | specs: 16 | - 'm4' 17 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/sbatch_multi_executors.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_sbatch_multi_executors: 3 | type: spack 4 | executor: "generic.local.(sh|bash)" 5 | description: "sbatch directives can be defined in spack schema" 6 | tags: [spack] 7 | executors: 8 | generic.local.sh: 9 | sbatch: ["-N 1", "-t 20"] 10 | generic.local.bash: 11 | sbatch: ["-N 8", "-t 10"] 12 | spack: 13 | root: $HOME/spack 14 | env: 15 | specs: 16 | - 'm4' 17 | activate: 18 | name: m4 19 | concretize: true 20 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/sbatch_with_spack.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | sbatch_field_with_spack: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Specify sbatch field with spack schema 6 | sbatch: ["-N1", "-q normal", "-t 10", "-M 30M"] 7 | spack: 8 | root: $HOME/spack 9 | install: 10 | specs: ['m4'] 11 | options: "-v" 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/skip_test.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | skip_test_in_spack: 3 | skip: True 4 | type: spack 5 | executor: generic.local.sh 6 | description: This test will be skipped 7 | spack: 8 | root: $HOME/spack 9 | install: 10 | specs: ['m4'] 11 | options: "-v" 12 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/spack_env_deactivate.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_env_deactivate_first: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "deactivate a spack environment first prior to activating it" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | dir: $HOME/spack-envs/m4 12 | deactivate: true 13 | activate: 14 | dir: $HOME/spack-envs/m4 15 | specs: 16 | - 'm4' 17 | install: 18 | option: '' 19 | post_cmds: | 20 | spack find 21 | rm -rf $HOME/spack-envs/m4 22 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/spack_load_valid_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_load_valid_example: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Run spack load for m4 package and run tests" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | load: 10 | options: '--only package' 11 | specs: [m4] 12 | test: 13 | run: 14 | specs: ['m4'] 15 | results: 16 | option: "-l" 17 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/var_and_env_declaration.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | var_declaration_in_spack: 3 | type: spack 4 | executor: generic.local.sh 5 | description: Define variables and environment variables in spack 6 | vars: 7 | FOO: "BAR" 8 | env: 9 | SPACK_GNUPGHOME: "$HOME/.gnupg" 10 | spack: 11 | root: $HOME/spack 12 | install: 13 | specs: ['m4'] 14 | options: "-v" 15 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/spack.schema.json/valid/vars_multi_executors.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | vars_multi_executors: 3 | type: spack 4 | executor: "generic.local.(sh|bash)" 5 | description: "variable declaration with multiple executors" 6 | tags: [spack] 7 | executors: 8 | generic.local.sh: 9 | vars: 10 | FOO: BAR 11 | generic.local.bash: 12 | vars: 13 | HELLO: WORLD 14 | spack: 15 | root: $HOME/spack 16 | env: 17 | specs: 18 | - 'm4' 19 | activate: 20 | name: m4 21 | concretize: true 22 | -------------------------------------------------------------------------------- /buildtest/schemas/examples/special_invalid_buildspecs/missing_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_type: 3 | executor: generic.local.bash 4 | description: "type key is missing, this is a required field" 5 | source: "src/hello.c" 6 | compilers: 7 | name: [intel] 8 | -------------------------------------------------------------------------------- /buildtest/schemas/global.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$id": "global.schema.json", 3 | "$schema": "http://json-schema.org/draft-07/schema#", 4 | "title": "global schema", 5 | "description": "buildtest global schema is validated for all buildspecs. The global schema defines top-level structure of buildspec and defintions that are inherited for sub-schemas", 6 | "type": "object", 7 | "required": ["buildspecs"], 8 | "additionalProperties": false, 9 | "properties": { 10 | "skip": { 11 | "$ref": "definitions.schema.json#/definitions/skip" 12 | }, 13 | "maintainers": { 14 | "type": "array", 15 | "description": "One or more maintainers or aliases", 16 | "uniqueItems": true, 17 | "minItems": 1, 18 | "items": { 19 | "type": "string" 20 | } 21 | }, 22 | "buildspecs": { 23 | "type": "object", 24 | "description": "This section is used to define one or more tests (buildspecs). Each test must be unique name", 25 | "propertyNames": { 26 | "pattern": "^[A-Za-z_.-][A-Za-z0-9_.-]*$", 27 | "maxLength": 48 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /buildtest/settings/aws.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: 4 | - .* 5 | description: Generic System 6 | moduletool: environment-modules 7 | pager: false 8 | file_traversal_limit: 1000 9 | buildspecs: 10 | rebuild: false 11 | count: 15 12 | format: name,description 13 | terse: false 14 | search: 15 | - $BUILDTEST_ROOT/aws_tutorial 16 | report: 17 | count: 25 18 | format: name,id,state,runtime,returncode 19 | executors: 20 | local: 21 | bash: 22 | description: submit jobs on local machine using bash shell 23 | shell: bash 24 | torque: 25 | e4spro-cluster: 26 | queue: e4spro-cluster 27 | compilers: 28 | find: 29 | gcc: ^(gcc) 30 | compiler: 31 | gcc: 32 | gcc_11.4.0: 33 | cc: /usr/bin/gcc 34 | fc: /usr/bin/gfortran 35 | cxx: /usr/bin/g++ 36 | gcc_12.3.0: 37 | cc: /usr/bin/gcc-12 38 | fc: /usr/bin/gfortran-12 39 | cxx: /usr/bin/g++-12 40 | cdash: 41 | url: https://my.cdash.org/ 42 | project: buildtest 43 | site: generic 44 | buildname: buildtest_aws_tutorial 45 | -------------------------------------------------------------------------------- /buildtest/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/tools/__init__.py -------------------------------------------------------------------------------- /buildtest/tools/modules.py: -------------------------------------------------------------------------------- 1 | def get_module_commands(modules): 2 | """Return a list of module command as a list of instructions based on 3 | ``module`` property which can be defined in the configuration or compiler schema 4 | 5 | Args: 6 | modules (dict): The module property specified in buildspec or configuration file 7 | 8 | Returns: 9 | list: a list of module commands 10 | """ 11 | 12 | if not modules or not isinstance(modules, dict): 13 | return 14 | 15 | module_cmd = [] 16 | 17 | # if purge is True and defined add module purge 18 | if modules.get("purge"): 19 | module_cmd += ["module purge"] 20 | 21 | if modules.get("restore"): 22 | module_cmd += [f"module restore {modules['restore']}"] 23 | 24 | if modules.get("swap"): 25 | module_cmd += [f"module swap {' '.join(modules['swap'])}"] 26 | 27 | if modules.get("load"): 28 | for name in modules["load"]: 29 | module_cmd += [f"module load {name}"] 30 | 31 | return module_cmd 32 | -------------------------------------------------------------------------------- /buildtest/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/buildtest/utils/__init__.py -------------------------------------------------------------------------------- /buildtest/utils/timer.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class TimerError(Exception): 5 | """A custom exception used to report errors in use of Timer class""" 6 | 7 | 8 | class Timer: 9 | def __init__(self): 10 | self._start_time = None 11 | self._duration = 0 12 | 13 | def start(self): 14 | """Start a new timer""" 15 | if self._start_time is not None: 16 | raise TimerError("Timer is running. Use .stop() to stop it") 17 | 18 | self._start_time = time.perf_counter() 19 | 20 | def stop(self): 21 | """Stop the timer, and report the elapsed time""" 22 | if self._start_time is None: 23 | raise TimerError("Timer is not running. Use .start() to start it") 24 | 25 | self._duration += time.perf_counter() - self._start_time 26 | self._start_time = None 27 | # return elapsed_time 28 | 29 | def duration(self): 30 | return round(self._duration, 3) 31 | -------------------------------------------------------------------------------- /docs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/__init__.py -------------------------------------------------------------------------------- /docs/_static/CDASH.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/_static/CDASH.png -------------------------------------------------------------------------------- /docs/_static/DiscoverBuildspecs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/_static/DiscoverBuildspecs.jpg -------------------------------------------------------------------------------- /docs/_static/GeneralPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/_static/GeneralPipeline.png -------------------------------------------------------------------------------- /docs/_static/ParserDiagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/_static/ParserDiagram.png -------------------------------------------------------------------------------- /docs/_static/buildspec-structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/_static/buildspec-structure.png -------------------------------------------------------------------------------- /docs/_templates/autoapi/index.rst: -------------------------------------------------------------------------------- 1 | buildtest API Reference 2 | ========================== 3 | 4 | .. toctree:: 5 | :titlesonly: 6 | 7 | {% for page in pages %} 8 | {% if page.top_level_object and page.display %} 9 | {{ page.include_path }} 10 | {% endif %} 11 | {% endfor %} 12 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | .. toctree:: 4 | 5 | api/index -------------------------------------------------------------------------------- /docs/aws_examples/compiler_list_yaml.txt: -------------------------------------------------------------------------------- 1 | $ buildtest -c /home/lbladmin/Documents/buildtest/buildtest/settings/aws.yml config compilers list --yaml 2 | gcc: 3 | gcc_11.4.0: 4 | cc: /usr/bin/gcc 5 | cxx: /usr/bin/g++ 6 | fc: /usr/bin/gfortran 7 | gcc_12.3.0: 8 | cc: /usr/bin/gcc-12 9 | cxx: /usr/bin/g++-12 10 | fc: /usr/bin/gfortran-12 11 | 12 | -------------------------------------------------------------------------------- /docs/aws_examples/container_executor_list.txt: -------------------------------------------------------------------------------- 1 | $ buildtest -c $BUILDTEST_ROOT/buildtest/settings/container_executor.yml config executors list --yaml 2 | executors: 3 | container: 4 | python: 5 | description: submit jobs on python container 6 | image: python:3.11.0 7 | platform: docker 8 | ubuntu: 9 | description: submit jobs on ubuntu container 10 | image: ubuntu:20.04 11 | platform: docker 12 | local: 13 | bash: 14 | description: submit jobs on local machine using bash shell 15 | shell: bash 16 | 17 | -------------------------------------------------------------------------------- /docs/command.rst: -------------------------------------------------------------------------------- 1 | Buildtest Command Reference 2 | ============================ 3 | 4 | .. argparse:: 5 | :ref: buildtest.cli.get_parser 6 | :prog: buildtest 7 | -------------------------------------------------------------------------------- /docs/configuring_buildtest.rst: -------------------------------------------------------------------------------- 1 | .. _configuring_buildtest: 2 | 3 | How to configure buildtest 4 | =========================== 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | configuring_buildtest/overview 10 | configuring_buildtest/executors 11 | configuring_buildtest/compilers 12 | configuring_buildtest/cli_to_buildtest_configuration 13 | configuring_buildtest/site_examples 14 | -------------------------------------------------------------------------------- /docs/contributing/buildtest_branch_settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/contributing/buildtest_branch_settings.png -------------------------------------------------------------------------------- /docs/contributing/buildtest_merge_options.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/contributing/buildtest_merge_options.png -------------------------------------------------------------------------------- /docs/contributing/coverage_locally.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/docs/contributing/coverage_locally.png -------------------------------------------------------------------------------- /docs/general_tests: -------------------------------------------------------------------------------- 1 | ../general_tests -------------------------------------------------------------------------------- /docs/getting_started.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started: 2 | 3 | 4 | Buildtest Command Line Reference 5 | ================================= 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | gettingstarted/buildingtest 11 | gettingstarted/buildspecs_interface 12 | gettingstarted/query_test_report -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | docutils==0.20.1 2 | readthedocs-sphinx-search==0.3.2 3 | sphinx==7.4.7 4 | sphinx-design==0.5.0 5 | sphinx-argparse==0.5.2 6 | sphinx-autoapi==3.0.0 7 | sphinx-rtd-theme==2.0.0 8 | sphinxcontrib-programoutput==0.17 9 | sphinxext-remoteliteralinclude==0.4.0 10 | sphinx-copybutton==0.5.2 11 | -------------------------------------------------------------------------------- /docs/scripting_examples/ex2.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from buildtest.cli.build import BuildTest 4 | from buildtest.defaults import BUILDTEST_ROOT, DEFAULT_SETTINGS_FILE 5 | 6 | input_buildspecs = [os.path.join(BUILDTEST_ROOT, "tutorials", "vars.yml")] 7 | 8 | 9 | # this will invoke buildtest build -b $BUILDTEST_ROOT/tutorials/vars.yml 10 | cmd = BuildTest(config_file=DEFAULT_SETTINGS_FILE, buildspecs=input_buildspecs) 11 | cmd.discover_buildspecs(printTable=True) 12 | cmd.parse_buildspecs(printTable=True) 13 | cmd.build_phase(printTable=True) 14 | -------------------------------------------------------------------------------- /docs/scripting_examples/ex2.py.out: -------------------------------------------------------------------------------- 1 | Searching by tagname: ['tutorials'] 2 | 3 | Discovered buildspecs: 4 | 5 | /Users/siddiq90/Documents/buildtest/tutorials/hello_world.yml 6 | /Users/siddiq90/Documents/buildtest/tutorials/shell_examples.yml 7 | /Users/siddiq90/Documents/buildtest/tutorials/compilers/gnu_hello.yml 8 | /Users/siddiq90/Documents/buildtest/tutorials/run_only_platform.yml 9 | /Users/siddiq90/Documents/buildtest/tutorials/vars.yml 10 | /Users/siddiq90/Documents/buildtest/tutorials/root_user.yml 11 | /Users/siddiq90/Documents/buildtest/tutorials/compilers/vecadd.yml 12 | /Users/siddiq90/Documents/buildtest/tutorials/compilers/pre_post_build_run.yml 13 | /Users/siddiq90/Documents/buildtest/tutorials/compilers/passing_args.yml 14 | /Users/siddiq90/Documents/buildtest/tutorials/skip_tests.yml 15 | /Users/siddiq90/Documents/buildtest/tutorials/shebang.yml 16 | /Users/siddiq90/Documents/buildtest/tutorials/selinux.yml 17 | /Users/siddiq90/Documents/buildtest/tutorials/pass_returncode.yml 18 | /Users/siddiq90/Documents/buildtest/tutorials/python-shell.yml 19 | /Users/siddiq90/Documents/buildtest/tutorials/environment.yml 20 | /Users/siddiq90/Documents/buildtest/tutorials/sleep.yml 21 | /Users/siddiq90/Documents/buildtest/tutorials/systemd.yml 22 | -------------------------------------------------------------------------------- /docs/tests: -------------------------------------------------------------------------------- 1 | ../tests -------------------------------------------------------------------------------- /docs/tutorials: -------------------------------------------------------------------------------- 1 | ../tutorials -------------------------------------------------------------------------------- /docs/writing_buildspecs.rst: -------------------------------------------------------------------------------- 1 | .. _writing_buildspecs: 2 | 3 | 4 | Writing Buildspecs 5 | =================== 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | :caption: Buildspec Reference 10 | 11 | writing_buildspecs/global 12 | writing_buildspecs/compilation 13 | writing_buildspecs/metrics 14 | writing_buildspecs/test_dependency 15 | writing_buildspecs/multi_executor 16 | writing_buildspecs/customize_shell 17 | writing_buildspecs/containers 18 | writing_buildspecs/status_check 19 | writing_buildspecs/comparison_operators -------------------------------------------------------------------------------- /examples/spack/clone_spack.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | clone_spack_automatically: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Clone spack automatically" 6 | tags: [ spack ] 7 | spack: 8 | env: 9 | create: 10 | name: 'spack-develop' 11 | post_cmds: | 12 | spack env list 13 | which spack 14 | 15 | clone_spack_and_specify_root: 16 | type: spack 17 | executor: generic.local.bash 18 | description: Clone spack explicitly and specify root 19 | tags: [ spack ] 20 | pre_cmds: git clone -b e4s-23.05 https://github.com/spack/spack.git /tmp/spack-demo 21 | spack: 22 | root: /tmp/spack-demo 23 | env: 24 | create: 25 | name: 'e4s' 26 | post_cmds: | 27 | spack env list 28 | which spack -------------------------------------------------------------------------------- /examples/spack/e4s_testsuite_mpich.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | mpich_e4s_testsuite: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run E4S Testsuite mpich test 6 | tags: [e4s] 7 | run: | 8 | spack install mpich 9 | git clone https://github.com/E4S-Project/testsuite 10 | cd testsuite 11 | bash test-all.sh --color-off validation_tests/mpich --print-logs 12 | -------------------------------------------------------------------------------- /examples/spack/env_create_directory.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_env_directory: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "create spack environment in directory" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | dir: $HOME/spack-envs/m4 12 | activate: 13 | dir: $HOME/spack-envs/m4 14 | specs: 15 | - 'm4' 16 | install: 17 | option: '' 18 | post_cmds: | 19 | spack find 20 | rm -rf $HOME/spack-envs/m4 -------------------------------------------------------------------------------- /examples/spack/env_create_manifest.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_env_create_from_manifest: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Create spack environment from spack.yaml" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | name: 'manifest_example' 12 | manifest: "$BUILDTEST_ROOT/examples/spack/example/spack.yaml" 13 | activate: 14 | name: 'manifest_example' 15 | concretize: true 16 | post_cmds: | 17 | spack env deactivate 18 | spack env remove -y manifest_example -------------------------------------------------------------------------------- /examples/spack/env_install.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | install_in_spack_env: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Install m4 and zlib in a spack environment named m4_zlib" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | compiler_find: true 10 | env: 11 | create: 12 | name: 'm4_zlib' 13 | specs: 14 | - 'm4' 15 | - 'zlib' 16 | activate: 17 | name: m4_zlib 18 | concretize: true 19 | install: 20 | option: '--keep-prefix' 21 | post_cmds: | 22 | spack env deactivate 23 | spack env remove -y m4_zlib -------------------------------------------------------------------------------- /examples/spack/example/spack.yaml: -------------------------------------------------------------------------------- 1 | spack: 2 | view: false 3 | specs: 4 | - m4 5 | - zlib 6 | - python -------------------------------------------------------------------------------- /examples/spack/install_specs.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | install_specs_example: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Install zlib from an existing spack instance" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | install: 10 | specs: ['zlib'] -------------------------------------------------------------------------------- /examples/spack/mirror_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | add_mirror: 3 | type: spack 4 | executor: generic.local.bash 5 | description: Declare spack mirror 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | mirror: 10 | spack_tutorial_mirror: /mirror 11 | post_cmds: | 12 | spack mirror list 13 | spack config blame mirrors 14 | 15 | add_mirror_in_spack_env: 16 | type: spack 17 | executor: generic.local.bash 18 | description: Declare spack mirror in spack environment 19 | tags: [spack] 20 | spack: 21 | root: $HOME/spack 22 | env: 23 | create: 24 | name: spack_mirror 25 | activate: 26 | name: spack_mirror 27 | mirror: 28 | spack_tutorial_mirror: /mirror 29 | post_cmds: | 30 | spack mirror list 31 | spack config blame mirrors -------------------------------------------------------------------------------- /examples/spack/pre_post_cmds.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | run_pre_post_commands: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Install zlib" 6 | tags: [spack] 7 | pre_cmds: | 8 | cat /etc/os-release 9 | gcc --version 10 | spack: 11 | root: $HOME/spack 12 | install: 13 | specs: ['zlib'] 14 | post_cmds: | 15 | spack location -i gcc@12.3.0 16 | spack --version -------------------------------------------------------------------------------- /examples/spack/remove_environment_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | remove_environment_automatically: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "remove spack environment automatically before creating a new environment" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | remove_environment: true 12 | name: remove_environment 13 | activate: 14 | name: remove_environment 15 | specs: 16 | - 'bzip2' 17 | concretize: true 18 | 19 | remove_environment_explicit: 20 | type: spack 21 | executor: generic.local.bash 22 | description: "remove spack environment explicitly using the 'rm' property" 23 | tags: [spack] 24 | spack: 25 | root: $HOME/spack 26 | env: 27 | rm: 28 | name: dummy 29 | create: 30 | name: dummy 31 | activate: 32 | name: dummy 33 | specs: 34 | - 'bzip2' 35 | concretize: true -------------------------------------------------------------------------------- /examples/spack/spack_env_deactivate.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_env_deactivate_first: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "deactivate a spack environment first prior to activating it" 6 | tags: [spack] 7 | pre_cmds: | 8 | rm -rf $HOME/spack-envs/m4 9 | spack: 10 | root: $HOME/spack 11 | env: 12 | create: 13 | dir: $HOME/spack-envs/m4 14 | deactivate: true 15 | activate: 16 | dir: $HOME/spack-envs/m4 17 | specs: 18 | - 'm4' 19 | install: 20 | option: '' 21 | post_cmds: | 22 | spack find 23 | rm -rf $HOME/spack-envs/m4 24 | -------------------------------------------------------------------------------- /examples/spack/spack_load.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_load_example: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Run spack load for m4 package and run tests" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | load: 10 | options: '--only package' 11 | specs: ['m4%gcc@12'] 12 | test: 13 | run: 14 | specs: ['m4%gcc@12'] 15 | results: 16 | option: "-l" 17 | -------------------------------------------------------------------------------- /examples/spack/spack_multiple_executor_sbatch.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_sbatch_multi_executors: 3 | type: spack 4 | executor: "generic.local.(sh|bash)" 5 | description: "sbatch directives can be defined in spack schema" 6 | tags: [spack] 7 | executors: 8 | generic.local.sh: 9 | sbatch: ["-N 1", "-t 30"] 10 | generic.local.bash: 11 | sbatch: ["-N 8", "-t 15"] 12 | spack: 13 | root: $HOME/spack 14 | test: 15 | run: 16 | specs: ['libsigsegv'] 17 | results: 18 | specs: ['libsigsegv'] 19 | option: "-l" -------------------------------------------------------------------------------- /examples/spack/spack_sbatch.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_sbatch_example: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "sbatch directives can be defined in spack schema" 6 | tags: [spack] 7 | sbatch: ["-N 1", "-n 8", "-t 30"] 8 | spack: 9 | root: $HOME/spack 10 | install: 11 | specs: ['zlib%clang', 'zlib%gcc'] -------------------------------------------------------------------------------- /examples/spack/spack_test.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_test_m4: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Run spack test for m4 package and report results" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | test: 10 | run: 11 | specs: ['m4'] 12 | results: 13 | option: "-l" -------------------------------------------------------------------------------- /examples/spack/spack_test_specs.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | spack_test_results_specs_format: 3 | type: spack 4 | executor: generic.local.bash 5 | description: "Run spack test results with spec format" 6 | tags: [spack] 7 | spack: 8 | root: $HOME/spack 9 | env: 10 | create: 11 | remove_environment: true 12 | name: spack_test_example 13 | activate: 14 | name: spack_test_example 15 | install: 16 | specs: ['libxml2', 'libsigsegv'] 17 | option: "--add" 18 | test: 19 | remove_tests: true 20 | run: 21 | specs: ['libxml2', 'libsigsegv'] 22 | results: 23 | option: '-l' 24 | specs: ['libxml2', 'libsigsegv'] 25 | post_cmds: | 26 | spack env deactivate 27 | spack env remove -y spack_test_example -------------------------------------------------------------------------------- /general_tests/configuration/disk_usage.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | root_disk_usage: 4 | executor: generic.local.bash 5 | type: script 6 | tags: [filesystem, storage] 7 | description: Check root disk usage and report if it exceeds threshold 8 | env: 9 | threshold: 90 10 | run: | 11 | root_disk_usage=`df -a / | tail -n 1 | awk '{print $5'} | sed 's/[^0-9]*//g'` 12 | # if root exceeds threshold 13 | if [ "$root_disk_usage" -gt "$threshold" ]; then 14 | echo "[WARNING] Root Disk Usage: $root_disk_usage% exceeded threshold of $threshold%" 15 | exit 1 16 | fi 17 | echo "[OK] Root disk is below threshold of $threshold%" 18 | -------------------------------------------------------------------------------- /general_tests/configuration/kernel_state.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | kernel_swapusage: 4 | type: script 5 | description: "Retrieve Kernel Swap Usage" 6 | tags: [configuration] 7 | executor: generic.local.bash 8 | run: /usr/sbin/sysctl kernel.swapusage -------------------------------------------------------------------------------- /general_tests/configuration/systemd-default-target.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | systemd_default_target: 4 | executor: generic.local.bash 5 | type: script 6 | tags: [system] 7 | description: check if default target is multi-user.target 8 | run: | 9 | if [ "multi-user.target" == `systemctl get-default` ]; then 10 | echo "multi-user is the default target"; 11 | exit 0 12 | fi 13 | echo "multi-user is not the default target"; 14 | exit 1 -------------------------------------------------------------------------------- /general_tests/containers/singularity/build.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | build_sif_from_dockerimage: 4 | type: script 5 | executor: generic.local.bash 6 | description: build sif image from docker image docker://godlovedc/lolcow 7 | tags: [containers, singularity] 8 | run: | 9 | singularity build mylolcow_latest.sif docker://godlovedc/lolcow 10 | singularity inspect mylolcow_latest.sif 11 | 12 | build_sandbox_image: 13 | type: script 14 | executor: generic.local.bash 15 | description: build sandbox image from docker image docker://godlovedc/lolcow 16 | tags: [containers, singularity] 17 | run: singularity build --sandbox mylolcow_latest_sandbox docker://godlovedc/lolcow 18 | 19 | build_remoteimages: 20 | type: script 21 | executor: generic.local.bash 22 | description: build remote hosted image from AWS 23 | tags: [containers, singularity] 24 | vars: 25 | IMAGE: alpine_oci_archive.sif 26 | run: | 27 | singularity pull https://s3.amazonaws.com/singularity-ci-public/alpine-oci-archive.tar 28 | singularity build $IMAGE oci-archive://alpine-oci-archive.tar 29 | singularity exec $IMAGE cat /etc/os-release 30 | -------------------------------------------------------------------------------- /general_tests/containers/singularity/inspect.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | inspect_image: 4 | type: script 5 | executor: generic.local.bash 6 | description: Inspect image via 'singularity inspect' 7 | tags: [containers, singularity] 8 | vars: 9 | IMAGE: alpine.sif 10 | run: | 11 | singularity pull $IMAGE library://alpine:latest 12 | singularity inspect --all $IMAGE 13 | singularity inspect -d $IMAGE 14 | singularity inspect -e $IMAGE 15 | singularity inspect -j $IMAGE 16 | singularity inspect -l $IMAGE 17 | singularity inspect -r $IMAGE 18 | singularity inspect -s $IMAGE 19 | singularity inspect -t $IMAGE 20 | -------------------------------------------------------------------------------- /general_tests/containers/singularity/pull.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | pullImage_dockerhub: 4 | type: script 5 | executor: generic.local.bash 6 | description: Pull image docker://godlovedc/lolcow from DockerHub 7 | tags: [containers, singularity] 8 | vars: 9 | IMAGE: lolcow_latest.sif 10 | run: | 11 | singularity pull $IMAGE docker://godlovedc/lolcow 12 | singularity inspect $IMAGE 13 | 14 | pullImage_sylabscloud: 15 | type: script 16 | executor: generic.local.bash 17 | description: Pull image library://alpine:latest from Sylabs Cloud 18 | tags: [containers, singularity] 19 | vars: 20 | IMAGE: alpine.sif 21 | run: | 22 | singularity pull $IMAGE library://alpine:latest 23 | singularity inspect $IMAGE 24 | 25 | pullImage_shub: 26 | type: script 27 | executor: generic.local.bash 28 | description: Pull image shub://vsoch/singularity-images from SingularityHub 29 | tags: [containers, singularity] 30 | vars: 31 | IMAGE: singularity-images.sif 32 | run: | 33 | singularity pull $IMAGE shub://vsoch/singularity-images 34 | singularity inspect $IMAGE 35 | -------------------------------------------------------------------------------- /general_tests/containers/singularity/run.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | runImage: 4 | type: script 5 | executor: generic.local.bash 6 | description: run container docker://godlovedc/lolcow 7 | tags: [containers, singularity] 8 | run: | 9 | singularity run docker://godlovedc/lolcow 10 | singularity exec docker://godlovedc/lolcow fortune -------------------------------------------------------------------------------- /general_tests/sched/lsf/bhosts.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | display_lsf_hosts: 4 | description: Show all hosts in LSF cluster 5 | type: script 6 | executor: generic.local.bash 7 | tags: lsf 8 | run: bhosts 9 | 10 | display_hosts_format: 11 | description: Show all hosts with column hostname and status 12 | type: script 13 | executor: generic.local.bash 14 | tags: lsf 15 | run: bhosts -o 'host_name status' 16 | 17 | bhosts_version: 18 | description: display version from bhosts command 19 | type: script 20 | executor: generic.local.bash 21 | tags: lsf 22 | run: bhosts -V -------------------------------------------------------------------------------- /general_tests/sched/lsf/bmgroups.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | show_host_groups: 4 | type: script 5 | executor: generic.local.bash 6 | description: Show information about host groups using bmgroup 7 | tags: lsf 8 | run: bmgroup -------------------------------------------------------------------------------- /general_tests/sched/lsf/bqueues.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | show_lsf_queues: 4 | type: script 5 | executor: generic.local.bash 6 | description: Show LSF queues 7 | tags: lsf 8 | run: bqueues 9 | 10 | show_lsf_queues_formatted: 11 | type: script 12 | executor: generic.local.bash 13 | description: Show LSF queues with formatted columns 14 | tags: lsf 15 | run: bqueues -o 'queue_name description priority status' 16 | 17 | show_lsf_queues_current_user: 18 | type: script 19 | executor: generic.local.bash 20 | description: Show LSF queues available for current user 21 | tags: lsf 22 | run: bqueues -u $USER 23 | 24 | -------------------------------------------------------------------------------- /general_tests/sched/lsf/bugroup.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | show_lsf_user_groups: 4 | type: script 5 | executor: generic.local.bash 6 | description: Show information about all LSF user groups 7 | tags: lsf 8 | run: bugroup -------------------------------------------------------------------------------- /general_tests/sched/lsf/lsinfo.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | show_lsf_configuration: 4 | type: script 5 | executor: generic.local.bash 6 | description: Show LSF configuration using lsinfo 7 | tags: lsf 8 | run: lsinfo 9 | 10 | show_lsf_models: 11 | type: script 12 | executor: generic.local.bash 13 | description: Show information about host models in LSF cluster 14 | tags: lsf 15 | run: lsinfo -m 16 | 17 | show_lsf_resources: 18 | type: script 19 | executor: generic.local.bash 20 | description: Show information about LSF resources 21 | tags: lsf 22 | run: lsinfo -r 23 | 24 | lsf_version: 25 | type: script 26 | executor: generic.local.bash 27 | description: Display lsf version using lsinfo 28 | tags: lsf 29 | run: lsinfo -V -------------------------------------------------------------------------------- /general_tests/sched/pbs/hostname.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | pbs_sleep: 4 | type: script 5 | executor: generic.pbs.workq 6 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 7 | run: sleep 10 -------------------------------------------------------------------------------- /general_tests/sched/slurm/sacctmgr.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | show_accounts: 3 | executor: generic.local.bash 4 | type: script 5 | description: run sacctmgr list accounts 6 | tags: [slurm] 7 | run: sacctmgr list accounts 8 | 9 | show_users: 10 | executor: generic.local.bash 11 | type: script 12 | description: run sacctmgr list users 13 | tags: [slurm] 14 | run: | 15 | sacctmgr list users 16 | sacctmgr list users $USER 17 | 18 | show_qos: 19 | executor: generic.local.bash 20 | type: script 21 | description: run sacctmgr list qos 22 | tags: [slurm] 23 | run: sacctmgr list qos 24 | 25 | show_tres: 26 | executor: generic.local.bash 27 | description: run sacctmgr list tres 28 | type: script 29 | tags: [slurm] 30 | run: sacctmgr list tres 31 | 32 | -------------------------------------------------------------------------------- /general_tests/sched/slurm/scontrol.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | slurm_config: 3 | type: script 4 | executor: generic.local.bash 5 | description: run scontrol show config 6 | tags: [slurm] 7 | run: scontrol show config 8 | 9 | show_partition: 10 | type: script 11 | executor: generic.local.bash 12 | description: run scontrol show partition 13 | tags: [slurm] 14 | run: scontrol show partition 15 | -------------------------------------------------------------------------------- /general_tests/sched/slurm/squeue.yml: -------------------------------------------------------------------------------- 1 | 2 | buildspecs: 3 | current_user_queue: 4 | executor: generic.local.bash 5 | type: script 6 | description: show all current pending jobs for current user (squeue -u $USER) 7 | tags: [slurm] 8 | run: squeue -u $USER 9 | 10 | show_all_jobs: 11 | executor: generic.local.bash 12 | type: script 13 | description: show all pending + running jobs (squeue -a) 14 | tags: [slurm] 15 | run: squeue --all 16 | -------------------------------------------------------------------------------- /logos/BuildTest_Primary_Center_3x1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/logos/BuildTest_Primary_Center_3x1.png -------------------------------------------------------------------------------- /logos/BuildTest_Primary_Center_4x3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/logos/BuildTest_Primary_Center_4x3.png -------------------------------------------------------------------------------- /logos/BuildTest_Primary_Left_3x1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/logos/BuildTest_Primary_Left_3x1.png -------------------------------------------------------------------------------- /logos/BuildTest_Primary_Right_3x1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/logos/BuildTest_Primary_Right_3x1.png -------------------------------------------------------------------------------- /logos/buildtest_secondary.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/logos/buildtest_secondary.jpg -------------------------------------------------------------------------------- /perlmutter_tutorial/ex1/.solution.txt: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | test_lmod_version: 3 | type: script 4 | executor: perlmutter.local.bash 5 | run: echo $LMOD_VERSION 6 | status: 7 | regex: 8 | stream: stdout 9 | exp: '^8.7.15$' -------------------------------------------------------------------------------- /perlmutter_tutorial/ex1/module_version.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | test_lmod_version: 3 | type: FIXME 4 | executors: 'perlmutter.local.bash' 5 | run: echo $LMOD_VERSION -------------------------------------------------------------------------------- /perlmutter_tutorial/ex2/.solution.sh: -------------------------------------------------------------------------------- 1 | #1 2 | buildtest buildspec find --tags 3 | 4 | #2 5 | buildtest buildspec find --helpfilter 6 | buildtest buildspec find --helpformat 7 | 8 | #3 9 | buildtest buildspec find --format name,description 10 | 11 | #4 12 | buildtest buildspec find --filter tags=e4s 13 | 14 | #5 15 | buildtest buildspec find invalid 16 | 17 | #6 18 | buildtest buildspec validate -t e4s 19 | 20 | #7 21 | buildtest buildspec show hello_world_openmp -------------------------------------------------------------------------------- /perlmutter_tutorial/ex3/.solution.sh: -------------------------------------------------------------------------------- 1 | #1 2 | buildtest report --helpfilter 3 | buildtest report --helpformat 4 | 5 | #2 6 | buildtest report --filter returncode=0 7 | 8 | #3 9 | buildtest report --filter tags=e4s 10 | 11 | #4 12 | buildtest report --fail --row-count 13 | -------------------------------------------------------------------------------- /perlmutter_tutorial/ex4/.solution.txt: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | stream_test: 3 | type: script 4 | executor: perlmutter.local.bash 5 | description: Run stream test 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: float 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | scale: 20 | type: float 21 | regex: 22 | exp: 'Scale:\s+(\S+)\s+.*' 23 | stream: stdout 24 | item: 1 25 | status: 26 | assert_ge: 27 | comparisons: 28 | - name: copy 29 | ref: 8000 30 | - name: scale 31 | ref: 8000 -------------------------------------------------------------------------------- /perlmutter_tutorial/ex4/stream.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | stream_test: 3 | type: script 4 | executor: perlmutter.local.bash 5 | description: Run stream test 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: float 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | scale: 20 | type: float 21 | regex: 22 | exp: 'Scale:\s+(\S+)\s+.*' 23 | stream: stdout 24 | item: 1 -------------------------------------------------------------------------------- /perlmutter_tutorial/ex5/.solution.txt: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname_perlmutter: 3 | description: run hostname on perlmutter 4 | type: script 5 | executor: 'perlmutter.slurm.(regular|debug)' 6 | tags: ["queues","jobs"] 7 | sbatch: ["-t 5", "-n 1", "-N 1", "-C cpu"] 8 | run: hostname -------------------------------------------------------------------------------- /perlmutter_tutorial/ex5/hostname.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname_perlmutter: 3 | description: run hostname on perlmutter 4 | type: script 5 | executor: 'perlmutter.slurm.debug' 6 | tags: ["queues","jobs"] 7 | sbatch: ["-t 5", "-n 1", "-N 1", "-C cpu"] 8 | run: hostname -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | distro 2 | isort 3 | # https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0 deprecates jsonschema.RefResolver 4 | jsonschema>=4.18 5 | lmodule 6 | PyYAML>=5.2 7 | requests 8 | rich 9 | coverage 10 | pytest 11 | isort 12 | black==24.3.0 13 | pyflakes 14 | # https://urllib3.readthedocs.io/en/latest/v2-migration-guide.html 15 | urllib3>=2.1.0 16 | -------------------------------------------------------------------------------- /scripts/pbs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pbspro/pbspro:18.1 2 | ENV PBS_START_MOM=1 3 | RUN yum install -y which git wget make gcc gfortran csh tcsh && \ 4 | # https://tecadmin.net/install-python-3-7-on-centos/ python 3 installation 5 | yum install -y gcc openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel && \ 6 | wget https://www.python.org/ftp/python/3.7.11/Python-3.7.11.tgz && \ 7 | tar xzf Python-3.7.11.tgz && \ 8 | cd Python-3.7.11 && \ 9 | ./configure --enable-optimizations && \ 10 | make altinstall && \ 11 | /opt/pbs/bin/qmgr -c "create node pbs" && \ 12 | /opt/pbs/bin/qmgr -c "set node pbs queue=workq" && \ 13 | /opt/pbs/bin/qmgr -c "set server job_history_enable=True" 14 | 15 | LABEL org.opencontainers.image.authors="shahzebmsiddiqui@gmail.com" 16 | -------------------------------------------------------------------------------- /scripts/pbs/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget --no-check-certificate https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh 4 | bash ~/miniconda.sh -b -p ~/miniconda 5 | export PATH=~/miniconda/bin:$PATH 6 | conda create -n buildtest -y 7 | source activate buildtest 8 | source $HOME/buildtest/setup.sh 9 | export BUILDTEST_CONFIGFILE=$BUILDTEST_ROOT/tests/settings/pbs.yml 10 | -------------------------------------------------------------------------------- /scripts/run_scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | FILES=../docs/scripting_examples/*.py 3 | for f in $FILES 4 | do 5 | python $f > $f.out 6 | done 7 | -------------------------------------------------------------------------------- /scripts/spack_container/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/spack/tutorial:sc23 2 | 3 | # upgrade git version see https://github.com/actions/checkout/issues/238 and https://phoenixnap.com/kb/add-apt-repository-command-not-found-ubuntu 4 | USER root 5 | RUN apt update && \ 6 | apt-get update && \ 7 | # need to install following package in-order to get apt-get-repository 8 | apt install -y software-properties-common && \ 9 | add-apt-repository ppa:git-core/ppa -y && \ 10 | apt-get update && \ 11 | apt-get install -y tcsh zsh && \ 12 | apt-get install -y git 13 | 14 | USER spack 15 | RUN git clone --depth=100 --branch=releases/v0.21 https://github.com/spack/spack.git ~/spack 16 | 17 | COPY /spack_setup.sh /etc/profile.d/spack_setup.sh 18 | COPY /modules.yaml /home/spack/spack/etc/spack/modules.yaml 19 | COPY /compilers.yaml /home/spack/spack/etc/spack/compilers.yaml 20 | 21 | USER spack 22 | RUN cd ~/spack && \ 23 | . ~/spack/share/spack/setup-env.sh && \ 24 | spack tutorial -y && \ 25 | spack install lmod python gcc@12.3.0 openmpi mpich && \ 26 | . $(spack location -i lmod)/lmod/lmod/init/bash \ 27 | spack module tcl refresh --delete-tree -y 28 | 29 | -------------------------------------------------------------------------------- /scripts/spack_container/modules.yaml: -------------------------------------------------------------------------------- 1 | modules: 2 | default: 3 | roots: 4 | tcl: /home/spack/modules 5 | tcl: 6 | hash_length: 0 7 | projections: 8 | all: '{name}/{version}-{compiler.name}-{compiler.version}' -------------------------------------------------------------------------------- /scripts/spack_container/regtest-tutorial.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . /etc/profile 3 | export PATH=/home/spack/.local/bin:$PATH 4 | . /home/spack/buildtest/scripts/spack_container/setup.sh 5 | pip3 install -r "${BUILDTEST_ROOT}/docs/requirements.txt" &> /dev/null 6 | python "${BUILDTEST_ROOT}/buildtest/tools/unittests.py" -------------------------------------------------------------------------------- /scripts/spack_container/run-tutorial-examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script can be run as follows: docker run -v $BUILDTEST_ROOT:/home/spack/buildtest ghcr.io/buildtesters/buildtest_spack:latest bash /home/spack/buildtest/scripts/spack_container/run-tutorial-examples.sh 3 | source /etc/profile 4 | source /home/spack/buildtest/scripts/spack_container/setup.sh 5 | python /home/spack/buildtest/scripts/spack_container/doc-examples.py -------------------------------------------------------------------------------- /scripts/spack_container/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script is used to setup buildtest inside container. One should typically bind mount BUILDTEST_ROOT on host operating 3 | # system inside container as follows: 4 | # docker run -it -v $BUILDTEST_ROOT:/home/spack/buildtest ghcr.io/buildtesters/buildtest_spack:latest 5 | 6 | module load python 7 | 8 | # setup python environment 9 | python -m venv "$HOME/pyenv/buildtest" 10 | source "$HOME/pyenv/buildtest/bin/activate" 11 | 12 | cd ~/buildtest || { echo "Unable to cd to 'buildtest' directory"; exit 1;} 13 | # installing buildtest 14 | source setup.sh 15 | 16 | export BUILDTEST_CONFIGFILE=$BUILDTEST_ROOT/buildtest/settings/spack_container.yml 17 | -------------------------------------------------------------------------------- /scripts/spack_container/spack_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | . /home/spack/spack/share/spack/setup-env.sh 3 | . "$(spack location -i lmod)/lmod/lmod/init/bash" 4 | spack module tcl refresh --delete-tree -y 5 | module use /home/spack/modules/linux-ubuntu22.04-x86_64_v3 6 | 7 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/__init__.py -------------------------------------------------------------------------------- /tests/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/builders/__init__.py -------------------------------------------------------------------------------- /tests/builders/assert_eq.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_eq.yml -------------------------------------------------------------------------------- /tests/builders/assert_ge.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_ge.yml -------------------------------------------------------------------------------- /tests/builders/assert_gt.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_gt.yml -------------------------------------------------------------------------------- /tests/builders/assert_le.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_le.yml -------------------------------------------------------------------------------- /tests/builders/assert_lt.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_lt.yml -------------------------------------------------------------------------------- /tests/builders/assert_ne.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_ne.yml -------------------------------------------------------------------------------- /tests/builders/assert_range.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/assert_range.yml -------------------------------------------------------------------------------- /tests/builders/contains.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/perf_checks/contains.yml -------------------------------------------------------------------------------- /tests/builders/exists.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/exists.yml -------------------------------------------------------------------------------- /tests/builders/file_and_dir_check.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_and_dir_check.yml -------------------------------------------------------------------------------- /tests/builders/file_count.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_count.yml -------------------------------------------------------------------------------- /tests/builders/file_count_file_traverse_limit.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_count_file_traverse_limit.yml -------------------------------------------------------------------------------- /tests/builders/file_count_filetype.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_count_filetype.yml -------------------------------------------------------------------------------- /tests/builders/file_count_pattern.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_count_pattern.yml -------------------------------------------------------------------------------- /tests/builders/file_linecount.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_linecount.yml -------------------------------------------------------------------------------- /tests/builders/file_linecount_failure.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_linecount_failure.yml -------------------------------------------------------------------------------- /tests/builders/file_linecount_invalid.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/file_linecount_invalid.yml -------------------------------------------------------------------------------- /tests/builders/is_symlink.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/is_symlink.yml -------------------------------------------------------------------------------- /tests/builders/linecount.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/linecount.yml -------------------------------------------------------------------------------- /tests/builders/metrics_file_regex_with_invalid_linenum.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/metrics/metrics_file_regex_with_invalid_linenum.yml -------------------------------------------------------------------------------- /tests/builders/metrics_file_regex_with_linenum.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/metrics/metrics_file_regex_with_linenum.yml -------------------------------------------------------------------------------- /tests/builders/metrics_regex_with_invalid_linenum.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/metrics/metrics_regex_with_invalid_linenum.yml -------------------------------------------------------------------------------- /tests/builders/metrics_regex_with_linenum.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/metrics/metrics_regex_with_linenum.yml -------------------------------------------------------------------------------- /tests/builders/metrics_with_regex_type.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/metrics/metrics_with_regex_type.yml -------------------------------------------------------------------------------- /tests/builders/post_run.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/post_run.yml -------------------------------------------------------------------------------- /tests/builders/regex_on_filename.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/regex_on_filename.yml -------------------------------------------------------------------------------- /tests/builders/regex_on_invalids.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | regex_on_invalid_fname: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test regex on an invalid file name 6 | run: echo "Hello" > hello.txt 7 | status: 8 | file_regex: 9 | - file: /fox.txt 10 | exp: '^(Hello)$' 11 | 12 | file_regex_with_invalid_expression: 13 | type: script 14 | executor: generic.local.bash 15 | description: Test regex on a valid file where regex is invaild 16 | run: echo "Jump" > jump.txt 17 | status: 18 | file_regex: 19 | - file: jump.txt 20 | exp: '^(Hello)$' 21 | -------------------------------------------------------------------------------- /tests/builders/runtime_status_test.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/runtime_status_test.yml -------------------------------------------------------------------------------- /tests/builders/specify_regex_type.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/specify_regex_type.yml -------------------------------------------------------------------------------- /tests/builders/status_regex.yml: -------------------------------------------------------------------------------- 1 | ../../tutorials/test_status/status_regex.yml -------------------------------------------------------------------------------- /tests/builders/stream_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | stream_openmp_c: 3 | executor: generic.local.bash 4 | type: script 5 | description: "STREAM Microbenchmark C Test with OpenMP" 6 | tags: ["benchmark"] 7 | compilers: 8 | name: ['^(builtin_gcc)'] 9 | default: 10 | gcc: 11 | cflags: -fopenmp -O2 12 | env: 13 | OMP_NUM_THREADS: 8 14 | run: | 15 | wget https://www.cs.virginia.edu/stream/FTP/Code/stream.c 16 | $BUILDTEST_CC $BUILDTEST_CFLAGS stream.c -o stream 17 | ./stream 18 | -------------------------------------------------------------------------------- /tests/buildsystem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/buildsystem/__init__.py -------------------------------------------------------------------------------- /tests/buildsystem/invalid_buildspecs/invalid_executor.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_executor: 3 | executor: not_valid_executor_name 4 | type: script 5 | run: hostname 6 | -------------------------------------------------------------------------------- /tests/buildsystem/invalid_buildspecs/invalid_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | type_mismatch: 3 | executor: generic.local.sh 4 | type: badvalue 5 | run: hostname 6 | -------------------------------------------------------------------------------- /tests/buildsystem/invalid_buildspecs/missing_executor.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_executor: 3 | type: script 4 | run: hostname 5 | -------------------------------------------------------------------------------- /tests/buildsystem/invalid_buildspecs/missing_type.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | missing_type: 3 | executor: generic.local.sh 4 | run: hostname 5 | -------------------------------------------------------------------------------- /tests/buildsystem/valid_builds/sched_directives.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | slurm_hostname: 3 | type: script 4 | executor: generic.local.bash 5 | description: Slurm batch generation check 6 | sbatch: ["-n 1", "-N 1", "-t 10"] 7 | run: hostname 8 | 9 | lsf_hostname: 10 | type: script 11 | executor: generic.local.bash 12 | description: LSF batch generation check 13 | bsub: ["-n 1", "-W 10"] 14 | run: hostname 15 | 16 | pbs_hostname: 17 | type: script 18 | executor: generic.local.bash 19 | description: PBS batch generation check 20 | cobalt: ["-l nodes=1", "-l walltime=10"] 21 | run: hostname 22 | -------------------------------------------------------------------------------- /tests/buildsystem/valid_buildspecs/environment.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | # Shell would be accepted to indicate a single line shell command (or similar) 3 | hello_dinosaur: 4 | executor: generic.local.bash 5 | type: script 6 | env: 7 | FIRST_NAME: avocado 8 | LAST_NAME: dinosaur 9 | run: printf "${FIRST_NAME} ${LAST_NAME}\n" 10 | -------------------------------------------------------------------------------- /tests/buildsystem/valid_buildspecs/python-shell.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | circle_area: 3 | executor: generic.local.bash 4 | type: script 5 | shell: python 6 | description: "Calculate circle of area given a radius" 7 | run: | 8 | import math 9 | radius = 2 10 | area = math.pi * radius * radius 11 | print("Circle Radius ", radius) 12 | print("Area of circle ", area) 13 | -------------------------------------------------------------------------------- /tests/buildsystem/valid_buildspecs/shell_examples.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | _bin_sh_shell: 3 | executor: generic.local.sh 4 | type: script 5 | description: "/bin/sh shell example" 6 | shell: /bin/sh 7 | run: "bzip2 --help" 8 | 9 | _bin_bash_shell: 10 | executor: generic.local.bash 11 | type: script 12 | description: "/bin/bash shell example" 13 | shell: /bin/bash 14 | run: "bzip2 -h" 15 | 16 | bash_shell: 17 | executor: generic.local.bash 18 | type: script 19 | description: "bash shell example" 20 | shell: bash 21 | run: "echo $SHELL" 22 | 23 | sh_shell: 24 | executor: generic.local.sh 25 | type: script 26 | description: "sh shell example" 27 | shell: sh 28 | run: "echo $SHELL" 29 | 30 | shell_options: 31 | executor: generic.local.sh 32 | type: script 33 | description: "shell options" 34 | shell: "sh -x" 35 | run: | 36 | echo $SHELL 37 | hostname 38 | -------------------------------------------------------------------------------- /tests/buildsystem/valid_buildspecs/slurm.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | slurm_down_nodes_reason: 3 | executor: generic.local.bash 4 | type: "script" 5 | description: "Show SLURM nodes that are down or drain with a reason" 6 | run: "sinfo -R" 7 | 8 | slurm_not_responding_nodes: 9 | executor: generic.local.bash 10 | type: "script" 11 | description: "Show a list of slurm nodes not responding" 12 | run: "sinfo -d" 13 | -------------------------------------------------------------------------------- /tests/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/cli/__init__.py -------------------------------------------------------------------------------- /tests/cli/cdash_examples/invalid_project.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: 4 | - .* 5 | description: Generic System 6 | moduletool: N/A 7 | file_traversal_limit: 1000 8 | cdash: 9 | url: "https://my.cdash.org" 10 | project: INVALID-PROJECT 11 | site: laptop 12 | executors: 13 | local: 14 | bash: 15 | description: submit jobs on local machine using bash shell 16 | shell: bash 17 | sh: 18 | description: submit jobs on local machine using sh shell 19 | shell: sh 20 | csh: 21 | description: submit jobs on local machine using csh shell 22 | shell: csh 23 | zsh: 24 | description: submit jobs on local machine using zsh shell 25 | shell: zsh 26 | python: 27 | description: submit jobs on local machine using python shell 28 | shell: python 29 | compilers: 30 | find: 31 | gcc: "^(gcc)" 32 | compiler: 33 | gcc: 34 | builtin_gcc: 35 | cc: /usr/bin/gcc 36 | fc: /usr/bin/gfortran 37 | cxx: /usr/bin/g++ 38 | -------------------------------------------------------------------------------- /tests/cli/cdash_examples/invalid_url.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: 4 | - .* 5 | description: Generic System 6 | moduletool: N/A 7 | file_traversal_limit: 1000 8 | cdash: 9 | url: "https://my.cdash.XYZ.org" 10 | project: buildtest 11 | site: laptop 12 | executors: 13 | local: 14 | bash: 15 | description: submit jobs on local machine using bash shell 16 | shell: bash 17 | sh: 18 | description: submit jobs on local machine using sh shell 19 | shell: sh 20 | csh: 21 | description: submit jobs on local machine using csh shell 22 | shell: csh 23 | zsh: 24 | description: submit jobs on local machine using zsh shell 25 | shell: zsh 26 | python: 27 | description: submit jobs on local machine using python shell 28 | shell: python 29 | compilers: 30 | find: 31 | gcc: "^(gcc)" 32 | compiler: 33 | gcc: 34 | builtin_gcc: 35 | cc: /usr/bin/gcc 36 | fc: /usr/bin/gfortran 37 | cxx: /usr/bin/g++ 38 | -------------------------------------------------------------------------------- /tests/cli/configuration/invalid_executors.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: 4 | - .* 5 | description: Generic System 6 | moduletool: none 7 | file_traversal_limit: 1000 8 | cdash: 9 | url: https://my.cdash.org 10 | project: buildtest 11 | site: laptop 12 | buildspecs: 13 | rebuild: False 14 | count: 15 15 | format: "name,description" 16 | terse: False 17 | report: 18 | count: 25 19 | format: "name,id,state,runtime,returncode" 20 | executors: 21 | local: 22 | bash: 23 | description: submit jobs on local machine using bash shell 24 | shell: bash 25 | zsh: 26 | description: submit jobs on local machine using zsh shell 27 | shell: zsh123 28 | 29 | compilers: 30 | find: 31 | gcc: "^(gcc)" 32 | compiler: 33 | gcc: 34 | builtin_gcc: 35 | cc: gcc 36 | fc: gfortran 37 | cxx: g++ 38 | -------------------------------------------------------------------------------- /tests/cli/test_commands.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.commands import list_buildtest_commands 2 | 3 | 4 | def test_buildtest_commands(): 5 | list_buildtest_commands() 6 | 7 | list_buildtest_commands(with_aliases=True) 8 | -------------------------------------------------------------------------------- /tests/cli/test_compilers/invalid_moduletool.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: [".*"] 4 | moduletool: none 5 | buildspecs: 6 | rebuild: False 7 | count: 15 8 | format: "name,description" 9 | terse: False 10 | report: 11 | count: 25 12 | format: "name,id,state,runtime,returncode" 13 | executors: 14 | local: 15 | bash: 16 | description: submit jobs on local machine using bash shell 17 | shell: bash 18 | sh: 19 | description: submit jobs on local machine using sh shell 20 | shell: sh 21 | csh: 22 | description: submit jobs on local machine using csh shell 23 | shell: csh 24 | python: 25 | description: submit jobs on local machine using python shell 26 | shell: python 27 | compilers: 28 | compiler: 29 | gcc: 30 | builtin_gcc: 31 | cc: /usr/bin/gcc 32 | fc: /usr/bin/gfortran 33 | cxx: /usr/bin/g++ 34 | -------------------------------------------------------------------------------- /tests/cli/test_compilers/missing_compiler_find.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: [".*"] 4 | moduletool: lmod 5 | buildspecs: 6 | rebuild: False 7 | count: 15 8 | format: "name,description" 9 | terse: False 10 | report: 11 | count: 25 12 | format: "name,id,state,runtime,returncode" 13 | executors: 14 | local: 15 | bash: 16 | description: submit jobs on local machine using bash shell 17 | shell: bash 18 | sh: 19 | description: submit jobs on local machine using sh shell 20 | shell: sh 21 | csh: 22 | description: submit jobs on local machine using csh shell 23 | shell: csh 24 | python: 25 | description: submit jobs on local machine using python shell 26 | shell: python 27 | compilers: 28 | compiler: 29 | gcc: 30 | builtin_gcc: 31 | cc: /usr/bin/gcc 32 | fc: /usr/bin/gfortran 33 | cxx: /usr/bin/g++ 34 | -------------------------------------------------------------------------------- /tests/cli/test_debugreport.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.debugreport import print_debug_report 2 | from buildtest.config import SiteConfiguration 3 | from buildtest.system import BuildTestSystem 4 | 5 | 6 | def test_debug_report(): 7 | system = BuildTestSystem() 8 | 9 | config = SiteConfiguration() 10 | print_debug_report(system, config) 11 | -------------------------------------------------------------------------------- /tests/cli/test_helpcolor.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.helpcolor import print_available_colors 2 | 3 | 4 | def test_buildtest_helpcolor(): 5 | print_available_colors() 6 | -------------------------------------------------------------------------------- /tests/cli/test_info.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.info import buildtest_info 2 | from buildtest.config import SiteConfiguration 3 | from buildtest.system import BuildTestSystem 4 | 5 | 6 | def test_buildtest_info(): 7 | config = SiteConfiguration() 8 | config.detect_system() 9 | config.validate() 10 | 11 | system = BuildTestSystem() 12 | buildtest_info(configuration=config, buildtest_system=system) 13 | -------------------------------------------------------------------------------- /tests/cli/test_show.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.show import buildtest_show 2 | 3 | 4 | def test_buildtest_show(): 5 | buildtest_show(command="build") 6 | buildtest_show(command="buildspec") 7 | buildtest_show(command="config") 8 | buildtest_show(command="cdash") 9 | buildtest_show(command="path") 10 | buildtest_show(command="history") 11 | buildtest_show(command="inspect") 12 | buildtest_show(command="report") 13 | buildtest_show(command="schema") 14 | buildtest_show(command="stylecheck") 15 | buildtest_show(command="unittests") 16 | -------------------------------------------------------------------------------- /tests/examples/pbs/hold.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | pbs_hold_job: 3 | type: script 4 | executor: generic.pbs.workq 5 | description: PBS Hold Job 6 | pbs: ["-l nodes=1", "-l walltime=00:02:00", "-h"] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /tests/examples/pbs/job_dependency.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname: 3 | type: script 4 | executor: generic.pbs.workq 5 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 6 | run: hostname 7 | 8 | whoami: 9 | type: script 10 | executor: generic.pbs.workq 11 | needs: [hostname] 12 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 13 | run: whoami 14 | -------------------------------------------------------------------------------- /tests/examples/pbs/multiple_tests.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | test1: 3 | type: script 4 | executor: generic.pbs.workq 5 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 6 | run: | 7 | date 8 | whoami 9 | sleep 10 10 | hostname -f 11 | 12 | test2: 13 | type: script 14 | executor: generic.pbs.workq 15 | pbs: ["-l nodes=1", "-l walltime=00:01:00"] 16 | run: echo "This is a sample job 2" 17 | 18 | test3: 19 | type: script 20 | executor: generic.pbs.workq 21 | pbs: ["-l nodes=1", "-l walltime=00:01:00"] 22 | run: echo "This is a sample job 3" 23 | -------------------------------------------------------------------------------- /tests/examples/pbs/pbs_job_state.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | pbs_sleep: 3 | type: script 4 | description: pass test based on PBS job state. 5 | executor: generic.pbs.workq 6 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 7 | run: sleep 5 8 | status: 9 | pbs_job_state: "H" 10 | -------------------------------------------------------------------------------- /tests/examples/pbs/sleep.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | pbs_sleep: 3 | type: script 4 | executor: generic.pbs.workq 5 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 6 | run: sleep 5 7 | -------------------------------------------------------------------------------- /tests/examples/perlmutter/hold_job.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hold_job: 3 | type: script 4 | executor: '(muller|perlmutter).slurm.debug' 5 | description: "Hold job will be cancelled by scheduler after maxpendtime reached" 6 | sbatch: ["-t 5", "-N 1", "-H", "-C cpu"] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /tests/examples/perlmutter/hostname.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname: 3 | type: script 4 | executor: '(muller|perlmutter).slurm.debug' 5 | description: "Run hostname via slurm debug queue" 6 | sbatch: ["-t 5", "-n 1", "-C cpu"] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /tests/examples/summit/hold_job.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hold_job: 3 | type: script 4 | executor: summit.lsf.batch 5 | description: Test job cancellation by holding job 6 | tags: [jobs] 7 | bsub: ["-W 10", "-nnodes 1", "-H"] 8 | run: jsrun hostname 9 | -------------------------------------------------------------------------------- /tests/examples/summit/hostname.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hostname: 3 | type: script 4 | executor: summit.lsf.batch 5 | description: Run hostname in batch job 6 | tags: [jobs] 7 | bsub: ["-W 10", "-nnodes 1"] 8 | run: jsrun hostname 9 | -------------------------------------------------------------------------------- /tests/examples/summit/lsf_job_state.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | lsf_job_state_example: 3 | type: script 4 | executor: summit.lsf.batch 5 | description: This job will only PASS if LSF Job state is EXIT 6 | tags: [jobs] 7 | bsub: ["-W 10", "-nnodes 1"] 8 | run: jsrun hostname 9 | status: 10 | lsf_job_state: EXIT 11 | -------------------------------------------------------------------------------- /tests/examples/torque/sleep.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | sleep_job: 3 | type: script 4 | executor: generic.torque.lbl 5 | pbs: ["-l nodes=1", "-l walltime=00:02:00"] 6 | run: sleep 5 7 | -------------------------------------------------------------------------------- /tests/examples/torque/sleep_cancel.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | sleep_job_cancel: 3 | type: script 4 | executor: generic.torque.lbl 5 | pbs: ["-l nodes=1", "-l walltime=00:00:05", "-h"] 6 | run: sleep 60 7 | -------------------------------------------------------------------------------- /tests/executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/executors/__init__.py -------------------------------------------------------------------------------- /tests/schema_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/schema_tests/__init__.py -------------------------------------------------------------------------------- /tests/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/schemas/__init__.py -------------------------------------------------------------------------------- /tests/settings/pbs.yml: -------------------------------------------------------------------------------- 1 | system: 2 | generic: 3 | hostnames: ['.*'] 4 | moduletool: none 5 | poolsize: 8 6 | max_jobs: 8 7 | pager: False 8 | file_traversal_limit: 1000 9 | buildspecs: 10 | rebuild: False 11 | count: 15 12 | format: "name,description" 13 | terse: False 14 | report: 15 | count: 25 16 | format: "name,id,state,runtime,returncode" 17 | executors: 18 | defaults: 19 | pollinterval: 5 20 | maxpendtime: 30 21 | local: 22 | bash: 23 | description: submit jobs on local machine using bash shell 24 | shell: bash 25 | sh: 26 | description: submit jobs on local machine using sh shell 27 | shell: sh 28 | pbs: 29 | workq: 30 | queue: workq 31 | compilers: 32 | compiler: 33 | gcc: 34 | default: 35 | cc: /usr/bin/gcc 36 | cxx: /usr/bin/g++ 37 | fc: /usr/bin/gfortran 38 | -------------------------------------------------------------------------------- /tests/sitecustomize.py: -------------------------------------------------------------------------------- 1 | import coverage 2 | 3 | coverage.process_startup() 4 | -------------------------------------------------------------------------------- /tests/test_exceptions.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from buildtest.exceptions import BuildTestError 4 | 5 | 6 | class TestBuildTestError: 7 | @pytest.mark.utility 8 | @pytest.mark.xfail( 9 | reason="Testing to see if exception of type BuildTestError is raised", 10 | raises=BuildTestError, 11 | ) 12 | def test_exception(self): 13 | """This test will check if we can raise Exception of type BuildTestError""" 14 | error = BuildTestError("Failed to run command") 15 | assert hasattr(error, "msg") 16 | assert isinstance(error.msg, str) 17 | raise error 18 | 19 | @pytest.mark.utility 20 | @pytest.mark.xfail( 21 | reason="Testing to see if exception of type BuildTestError is raised", 22 | raises=BuildTestError, 23 | ) 24 | def test_multi_args(self): 25 | """This test will check if we can raise Exception of type BuildTestError""" 26 | error = BuildTestError("This is", "an", "exception") 27 | assert isinstance(error.msg, str) 28 | raise error 29 | -------------------------------------------------------------------------------- /tests/test_log.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | 3 | from buildtest.log import init_logfile 4 | from buildtest.utils.file import read_file 5 | 6 | 7 | def test_BuildTestLogger(): 8 | tf = tempfile.NamedTemporaryFile() 9 | 10 | logger = init_logfile(tf.name) 11 | 12 | # ensure we have effective level of 10 (DEBUG) 13 | assert logger.getEffectiveLevel() == 10 14 | 15 | # tf = tempfile.NamedTemporaryFile() 16 | 17 | assert logger.name == "buildtest" 18 | 19 | # writing message at each log level 20 | logger.debug("DEBUG MESSAGE") 21 | logger.info("INFO MESSAGE") 22 | logger.warning("WARNING MESSAGE!") 23 | logger.error("ERROR MESSAGE!!") 24 | logger.critical("CRITICAL MESSAGE!!!") 25 | content = read_file(tf.name) 26 | print(content) 27 | -------------------------------------------------------------------------------- /tests/test_pbs.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from buildtest.cli.build import BuildTest 6 | from buildtest.config import SiteConfiguration 7 | from buildtest.scheduler.detection import PBS 8 | from buildtest.utils.file import walk_tree 9 | 10 | 11 | def test_pbs(): 12 | """Need to figure out a PBS environment where to run this regression test.""" 13 | pbs = PBS() 14 | if not pbs.active(): 15 | pytest.skip("Test runs only on PBS Cluster") 16 | 17 | here = os.path.dirname(os.path.abspath(__file__)) 18 | settings_file = os.path.join(here, "settings", "pbs.yml") 19 | 20 | bc = SiteConfiguration(settings_file) 21 | bc.detect_system() 22 | bc.validate() 23 | 24 | buildspec_files = walk_tree(os.path.join(here, "examples", "pbs")) 25 | 26 | cmd = BuildTest(configuration=bc, buildspecs=buildspec_files) 27 | cmd.build() 28 | 29 | cmd = BuildTest( 30 | configuration=bc, 31 | buildspecs=[os.path.join(here, "examples", "pbs", "sleep.yml")], 32 | numprocs=[1, 2, 4], 33 | poll_interval=5, 34 | ) 35 | cmd.build() 36 | -------------------------------------------------------------------------------- /tests/test_stats.py: -------------------------------------------------------------------------------- 1 | from buildtest.cli.build import BuildTest 2 | from buildtest.cli.buildspec import BuildspecCache 3 | from buildtest.cli.report import Report 4 | from buildtest.cli.stats import stats_cmd 5 | from buildtest.config import SiteConfiguration 6 | 7 | 8 | def test_stats(): 9 | configuration = SiteConfiguration() 10 | configuration.detect_system() 11 | configuration.validate() 12 | 13 | BuildspecCache(rebuild=True, configuration=configuration) 14 | 15 | cmd = BuildTest(configuration=configuration, tags=["pass"]) 16 | cmd.build() 17 | 18 | report = Report(configuration=configuration) 19 | name = report.get_random_tests() 20 | stats_cmd(name[0], configuration=configuration) 21 | -------------------------------------------------------------------------------- /tests/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/tools/__init__.py -------------------------------------------------------------------------------- /tests/tools/test_editor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from buildtest.tools.editor import set_editor 5 | 6 | 7 | def test_editor(): 8 | set_editor("vi") 9 | set_editor("vim") 10 | set_editor("emacs") 11 | set_editor("nano") 12 | 13 | # an invalid editor will force this method to return full path to default editor which is 'vi' 14 | assert set_editor("notepad") == shutil.which("vi") 15 | 16 | os.environ["EDITOR"] = "notepad" 17 | assert set_editor("notepad") == shutil.which("vi") 18 | -------------------------------------------------------------------------------- /tests/tools/test_stylecheck.py: -------------------------------------------------------------------------------- 1 | from buildtest.tools.stylecheck import run_style_checks 2 | 3 | 4 | def test_run_style_check(): 5 | run_style_checks( 6 | no_black=False, no_isort=False, no_pyflakes=False, apply_stylechecks=False 7 | ) 8 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildtesters/buildtest/e55a77625bc1dca54694153d61dd717a12ec1872/tests/utils/__init__.py -------------------------------------------------------------------------------- /tests/utils/test_command.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from buildtest.exceptions import BuildTestError 4 | from buildtest.utils.command import BuildTestCommand 5 | 6 | 7 | @pytest.mark.utility 8 | class TestBuildTestCommand: 9 | def test_command(self): 10 | cmd = "hostname" 11 | a = BuildTestCommand(cmd) 12 | a.execute() 13 | 14 | out, err, ret = a.get_output(), a.get_error(), a.returncode() 15 | 16 | print(f"Command: {cmd}") 17 | print(f"Output: {out}") 18 | print(f"Error: {err}") 19 | print(f"Return Code: {ret}") 20 | assert 0 == ret 21 | 22 | def test_error_command(self): 23 | cmd = "xyz" 24 | a = BuildTestCommand(cmd) 25 | a.execute() 26 | 27 | out, err, ret = a.get_output(), a.get_error(), a.returncode() 28 | 29 | print(f"Command: {cmd}") 30 | print(f"Output: {out}") 31 | print(f"Error: {err}") 32 | print(f"Return Code: {ret}") 33 | 34 | assert 0 != ret 35 | 36 | def test_invalid_type(self): 37 | query = ["hostname"] 38 | # input type must be a string otherwise it will raise an exception 39 | with pytest.raises(BuildTestError): 40 | BuildTestCommand(query) 41 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36,py37,py38,report 3 | 4 | [testenv] 5 | deps = 6 | pytest 7 | pytest-cov 8 | -rrequirements.txt 9 | skip_install = true 10 | commands = pytest --cov --cov-append -vra tests 11 | 12 | [testenv:report] 13 | deps = coverage 14 | skip_install = true 15 | commands = coverage report -m 16 | -------------------------------------------------------------------------------- /tutorials/add_numbers.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | add_numbers: 3 | type: script 4 | executor: generic.local.bash 5 | description: Add X+Y 6 | tags: [tutorials] 7 | vars: 8 | X: 1 9 | Y: 2 10 | run: echo "$X+$Y=" $(($X+$Y)) 11 | -------------------------------------------------------------------------------- /tutorials/burstbuffer_datawarp_executors.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | create_burst_buffer_executors: 3 | type: script 4 | executor: "generic.local.(sh|bash)" 5 | sbatch: ["-N 1", "-t 10", "-C knl"] 6 | description: Create a burst buffer for multiple executors 7 | tags: [jobs] 8 | executors: 9 | generic.local.sh: 10 | BB: 11 | - create_persistent name=buffer1 capacity=10GB access_mode=striped type=scratch 12 | DW: 13 | - persistentdw name=buffer1 14 | generic.local.bash: 15 | BB: 16 | - create_persistent name=buffer2 capacity=10GB access_mode=striped type=scratch 17 | DW: 18 | - persistentdw name=buffer2 19 | run: hostname 20 | -------------------------------------------------------------------------------- /tutorials/compilation/compiler_exclude.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | compiler_exclude_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Example excluding a particular compiler 6 | compilers: 7 | name: ['gcc'] 8 | exclude: ['gcc_6.5.0'] 9 | run: | 10 | $BUILDTEST_CC hello.c -o hello_c 11 | ./hello_c 12 | -------------------------------------------------------------------------------- /tutorials/compilation/hello.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | printf("Hello, World in C\n"); 5 | return 0; 6 | } 7 | -------------------------------------------------------------------------------- /tutorials/compilation/hello.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { 4 | std::cout << "Hello, World in C++" << std::endl; 5 | return 0; 6 | } 7 | -------------------------------------------------------------------------------- /tutorials/compilation/hello.f: -------------------------------------------------------------------------------- 1 | PROGRAM HelloWorld 2 | PRINT *, 'Hello, World in Fortran' 3 | END PROGRAM HelloWorld 4 | -------------------------------------------------------------------------------- /tutorials/compilation/hello_world_compilation.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_c_cpp: 3 | type: script 4 | executor: generic.local.bash 5 | description: Hello world compilation in C and C++ 6 | compilers: 7 | name: ['builtin_gcc'] 8 | run: | 9 | $BUILDTEST_CC hello.c -o hello_c 10 | $BUILDTEST_CXX hello.cpp -o hello_cpp 11 | ./hello_c 12 | ./hello_cpp 13 | -------------------------------------------------------------------------------- /tutorials/compilation/stream.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | stream_openmp_c: 3 | executor: generic.local.bash 4 | type: script 5 | description: "STREAM Microbenchmark C Test with OpenMP" 6 | tags: ["benchmark"] 7 | compilers: 8 | name: ['(builtin_gcc)'] 9 | default: 10 | gcc: 11 | cflags: -fopenmp -O2 12 | env: 13 | OMP_NUM_THREADS: 8 14 | run: | 15 | curl https://www.cs.virginia.edu/stream/FTP/Code/stream.c -o stream.c 16 | $BUILDTEST_CC $BUILDTEST_CFLAGS stream.c -o stream 17 | ./stream 18 | -------------------------------------------------------------------------------- /tutorials/containers/bind_mounts.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | bind_mount_in_container: 3 | type: script 4 | executor: generic.local.bash 5 | description: run a python script in container 6 | container: 7 | platform: "docker" 8 | image: ubuntu:latest 9 | mounts: "/tmp:/tmp" 10 | command: bash -c "echo 'hello world' > /tmp/hello.txt" 11 | run: | 12 | cat /tmp/hello.txt 13 | -------------------------------------------------------------------------------- /tutorials/containers/container_executor/python_container.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | python_container_example: 3 | type: script 4 | executor: generic.container.python 5 | description: run test in a python container 6 | run: | 7 | python -c "import sys; print(sys.version)" 8 | -------------------------------------------------------------------------------- /tutorials/containers/container_executor/ubuntu.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | ubuntu_container_example: 3 | type: script 4 | executor: generic.container.ubuntu 5 | description: run test in a container executor 6 | run: | 7 | 8 | echo "USER: " $(whoami) 9 | echo "HOMEDIR: " $HOME 10 | echo "WORKDIR: " $(pwd) 11 | df -h 12 | echo "************************" 13 | ls -l 14 | echo "************************" 15 | cat /etc/os-release 16 | 17 | -------------------------------------------------------------------------------- /tutorials/containers/hello_world.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_docker: 3 | type: script 4 | executor: generic.local.bash 5 | description: run hello-world container with docker 6 | container: 7 | platform: "docker" 8 | image: hello-world 9 | run: echo 'Test Complete!' 10 | -------------------------------------------------------------------------------- /tutorials/containers/hello_world_singularity.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world_singularity: 3 | type: script 4 | executor: generic.local.bash 5 | description: run hello-world container using singularity 6 | container: 7 | platform: "singularity" 8 | image: 'docker://hello-world' 9 | run: echo 'Test Complete!' -------------------------------------------------------------------------------- /tutorials/containers/run_commands.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | container_commands_ubuntu: 3 | type: script 4 | executor: generic.local.bash 5 | description: run arbitrary linux commands in ubuntu container 6 | container: 7 | platform: "docker" 8 | image: ubuntu:latest 9 | command: bash -c "cat /etc/os-release" 10 | run: | 11 | ls -l /etc/os-release || true 12 | 13 | container_options: 14 | type: script 15 | executor: generic.local.bash 16 | description: run arbitrary linux commands in ubuntu container 17 | container: 18 | platform: "docker" 19 | image: ubuntu:latest 20 | options: --hostname myhostname 21 | command: hostname 22 | run: hostname 23 | -------------------------------------------------------------------------------- /tutorials/containers/run_script.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | run_script_in_container: 3 | type: script 4 | executor: generic.local.bash 5 | description: run a python script in container 6 | container: 7 | platform: "docker" 8 | image: python:latest 9 | command: bash -c "python /buildtest/script.py" 10 | run: | 11 | python script.py -------------------------------------------------------------------------------- /tutorials/containers/script.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | print("Python version: ", sys.version) 4 | -------------------------------------------------------------------------------- /tutorials/csh_shell_examples.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | csh_shell: 3 | executor: generic.local.csh 4 | type: script 5 | description: "csh shell example" 6 | shell: csh 7 | tags: [tutorials] 8 | vars: 9 | file: "/etc/csh.cshrc" 10 | run: | 11 | if (! -e $file) then 12 | echo "$file file not found" 13 | exit 1 14 | else 15 | echo "$file file found" 16 | endif 17 | -------------------------------------------------------------------------------- /tutorials/environment.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | bash_env_variables: 3 | executor: generic.local.bash 4 | description: Declare environment variables in default shell (bash) 5 | type: script 6 | env: 7 | FIRST_NAME: avocado 8 | LAST_NAME: dinosaur 9 | tags: [tutorials] 10 | run: | 11 | hostname 12 | whoami 13 | echo $USER 14 | printf "${FIRST_NAME} ${LAST_NAME}\n" 15 | 16 | csh_env_declaration: 17 | executor: generic.local.csh 18 | type: script 19 | description: "csh shell example to declare environment variables" 20 | shell: /bin/csh 21 | tags: [tutorials] 22 | env: 23 | SHELL_NAME: "csh" 24 | run: echo "This is running $SHELL_NAME" 25 | 26 | tcsh_env_declaration: 27 | executor: generic.local.csh 28 | type: script 29 | description: "tcsh shell example to declare environment variables" 30 | shell: /bin/tcsh 31 | tags: [tutorials] 32 | env: 33 | path: "/usr/local/bin:$PATH" 34 | run: echo $path 35 | -------------------------------------------------------------------------------- /tutorials/gcc_version.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | gcc_version: 3 | type: script 4 | description: Print gcc version 5 | executor: generic.local.bash 6 | run: gcc --version -------------------------------------------------------------------------------- /tutorials/hello.py: -------------------------------------------------------------------------------- 1 | print("Hello World") 2 | -------------------------------------------------------------------------------- /tutorials/hello_world.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | hello_world: 3 | executor: generic.local.bash 4 | type: script 5 | tags: tutorials 6 | description: "hello world example" 7 | run: echo "hello world!" 8 | maintainers: 9 | - "@shahzebsiddiqui" 10 | -------------------------------------------------------------------------------- /tutorials/invalid_buildspec_section.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_type: 3 | executor: generic.local.bash 4 | type: badscript 5 | description: invalid type 6 | tags: [tutorials, fail] 7 | env: 8 | Address: 65 Whale Drive 9 | City: Manchester 10 | run: | 11 | echo "I Live on $Address in $City" 12 | -------------------------------------------------------------------------------- /tutorials/invalid_executor.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | wrongexecutor: 3 | executor: badexecutor 4 | type: script 5 | description: valid test but invalid executor name so test won't run 6 | tags: [tutorials, fail] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /tutorials/invalid_tags.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | duplicate_string_tags: 3 | type: script 4 | executor: generic.local.bash 5 | description: duplicate strings in tags list is not allowed 6 | tags: [network, network] 7 | run: hostname 8 | -------------------------------------------------------------------------------- /tutorials/job_dependency/ex1.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | jobA: 3 | type: script 4 | executor: generic.local.bash 5 | description: no job dependency 6 | run: | 7 | echo "This job has no dependency" 8 | sleep 5 9 | 10 | jobB: 11 | type: script 12 | executor: generic.local.bash 13 | description: job dependency on jobA 14 | needs: [jobA] 15 | run: | 16 | echo "This job depends on jobA" 17 | sleep 2 18 | 19 | jobC: 20 | type: script 21 | executor: generic.local.bash 22 | description: job dependency on jobA and jobB 23 | needs: [jobA, jobB] 24 | run: | 25 | echo "This job depends on jobA and jobB" 26 | sleep 2 27 | -------------------------------------------------------------------------------- /tutorials/job_dependency/ex2.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | test1: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test will pass with exit 1 6 | run: exit 1 7 | status: 8 | state: PASS 9 | 10 | test2: 11 | type: script 12 | executor: generic.local.bash 13 | description: This test will run if test1 has returncode 1 14 | run: exit 2 15 | status: 16 | state: PASS 17 | needs: 18 | - test1: 19 | returncode: 1 20 | 21 | test3: 22 | type: script 23 | executor: generic.local.bash 24 | description: This test will run if test1 has returncode 1 and test2 has returncode 2 25 | run: exit 1 26 | status: 27 | state: PASS 28 | needs: 29 | - test1: 30 | returncode: 1 31 | - test2: 32 | returncode: 2 33 | -------------------------------------------------------------------------------- /tutorials/job_dependency/ex4.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | runtime_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test will sleep 5 second but will fail due to runtime 2sec 6 | status: 7 | runtime: 8 | min: 2.0 9 | run: sleep 5 10 | 11 | runtime_test_pass: 12 | type: script 13 | executor: generic.local.bash 14 | description: This test will run when runtime_test_pass is PASS 15 | needs: 16 | - runtime_test: 17 | state: PASS 18 | run: echo "Performing some action when test PASS" 19 | 20 | runtime_test_fail: 21 | type: script 22 | executor: generic.local.bash 23 | description: This test will run when runtime_test_pass is FAIL 24 | needs: 25 | - runtime_test: 26 | state: FAIL 27 | run: echo "Performing some action when test FAIL" 28 | -------------------------------------------------------------------------------- /tutorials/maintainers_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | foo_bar: 3 | type: script 4 | executor: generic.local.sh 5 | tags: tutorials 6 | description: "prints variable $FOO" 7 | vars: 8 | FOO: BAR 9 | run: echo $FOO 10 | 11 | maintainers: 12 | - "@johndoe" 13 | - "@bobsmith" -------------------------------------------------------------------------------- /tutorials/metrics/invalid_metric_name.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | invalid_metrics_name: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Metrics name does not follow pattern" 6 | run: echo "hello" > file.txt 7 | metrics: 8 | (foo-bar: 9 | type: str 10 | regex: 11 | stream: stdout 12 | exp: "BAR" -------------------------------------------------------------------------------- /tutorials/metrics/invalid_metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metrics_with_regex_and_file_regex_not_allowed: 3 | type: script 4 | executor: generic.local.bash 5 | description: This is an invalid metric because 'file_regex' and 'regex' are both specified 6 | run: echo "hello" > file.txt 7 | metrics: 8 | hello: 9 | type: str 10 | file_regex: 11 | file: file.txt 12 | exp: "BAR" 13 | regex: 14 | stream: stdout 15 | exp: "BAR" 16 | -------------------------------------------------------------------------------- /tutorials/metrics/metrics_file_regex_invalid_file.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_file_regex_invalid_file: 3 | executor: generic.local.bash 4 | type: script 5 | description: capture result metric from file path when we have invalid file path 6 | run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" > hpcg.txt 7 | tags: tutorials 8 | metrics: 9 | hpcg_rating: 10 | type: float 11 | file_regex: 12 | exp: '(\d+\.\d+)$' 13 | file: bad_file.txt -------------------------------------------------------------------------------- /tutorials/metrics/metrics_file_regex_with_invalid_linenum.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_file_regex_with_linenum_failure_example: 3 | executor: generic.local.bash 4 | type: script 5 | description: capture result metric from file path 6 | run: | 7 | echo -e "HPCG result is INVALID with a GFLOP/s rating of=28.1215" > hpcg.txt 8 | echo -e "HPCG result is VALID with a GFLOP/s rating of=68.9888" >> hpcg.txt 9 | tags: tutorials 10 | metrics: 11 | second_line: 12 | type: float 13 | file_regex: 14 | exp: '(\d+\.\d+)$' 15 | linenum: 10 16 | file: hpcg.txt 17 | -------------------------------------------------------------------------------- /tutorials/metrics/metrics_file_regex_with_linenum.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_file_regex_with_linenum_example: 3 | executor: generic.local.bash 4 | type: script 5 | description: capture result metric from file path 6 | run: | 7 | echo -e "HPCG result is INVALID with a GFLOP/s rating of=28.1215" > hpcg.txt 8 | echo -e "HPCG result is VALID with a GFLOP/s rating of=68.9888" >> hpcg.txt 9 | tags: tutorials 10 | metrics: 11 | last_line: 12 | type: float 13 | file_regex: 14 | exp: '(\d+\.\d+)$' 15 | linenum: -1 16 | file: hpcg.txt 17 | without_linenum: 18 | type: float 19 | file_regex: 20 | exp: '(\d+\.\d+)$' 21 | file: hpcg.txt 22 | status: 23 | assert_eq: 24 | comparisons: 25 | - name: last_line 26 | ref: 68.9888 27 | - name: without_linenum 28 | ref: 28.1215 29 | -------------------------------------------------------------------------------- /tutorials/metrics/metrics_regex.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_regex_example: 3 | executor: generic.local.bash 4 | type: script 5 | description: capture result metric from output 6 | run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" 7 | tags: tutorials 8 | metrics: 9 | hpcg_rating_stream: 10 | type: float 11 | regex: 12 | exp: '(\d+\.\d+)$' 13 | stream: stdout 14 | hpcg_state_stream: 15 | type: str 16 | regex: 17 | exp: '(VALID)' 18 | stream: stdout 19 | 20 | metric_file_regex: 21 | executor: generic.local.bash 22 | type: script 23 | description: capture result metric from file path 24 | run: echo "HPCG result is VALID with a GFLOP/s rating of=63.6515" > hpcg.txt 25 | tags: tutorials 26 | metrics: 27 | hpcg_rating_file: 28 | type: float 29 | file_regex: 30 | exp: '(\d+\.\d+)$' 31 | file: hpcg.txt 32 | hpcg_state_file: 33 | type: str 34 | file_regex: 35 | exp: '(VALID)' 36 | file: hpcg.txt 37 | -------------------------------------------------------------------------------- /tutorials/metrics/metrics_regex_with_invalid_linenum.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_regex_linenum_failure_example: 3 | executor: generic.local.bash 4 | type: script 5 | description: invalid linenum can result in failure 6 | run: | 7 | echo "This is line: 1" 8 | echo "This is line: 2" 9 | echo "This is line: 3" 10 | echo "This is line: 4" 11 | tags: tutorials 12 | metrics: 13 | second_line: 14 | type: str 15 | regex: 16 | exp: 'This is line: \d' 17 | stream: stdout 18 | linenum: 10 19 | -------------------------------------------------------------------------------- /tutorials/metrics/metrics_regex_with_linenum.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | metric_regex_with_linenum_example: 3 | executor: generic.local.bash 4 | type: script 5 | description: capture result metric from output 6 | run: | 7 | echo "This is line: 1" 8 | echo "This is line: 2" 9 | echo "This is line: 3" 10 | echo "This is line: 4" 11 | tags: tutorials 12 | metrics: 13 | second_line: 14 | type: str 15 | regex: 16 | exp: 'This is line: \d' 17 | stream: stdout 18 | linenum: 1 19 | without_linenum: 20 | type: str 21 | regex: 22 | exp: 'This is line: \d' 23 | stream: stdout 24 | status: 25 | assert_eq: 26 | comparisons: 27 | - name: second_line 28 | ref: "This is line: 2" 29 | - name: without_linenum 30 | ref: "This is line: 1" 31 | -------------------------------------------------------------------------------- /tutorials/metrics/missing_required_in_metrics.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | required_property_in_metrics: 3 | type: script 4 | executor: generic.local.bash 5 | description: metrics must have a file_regex or regex property to define how to capture metric 6 | run: echo "hello" 7 | metrics: 8 | foo: 9 | type: str -------------------------------------------------------------------------------- /tutorials/multi_executors/executor_regex_script.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | multiple_executors: 3 | type: script 4 | executor: 'generic.local.(bash|sh)' 5 | description: run test with executor generic.local.bash and generic.local.sh executor 6 | tags: [tutorials] 7 | run: date -------------------------------------------------------------------------------- /tutorials/multi_executors/executor_scheduler.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | executors_sbatch_declaration: 3 | type: script 4 | executor: 'generic.local.(bash|sh)' 5 | description: Declaring env and vars by executors section 6 | tags: [tutorials] 7 | run: hostname 8 | sbatch: ["-N 4"] 9 | executors: 10 | generic.local.bash: 11 | sbatch: ["-n 4", "-N 1", "-t 30"] 12 | generic.local.sh: 13 | sbatch: ["-n 8", "-N 1", "-t 60"] 14 | -------------------------------------------------------------------------------- /tutorials/multi_executors/executors_var_env_declaration.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | executors_vars_env_declaration: 3 | type: script 4 | executor: 'generic.local.(bash|sh)' 5 | description: Declaring env and vars by executors section 6 | tags: [tutorials] 7 | run: | 8 | echo "X:" $X 9 | echo "Y:" $Y 10 | echo $SHELL 11 | executors: 12 | generic.local.bash: 13 | vars: 14 | X: 1 15 | Y: 3 16 | env: 17 | SHELL: bash 18 | generic.local.sh: 19 | vars: 20 | X: 2 21 | Y: 4 22 | env: 23 | SHELL: sh 24 | -------------------------------------------------------------------------------- /tutorials/multi_executors/status_by_executors.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | status_returncode_by_executors: 3 | type: script 4 | executor: "generic.local.(bash|sh)" 5 | description: define status per executor type. 6 | tags: [tutorials] 7 | run: exit 0 8 | executors: 9 | generic.local.bash: 10 | status: 11 | returncode: [0, 2] 12 | generic.local.sh: 13 | status: 14 | returncode: 1 15 | -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_eq.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_eq_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test for assert equality 6 | vars: 7 | X: 1 8 | Y: 1.5 9 | first: John 10 | last: Smith 11 | run: | 12 | echo "X: $X" 13 | echo "Y: $Y" 14 | echo "Name: $first $last" 15 | metrics: 16 | x: 17 | type: int 18 | regex: 19 | stream: stdout 20 | exp: 'X:\s+(\S+)\s+.*' 21 | item: 1 22 | y: 23 | type: float 24 | regex: 25 | stream: stdout 26 | exp: 'Y:\s+(\S+)\s+.*' 27 | item: 1 28 | first: 29 | type: str 30 | regex: 31 | stream: stdout 32 | exp: 'Name:\s+(\S+)\s+.*' 33 | item: 1 34 | last: 35 | type: str 36 | regex: 37 | stream: stdout 38 | exp: '(Smith)$' 39 | item: 1 40 | status: 41 | assert_eq: 42 | comparisons: 43 | - name: x 44 | ref: 1 45 | - name: y 46 | ref: 1.5 47 | - name: first 48 | ref: John 49 | - name: last 50 | ref: Smith 51 | -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_eq_exceptions.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_eq_invalid_metric: 3 | type: script 4 | executor: generic.local.bash 5 | description: An invalid metric name will cause failure 6 | vars: 7 | X: 1 8 | run: | 9 | echo "X: $X" 10 | metrics: 11 | x: 12 | type: int 13 | regex: 14 | stream: stdout 15 | exp: 'X:\s+(\S+)\s+.*' 16 | item: 1 17 | status: 18 | assert_eq: 19 | comparisons: 20 | - name: x 21 | ref: 1 22 | - name: invalid_metric 23 | ref: 'hello' 24 | assert_eq_mismatch: 25 | type: script 26 | executor: generic.local.bash 27 | description: This test will fail because there is a mismatch in metric x assert equality 28 | vars: 29 | X: 1 30 | run: | 31 | echo "X: $X" 32 | metrics: 33 | x: 34 | type: int 35 | regex: 36 | stream: stdout 37 | exp: 'X:\s+(\S+)\s+.*' 38 | item: 1 39 | status: 40 | assert_eq: 41 | comparisons: 42 | - name: x 43 | ref: 2 -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_gt.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_gt_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run stream test with metrics example using assert greater than. 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: float 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | scale: 20 | type: float 21 | regex: 22 | exp: 'Scale:\s+(\S+)\s+.*' 23 | stream: stdout 24 | item: 1 25 | add: 26 | type: float 27 | regex: 28 | exp: 'Add:\s+(\S+)\s+.*' 29 | stream: stdout 30 | item: 1 31 | triad: 32 | type: float 33 | regex: 34 | exp: 'Triad:\s+(\S+)\s+.*' 35 | stream: stdout 36 | item: 1 37 | status: 38 | assert_gt: 39 | comparisons: 40 | - name: copy 41 | ref: 5000 42 | - name: scale 43 | ref: 5500 44 | - name: add 45 | ref: 6000 46 | - name: triad 47 | ref: 6500 48 | -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_le.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_le_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run stream test with metrics example using assert less than equal 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: float 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | scale: 20 | type: float 21 | regex: 22 | exp: 'Scale:\s+(\S+)\s+.*' 23 | stream: stdout 24 | item: 1 25 | add: 26 | type: float 27 | regex: 28 | exp: 'Add:\s+(\S+)\s+.*' 29 | stream: stdout 30 | item: 1 31 | triad: 32 | type: float 33 | regex: 34 | exp: 'Triad:\s+(\S+)\s+.*' 35 | stream: stdout 36 | item: 1 37 | status: 38 | assert_le: 39 | comparisons: 40 | - name: copy 41 | ref: 5000 42 | - name: scale 43 | ref: 5500 44 | - name: add 45 | ref: 6000 46 | - name: triad 47 | ref: 6500 48 | -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_lt.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_lt_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Run stream test with metrics example using assert less than 6 | env: 7 | OMP_NUM_THREADS: 4 8 | run: | 9 | wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c 10 | gcc -openmp -o stream stream.c 11 | ./stream 12 | metrics: 13 | copy: 14 | type: float 15 | regex: 16 | exp: 'Copy:\s+(\S+)\s+.*' 17 | stream: stdout 18 | item: 1 19 | scale: 20 | type: float 21 | regex: 22 | exp: 'Scale:\s+(\S+)\s+.*' 23 | stream: stdout 24 | item: 1 25 | add: 26 | type: float 27 | regex: 28 | exp: 'Add:\s+(\S+)\s+.*' 29 | stream: stdout 30 | item: 1 31 | triad: 32 | type: float 33 | regex: 34 | exp: 'Triad:\s+(\S+)\s+.*' 35 | stream: stdout 36 | item: 1 37 | status: 38 | assert_lt: 39 | comparisons: 40 | - name: copy 41 | ref: 5000 42 | - name: scale 43 | ref: 5500 44 | - name: add 45 | ref: 6000 46 | - name: triad 47 | ref: 6500 48 | -------------------------------------------------------------------------------- /tutorials/perf_checks/assert_ne.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | assert_ne_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test for assert not equal 6 | vars: 7 | X: 1 8 | Y: 1.5 9 | first: John 10 | last: Smith 11 | run: | 12 | echo "X: $X" 13 | echo "Y: $Y" 14 | echo "Name: $first $last" 15 | metrics: 16 | x: 17 | type: int 18 | regex: 19 | stream: stdout 20 | exp: 'X:\s+(\S+)\s+.*' 21 | item: 1 22 | y: 23 | type: float 24 | regex: 25 | stream: stdout 26 | exp: 'Y:\s+(\S+)\s+.*' 27 | item: 1 28 | first: 29 | type: str 30 | regex: 31 | stream: stdout 32 | exp: 'Name:\s+(\S+)\s+.*' 33 | item: 1 34 | last: 35 | type: str 36 | regex: 37 | stream: stdout 38 | exp: '(Smith)$' 39 | item: 1 40 | status: 41 | assert_ne: 42 | comparisons: 43 | - name: x 44 | ref: 2 45 | - name: y 46 | ref: 2.5 47 | - name: first 48 | ref: Robert 49 | - name: last 50 | ref: Brown 51 | -------------------------------------------------------------------------------- /tutorials/perf_checks/contains.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | contains_and_not_contains: 3 | type: script 4 | executor: generic.local.bash 5 | description: Status check based on contains and not contains where test pass 6 | vars: 7 | X: 1 8 | run: | 9 | echo "X: $X" 10 | metrics: 11 | x: 12 | type: int 13 | regex: 14 | stream: stdout 15 | exp: 'X:\s+(\S+)\s+.*' 16 | item: 1 17 | status: 18 | contains: 19 | comparisons: 20 | - name: x 21 | ref: [1, 2, 4, 8] 22 | not_contains: 23 | comparisons: 24 | - name: x 25 | ref: [2, 4] 26 | assert_contains_fail: 27 | type: script 28 | executor: generic.local.bash 29 | description: Status check based on contains where test fails 30 | vars: 31 | X: 1 32 | run: | 33 | echo "X: $X" 34 | metrics: 35 | x: 36 | type: int 37 | regex: 38 | stream: stdout 39 | exp: 'X:\s+(\S+)\s+.*' 40 | item: 1 41 | status: 42 | contains: 43 | comparisons: 44 | - name: x 45 | ref: ['1', 2, 4, 8] 46 | -------------------------------------------------------------------------------- /tutorials/post_run.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | post_run_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: post run example that will remove symbolic link 6 | run: | 7 | ln -s $HOME/.bashrc $HOME/.bashrc_link 8 | mkdir demo 9 | post_run: | 10 | unlink $HOME/.bashrc_link 11 | rmdir demo 12 | status: 13 | is_dir: 14 | - demo 15 | is_symlink: 16 | - $HOME/.bashrc_link 17 | -------------------------------------------------------------------------------- /tutorials/python-hello.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | python_hello: 3 | type: script 4 | description: Hello World python 5 | executor: generic.local.bash 6 | tags: python 7 | run: python hello.py 8 | 9 | -------------------------------------------------------------------------------- /tutorials/python-shell.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | circle_area: 3 | executor: generic.local.bash 4 | type: script 5 | shell: python 6 | description: "Calculate circle of area given a radius" 7 | tags: [tutorials, python] 8 | run: | 9 | import math 10 | radius = 2 11 | area = math.pi * radius * radius 12 | print("Circle Radius ", radius) 13 | print("Area of circle ", area) 14 | -------------------------------------------------------------------------------- /tutorials/shebang.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | bash_login_shebang: 3 | type: script 4 | executor: generic.local.bash 5 | shebang: "#!/bin/bash -l" 6 | description: customize shebang line with bash login shell 7 | tags: tutorials 8 | run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell' 9 | status: 10 | regex: 11 | exp: "^Login Shell$" 12 | stream: stdout 13 | 14 | bash_nonlogin_shebang: 15 | type: script 16 | executor: generic.local.bash 17 | shebang: "#!/bin/bash" 18 | description: customize shebang line with default bash (nonlogin) shell 19 | tags: tutorials 20 | run: shopt -q login_shell && echo 'Login Shell' || echo 'Not Login Shell' 21 | status: 22 | regex: 23 | exp: "^Not Login Shell$" 24 | stream: stdout 25 | -------------------------------------------------------------------------------- /tutorials/shell_examples.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | _bin_sh_shell: 3 | executor: generic.local.sh 4 | type: script 5 | description: "/bin/sh shell example" 6 | shell: /bin/sh 7 | tags: [tutorials] 8 | run: "bzip2 --help" 9 | 10 | _bin_bash_shell: 11 | executor: generic.local.bash 12 | type: script 13 | description: "/bin/bash shell example" 14 | shell: /bin/bash 15 | tags: [tutorials] 16 | run: "bzip2 -h" 17 | 18 | bash_shell: 19 | executor: generic.local.bash 20 | type: script 21 | description: "bash shell example" 22 | shell: bash 23 | tags: [tutorials] 24 | run: "echo $SHELL" 25 | 26 | sh_shell: 27 | executor: generic.local.sh 28 | type: script 29 | description: "sh shell example" 30 | shell: sh 31 | tags: [tutorials] 32 | run: "echo $SHELL" 33 | 34 | shell_options: 35 | executor: generic.local.sh 36 | type: script 37 | description: "shell options" 38 | shell: "sh -x" 39 | tags: [tutorials] 40 | run: | 41 | echo $SHELL 42 | hostname 43 | -------------------------------------------------------------------------------- /tutorials/skip_buildspec.yml: -------------------------------------------------------------------------------- 1 | skip: yes 2 | buildspecs: 3 | skip_all_tests: 4 | type: script 5 | executor: generic.local.bash 6 | description: "All test in this buildspec are skipped" 7 | tags: [tutorials] 8 | run: hostname 9 | 10 | this_test_is_also_skipped: 11 | type: script 12 | skip: no 13 | executor: generic.local.bash 14 | description: "This test is also skipped even if skip is defined in test" 15 | tags: [ tutorials ] 16 | run: hostname -------------------------------------------------------------------------------- /tutorials/skip_tests.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | skip: 3 | type: script 4 | executor: generic.local.bash 5 | description: This test is skipped 6 | skip: Yes 7 | tags: [tutorials] 8 | run: hostname 9 | 10 | unskipped: 11 | type: script 12 | executor: generic.local.bash 13 | description: This test is not skipped 14 | skip: No 15 | tags: [tutorials] 16 | run: hostname 17 | -------------------------------------------------------------------------------- /tutorials/sleep.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | sleep: 3 | type: script 4 | executor: generic.local.bash 5 | description: sleep 2 seconds 6 | tags: [tutorials] 7 | vars: 8 | SLEEP_TIME: 2 9 | run: sleep $SLEEP_TIME -------------------------------------------------------------------------------- /tutorials/strict_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | linux_strict_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: "This example test will show how returncode will change when using --strict flag" 6 | run: | 7 | echo "This is a test" 8 | ls -l /BAD_PATH 9 | echo "This is another test" -------------------------------------------------------------------------------- /tutorials/summary_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | summary_example: 3 | type: script 4 | executor: generic.local.bash 5 | description: The summary field can be a multi-line string and exceed 80 char 6 | tags: [tutorials] 7 | summary: | 8 | This is a long description of test that 9 | can exceed 80 characters and be multiline 10 | run: hostname 11 | -------------------------------------------------------------------------------- /tutorials/tags_example.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | string_tag: 3 | type: script 4 | executor: generic.local.bash 5 | description: tags can be a string 6 | tags: network 7 | run: hostname 8 | 9 | list_of_strings_tags: 10 | type: script 11 | executor: generic.local.bash 12 | description: tags can be a list of strings 13 | tags: [network, ping] 14 | run: ping -c 4 www.google.com 15 | -------------------------------------------------------------------------------- /tutorials/test_status/exists.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | status_exists: 3 | type: script 4 | executor: generic.local.bash 5 | description: status check based for file and directory 6 | run: | 7 | mkdir -p $HOME/dirA 8 | mkdir -p /tmp/ABC 9 | touch file1 10 | status: 11 | exists: 12 | - $HOME/dirA 13 | - ~/.bashrc 14 | - /tmp/ABC 15 | - file1 16 | status_exists_failure: 17 | type: script 18 | executor: generic.local.bash 19 | description: status check failure for existence 20 | run: touch foo 21 | status: 22 | exists: 23 | - bar 24 | -------------------------------------------------------------------------------- /tutorials/test_status/explicit_state.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | always_pass: 3 | type: script 4 | executor: 'generic.local.sh' 5 | description: This test will always 'PASS' 6 | run: exit 1 7 | status: 8 | state: PASS 9 | 10 | always_fail: 11 | type: script 12 | executor: 'generic.local.sh' 13 | description: This test will always 'FAIL' 14 | run: exit 0 15 | status: 16 | state: FAIL 17 | 18 | test_fail_returncode_match: 19 | type: script 20 | executor: 'generic.local.sh' 21 | description: This test will 'FAIL' even if we have returncode match 22 | run: exit 1 23 | status: 24 | state: FAIL 25 | returncode: 1 26 | 27 | test_pass_returncode_mismatch: 28 | type: script 29 | executor: 'generic.local.sh' 30 | description: This test will 'PASS' even if we have returncode mismatch 31 | run: exit 1 32 | status: 33 | state: PASS 34 | returncode: 2 35 | -------------------------------------------------------------------------------- /tutorials/test_status/file_and_dir_check.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_and_dir_checks: 3 | type: script 4 | executor: generic.local.bash 5 | description: status check for files and directories 6 | run: hostname 7 | status: 8 | is_dir: 9 | - $HOME 10 | - $HOME/.bashrc 11 | - /tmp 12 | combined_file_and_dir_checks: 13 | type: script 14 | executor: generic.local.bash 15 | description: status check for files and directories 16 | run: hostname 17 | status: 18 | is_dir: 19 | - $HOME 20 | - /tmp 21 | is_file: 22 | - $HOME/.bashrc 23 | -------------------------------------------------------------------------------- /tutorials/test_status/file_count.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_on_directory: 3 | type: script 4 | executor: generic.local.bash 5 | description: file count check in directory 6 | run: | 7 | mkdir -p foo 8 | touch foo/{1..5} 9 | status: 10 | file_count: 11 | - dir: foo 12 | count: 5 13 | file_count_by_extension: 14 | type: script 15 | executor: generic.local.bash 16 | description: file count by extension 17 | run: | 18 | mkdir -p foo/bar 19 | touch foo/{1..5}.sh 20 | touch foo/bar/{1..3}.py foo/bar/{1..3}.txt 21 | status: 22 | file_count: 23 | - dir: foo 24 | ext: '.sh' 25 | depth: 1 26 | count: 5 27 | - dir: foo/bar 28 | ext: ['.py', '.txt'] 29 | count: 6 30 | -------------------------------------------------------------------------------- /tutorials/test_status/file_count_file_traverse_limit.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_traverse_limit: 3 | type: script 4 | executor: generic.local.bash 5 | description: Use of file_traverse_limit to limit number of files searched in a directory 6 | run: | 7 | mkdir foo 8 | touch foo/{1..99}.txt 9 | status: 10 | file_count: 11 | - dir: foo 12 | count: 50 13 | file_traverse_limit: 50 14 | - dir: foo 15 | count: 10 16 | file_traverse_limit: 20 17 | -------------------------------------------------------------------------------- /tutorials/test_status/file_count_filetype.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_count_by_filetype: 3 | type: script 4 | executor: generic.local.bash 5 | description: Count the number of directories and symbolic links 6 | run: | 7 | mkdir -p foo/{bar,baz} 8 | find foo -type dir 9 | ln -s foo/bar foo/bar.link 10 | ln -s foo/baz foo/baz.link 11 | status: 12 | file_count: 13 | - dir: foo 14 | count: 3 15 | filetype: 'dir' 16 | - dir: foo 17 | count: 2 18 | filetype: 'symlink' 19 | -------------------------------------------------------------------------------- /tutorials/test_status/file_exists_exception.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_exists_failure: 3 | type: script 4 | executor: generic.local.bash 5 | description: this test will fail validation, because item must be a string 6 | run: mkdir -p 1 7 | status: 8 | exists: [1] -------------------------------------------------------------------------------- /tutorials/test_status/file_exists_with_number.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_exists_pass: 3 | type: script 4 | executor: generic.local.bash 5 | description: this test will pass 6 | run: mkdir -p 1 7 | status: 8 | exists: [ '1' ] -------------------------------------------------------------------------------- /tutorials/test_status/file_linecount.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_linecount: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Perform linecount comparison on files" 6 | run: | 7 | for i in {1..10}; do 8 | echo $i >> count.txt 9 | done 10 | echo "hello world" > hello.txt 11 | status: 12 | file_linecount: 13 | - file: count.txt 14 | count: 10 15 | - file: hello.txt 16 | count: 1 17 | -------------------------------------------------------------------------------- /tutorials/test_status/file_linecount_failure.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_linecount_exceptions: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Performing file count on directory or invalid files can result in failure" 6 | run: | 7 | for i in {1..10}; do 8 | echo $i >> count.txt 9 | done 10 | echo "hello world" > hello.txt 11 | status: 12 | file_linecount: 13 | - file: /tmp 14 | count: 10 15 | - file: /badfile.txt 16 | count: 1 17 | -------------------------------------------------------------------------------- /tutorials/test_status/file_linecount_invalid.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | file_linecount_negative_value: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Performing file count with a negative count value will result in error" 6 | run: | 7 | > empty.txt 8 | status: 9 | file_linecount: 10 | - file: empty.txt 11 | count: -1 12 | -------------------------------------------------------------------------------- /tutorials/test_status/is_symlink.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | symlink_test: 3 | type: script 4 | executor: generic.local.bash 5 | description: status check based on symbolic link 6 | run: | 7 | ln -s /tmp scratch 8 | ln -s $HOME/.bashrc $HOME/.bashrc_link 9 | status: 10 | is_symlink: 11 | - scratch 12 | - $HOME/.bashrc_link 13 | - ~/.bashrc_link 14 | -------------------------------------------------------------------------------- /tutorials/test_status/linecount.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | linecount_stdout: 3 | type: script 4 | executor: generic.local.bash 5 | description: "Write 10 lines to stdout and run linecount check" 6 | run: | 7 | for i in {1..10}; do 8 | echo $i 9 | done 10 | status: 11 | linecount: 12 | stream: stdout 13 | count: 10 14 | 15 | linecount_stderr_mismatch: 16 | type: script 17 | executor: generic.local.bash 18 | description: "Write 10 lines to stderr and run linecount check" 19 | run: | 20 | for i in {1..10}; do 21 | echo $i >&2 22 | done 23 | status: 24 | linecount: 25 | stream: stderr 26 | count: 5 27 | -------------------------------------------------------------------------------- /tutorials/test_status/mode.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | status_logical_and: 3 | type: script 4 | executor: 'generic.local.bash' 5 | description: 'Using logical AND to check status' 6 | run: | 7 | echo "This is a test" 8 | exit 1 9 | status: 10 | mode: and 11 | returncode: 1 12 | regex: 13 | stream: stdout 14 | exp: 'This is a test' 15 | 16 | status_logical_or: 17 | type: script 18 | executor: 'generic.local.bash' 19 | description: 'Using logical OR to check status' 20 | run: | 21 | echo "This is a test" 22 | exit 1 23 | status: 24 | mode: or 25 | returncode: 0 26 | regex: 27 | stream: stdout 28 | exp: 'This is a test' 29 | -------------------------------------------------------------------------------- /tutorials/test_status/pass_returncode.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | 3 | exit1_fail: 4 | executor: generic.local.bash 5 | type: script 6 | description: exit 1 by default is FAIL 7 | tags: [tutorials, fail] 8 | run: exit 1 9 | 10 | exit1_pass: 11 | executor: generic.local.bash 12 | type: script 13 | description: report exit 1 as PASS 14 | run: exit 1 15 | tags: [tutorials, pass] 16 | status: 17 | returncode: [1] 18 | 19 | returncode_list_mismatch: 20 | executor: generic.local.bash 21 | type: script 22 | description: exit 2 failed since it failed to match returncode 1 23 | run: exit 2 24 | tags: [tutorials, fail] 25 | status: 26 | returncode: [1, 3] 27 | 28 | returncode_int_match: 29 | executor: generic.local.bash 30 | type: script 31 | description: exit 128 matches returncode 128 32 | run: exit 128 33 | tags: [tutorials, pass] 34 | status: 35 | returncode: 128 36 | -------------------------------------------------------------------------------- /tutorials/test_status/regex_on_filename.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | regex_on_multiple_files: 3 | type: script 4 | executor: generic.local.bash 5 | description: Test regex on multiple files 6 | run: | 7 | echo "Hello" > hello.txt 8 | buildtest --help > buildtest_help.txt 9 | status: 10 | file_regex: 11 | - file: hello.txt 12 | exp: '^(Hello)$' 13 | - file: buildtest_help.txt 14 | exp: '^(usage: buildtest)' 15 | 16 | regex_on_directory_not_supported: 17 | type: script 18 | executor: generic.local.bash 19 | description: Test regex on directory is not supported 20 | run: | 21 | mkdir -p hello 22 | echo "Hello" > hello/hello.txt 23 | status: 24 | file_regex: 25 | - file: hello 26 | exp: '^(Hello)$' 27 | 28 | file_expansion_supported: 29 | type: script 30 | executor: generic.local.bash 31 | description: Test regex with variable and shell expansion 32 | run: | 33 | echo "Hello" > $BUILDTEST_ROOT/hello.txt 34 | echo "Hello" > $HOME/hello.txt 35 | status: 36 | file_regex: 37 | - file: $BUILDTEST_ROOT/hello.txt 38 | exp: '^(Hello)$' 39 | - file: ~/hello.txt 40 | exp: '^(Hello)$' 41 | -------------------------------------------------------------------------------- /tutorials/test_status/status_regex.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | status_regex_stdout_pass: 3 | executor: generic.local.bash 4 | type: script 5 | tags: [system] 6 | description: Pass test based on regular expression 7 | run: echo "PASS" 8 | status: 9 | regex: 10 | stream: stdout 11 | exp: "^(PASS)$" 12 | 13 | status_regex_stdout_fail: 14 | executor: generic.local.bash 15 | type: script 16 | tags: [system] 17 | description: Pass test based on regular expression 18 | run: echo "FAIL" 19 | status: 20 | regex: 21 | stream: stdout 22 | exp: "^(123FAIL)$" 23 | 24 | status_regex_stderr_pass: 25 | executor: generic.local.bash 26 | type: script 27 | tags: [system] 28 | description: Pass test based on regular expression 29 | run: echo "PASS" 1>&2 30 | status: 31 | regex: 32 | stream: stderr 33 | exp: '^(PASS)$' 34 | 35 | status_regex_stderr_fail: 36 | executor: generic.local.bash 37 | type: script 38 | tags: [system] 39 | description: Pass test based on regular expression 40 | run: echo "FAIL" 1>&2 41 | status: 42 | regex: 43 | stream: stderr 44 | exp: "^(123FAIL)$" 45 | -------------------------------------------------------------------------------- /tutorials/vars.yml: -------------------------------------------------------------------------------- 1 | buildspecs: 2 | variables_bash: 3 | type: script 4 | executor: generic.local.bash 5 | description: Declare shell variables in bash 6 | tags: [tutorials] 7 | vars: 8 | X: 1 9 | Y: 2 10 | literalstring: this is a literal string 11 | singlequote: \'singlequote\' 12 | doublequote: \"doublequote\" 13 | current_user: "$(whoami)" 14 | num_files: "`find $HOME -type f -maxdepth 1 | wc -l`" 15 | multiline_string: | 16 | Hello my name is Bob \n 17 | I am 30 years old 18 | 19 | 20 | run: | 21 | echo "$X+$Y="$(($X+$Y)) 22 | echo $literalstring 23 | echo $singlequote 24 | echo $doublequote 25 | echo "current user:" $current_user 26 | echo "number of files:" $num_files 27 | echo -e $multiline_string 28 | --------------------------------------------------------------------------------