├── tests ├── __init__.py └── sdc_scissor │ ├── __init__.py │ ├── can_api │ ├── __init__.py │ └── test_can_bus_handler.py │ ├── pynguin │ ├── __init__.py │ ├── test_sdc_scissor_obstacle_api_beamng_delineator.py │ └── test_sdc_scissor_obstacle_api_beamng_bump.py │ ├── obstacle_api │ ├── __init__.py │ └── test_beamng_tree.py │ ├── simulator_api │ ├── __init__.py │ ├── test_simulator_factory.py │ └── test_beamng_simulator.py │ ├── testing_api │ ├── __init__.py │ ├── test_test_loader.py │ ├── test_test_validator.py │ └── test_test_monitor.py │ ├── feature_extraction_api │ ├── __init__.py │ └── test_parameterized_uniform_strategy.py │ └── machine_learning_api │ ├── __init__.py │ ├── test_csv_loader.py │ └── test_predictor.py ├── sdc_scissor ├── __init__.py ├── can_api │ ├── __init__.py │ └── can_msg_generator.py ├── obstacle_api │ ├── __init__.py │ ├── carla_bump.py │ ├── carla_tree.py │ ├── carla_delineator.py │ ├── obstacle_factory.py │ ├── beamng_delineator.py │ ├── beamng_bump.py │ ├── beamng_tree.py │ ├── delineator.py │ ├── tree.py │ ├── carla_obstacle_factory.py │ ├── beamng_obstacle_factory.py │ └── bump.py ├── simulator_api │ ├── __init__.py │ ├── simulator_factory.py │ └── abstract_simulator.py ├── testing_api │ ├── __init__.py │ ├── test_generators │ │ ├── ambiegen │ │ │ ├── __init__.py │ │ │ ├── Utils │ │ │ │ ├── duplicate_elimination.py │ │ │ │ ├── test_case_problem.py │ │ │ │ ├── generate_test_case_sampling.py │ │ │ │ ├── solution.py │ │ │ │ ├── test_case_crossover.py │ │ │ │ └── test_case_mutation.py │ │ │ ├── config.py │ │ │ └── ambiegen_generator.py │ │ ├── frenetic │ │ │ ├── __init__.py │ │ │ └── src │ │ │ │ ├── __init__.py │ │ │ │ ├── utils │ │ │ │ ├── __init__.py │ │ │ │ └── frenet.py │ │ │ │ ├── generators │ │ │ │ ├── __init__.py │ │ │ │ └── base_frenet_generator.py │ │ │ │ ├── frenetic-sbst21-preprint.pdf │ │ │ │ └── LICENSE │ │ └── frenetic_v │ │ │ ├── __init__.py │ │ │ └── src │ │ │ ├── __init__.py │ │ │ ├── utils │ │ │ ├── __init__.py │ │ │ └── frenet.py │ │ │ ├── generators │ │ │ └── __init__.py │ │ │ ├── frenetic-sbst21-preprint.pdf │ │ │ └── LICENSE │ ├── road_model.py │ ├── test_plotter.py │ ├── test.py │ ├── test_loader.py │ └── test_validator.py ├── sdc_prioritizer │ ├── __init__.py │ ├── testPrioritization │ │ ├── __init__.py │ │ ├── crossover │ │ │ └── PMX.py │ │ ├── problem │ │ │ └── TestPrioritizationMultiObjectiveProblem.py │ │ ├── mutation │ │ │ └── HybridMut.py │ │ └── README.md │ ├── setup.py │ ├── requirements.txt │ ├── README.md │ ├── code │ │ ├── r-script │ │ │ ├── better_rate.pdf │ │ │ ├── medians_APFDc.csv │ │ │ ├── utils.R │ │ │ ├── vsbl.csv │ │ │ ├── vsrand.csv │ │ │ ├── table.R │ │ │ ├── plot.R │ │ │ └── gavsga.csv │ │ ├── objectives.m │ │ ├── plotResults.m │ │ ├── fitness.m │ │ ├── featureSelection.m │ │ ├── faultDetection.m │ │ ├── main.m │ │ ├── mainGreedy.m │ │ ├── permutationCrossover.m │ │ ├── initialPopulation.m │ │ ├── runGreedy.m │ │ ├── python │ │ │ └── make-single-csv.py │ │ ├── permutationMutation.m │ │ └── runSearch.m │ ├── setup.cfg │ ├── datasets │ │ └── fullroad │ │ │ └── BeamNG_AI │ │ │ ├── README.md │ │ │ └── BeamNG_RF_1_5 │ │ │ └── filter.py │ ├── docker_scripts │ │ ├── build-test-prioritization-image.sh │ │ └── run-test-prioritization-container.sh │ └── Dockerfile.testPrioritization ├── feature_extraction_api │ ├── __init__.py │ ├── segmentation_strategy.py │ ├── parameterized_uniform_strategy.py │ └── road_geometry_calculator.py ├── machine_learning_api │ ├── __init__.py │ ├── csv_loader.py │ └── predictor.py └── config.py ├── levels_template └── tig │ ├── main │ ├── MissionGroup │ │ ├── Water │ │ │ └── items.level.json │ │ ├── PlayerDropPoints │ │ │ └── items.level.json │ │ ├── items.level.readme.txt │ │ ├── vegetation │ │ │ └── items.level.json │ │ ├── CameraBookmarks │ │ │ └── items.level.json │ │ ├── items.level.json │ │ ├── sky_and_sun │ │ │ └── items.level.json │ │ └── generated │ │ │ └── items.level.json-readme.txt │ └── items.level.json │ ├── tig-version.json │ ├── art │ ├── road │ │ ├── line_white_d.dds │ │ ├── line_white_n.dds │ │ ├── line_yellow_d.dds │ │ ├── road_asphalt_2lane_d.dds │ │ ├── road_asphalt_2lane_n.dds │ │ ├── road_asphalt_2lane_s.dds │ │ ├── road_rubber_sticky_d.dds │ │ └── materials.cs │ └── terrains │ │ ├── Grass-01-D.dds │ │ ├── Grass-01-N.dds │ │ ├── Macro_grass.dds │ │ ├── Overlay_Grass-01.dds │ │ └── materials.cs │ ├── terrain.terrainheightmap.png │ ├── info.json │ ├── main.decals.json │ └── terrain.terrain.json ├── docs ├── contributing.rst ├── code_of_conduct.rst ├── images │ ├── logo.png │ ├── beamng-test.png │ ├── readthedocs.png │ ├── sdc-test-case.png │ ├── github_logo_icon.png │ ├── sdc-scissor-APIs.jpg │ ├── test-prediction.png │ ├── diversity-feature.jpeg │ ├── teaser-system-view.png │ └── sdc-scissor-architecture.jpg ├── c4model │ ├── structurizr-1-Diagram1.png │ ├── structurizr-1-Diagram2.png │ ├── structurizr-1-Diagram3.png │ ├── structurizr-1-Diagram1-key.png │ ├── structurizr-1-Diagram2-key.png │ ├── structurizr-1-Diagram3-key.png │ └── workspace.dsl ├── api │ ├── index.rst │ ├── simulator_api.rst │ ├── testing_api.rst │ ├── machine_learning_api.rst │ ├── feature_extraction_api.rst │ └── sdc_scissor_cli.rst ├── software_guidebook │ ├── software_architecture.md │ ├── infrastructure_architecture.md │ ├── decisions │ │ ├── index.rst │ │ ├── 0002-config-file-for-cli.md │ │ ├── 0003-use-conventional-commit-messages.md │ │ ├── 0001-refactor-complex-sdc-scissor-v1.md │ │ ├── 0000-use-markdown-any-decision-records.md │ │ └── adr-template.md │ ├── constraints.md │ ├── introduction.md │ ├── quality_attributes.md │ ├── context.md │ ├── principles.md │ ├── operation_and_support.md │ ├── deployment.md │ └── functional_overview.md ├── Makefile ├── make.bat ├── user_documentation │ ├── machine_learning.md │ ├── test_generation.md │ ├── test_prioritization.md │ ├── quick_start.md │ └── feature_extraction.md ├── index.rst └── conf.py ├── sample_configs ├── extract-features.yml ├── evaluate-models.yml ├── gen-can-msg.yml └── label-tests.yml ├── influxdb └── docker-compose.yml ├── .dockerignore ├── .readthedocs.yaml ├── Dockerfile ├── dbc_maps ├── dbc_map_carla.json └── dbc_map_beamng.json ├── .github ├── ISSUE_TEMPLATE │ ├── feature.md │ └── bug.md └── workflows │ ├── cd.yml │ └── ci.yml ├── .pre-commit-config.yaml ├── sonar-project.properties ├── sample_candb └── beamng_pipeline_sample.dbc ├── CITATION.cff ├── CONTRIBUTING.md ├── pyproject.toml └── .gitignore /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/can_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/simulator_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/can_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/pynguin/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/obstacle_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/simulator_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/testing_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/feature_extraction_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/machine_learning_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/feature_extraction_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/sdc_scissor/machine_learning_api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/testPrioritization/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/Water/items.level.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/PlayerDropPoints/items.level.json: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /levels_template/tig/tig-version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 0.7 3 | } 4 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/generators/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/generators/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | setuptools.setup() 4 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.md 2 | :parser: myst_parser.sphinx_ 3 | -------------------------------------------------------------------------------- /docs/code_of_conduct.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CODE_OF_CONDUCT.md 2 | :parser: myst_parser.sphinx_ 3 | -------------------------------------------------------------------------------- /tests/sdc_scissor/simulator_api/test_simulator_factory.py: -------------------------------------------------------------------------------- 1 | class TestSimulatorFactory: 2 | pass 3 | -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/logo.png -------------------------------------------------------------------------------- /docs/images/beamng-test.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/beamng-test.png -------------------------------------------------------------------------------- /docs/images/readthedocs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/readthedocs.png -------------------------------------------------------------------------------- /docs/images/sdc-test-case.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/sdc-test-case.png -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/requirements.txt: -------------------------------------------------------------------------------- 1 | pymoo==0.5.0 2 | numpy==1.20.0 3 | pyrecorder==0.2.1 4 | scikit-learn==1.0.2 5 | -------------------------------------------------------------------------------- /docs/images/github_logo_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/github_logo_icon.png -------------------------------------------------------------------------------- /docs/images/sdc-scissor-APIs.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/sdc-scissor-APIs.jpg -------------------------------------------------------------------------------- /docs/images/test-prediction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/test-prediction.png -------------------------------------------------------------------------------- /docs/images/diversity-feature.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/diversity-feature.jpeg -------------------------------------------------------------------------------- /docs/images/teaser-system-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/teaser-system-view.png -------------------------------------------------------------------------------- /sample_configs/extract-features.yml: -------------------------------------------------------------------------------- 1 | command: 'extract-features' 2 | options: 3 | tests: 'sample_tests' 4 | segmentation: 'angle-based' 5 | -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram1.png -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram2.png -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram3.png -------------------------------------------------------------------------------- /docs/images/sdc-scissor-architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/images/sdc-scissor-architecture.jpg -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram1-key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram1-key.png -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram2-key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram2-key.png -------------------------------------------------------------------------------- /docs/c4model/structurizr-1-Diagram3-key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/docs/c4model/structurizr-1-Diagram3-key.png -------------------------------------------------------------------------------- /levels_template/tig/main/items.level.json: -------------------------------------------------------------------------------- 1 | {"name":"MissionGroup","class":"SimGroup","persistentId":"9e79f9e5-f71f-4111-8361-cfb22c3f9112","Enabled":"1"} 2 | -------------------------------------------------------------------------------- /sample_configs/evaluate-models.yml: -------------------------------------------------------------------------------- 1 | command: 'evaluate-models' 2 | options: 3 | csv: 'sample_tests/road_features.csv' 4 | models_dir: 'trained_models' 5 | -------------------------------------------------------------------------------- /levels_template/tig/art/road/line_white_d.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/line_white_d.dds -------------------------------------------------------------------------------- /levels_template/tig/art/road/line_white_n.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/line_white_n.dds -------------------------------------------------------------------------------- /levels_template/tig/art/road/line_yellow_d.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/line_yellow_d.dds -------------------------------------------------------------------------------- /levels_template/tig/art/terrains/Grass-01-D.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/terrains/Grass-01-D.dds -------------------------------------------------------------------------------- /levels_template/tig/art/terrains/Grass-01-N.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/terrains/Grass-01-N.dds -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/README.md: -------------------------------------------------------------------------------- 1 | # COSMOS-WP5.3 2 | 3 | ## Structure 4 | - datasets 5 | - fullroad 6 | - BeamNG_AI 7 | - Driver_AI 8 | -------------------------------------------------------------------------------- /levels_template/tig/art/terrains/Macro_grass.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/terrains/Macro_grass.dds -------------------------------------------------------------------------------- /levels_template/tig/terrain.terrainheightmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/terrain.terrainheightmap.png -------------------------------------------------------------------------------- /levels_template/tig/art/road/road_asphalt_2lane_d.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/road_asphalt_2lane_d.dds -------------------------------------------------------------------------------- /levels_template/tig/art/road/road_asphalt_2lane_n.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/road_asphalt_2lane_n.dds -------------------------------------------------------------------------------- /levels_template/tig/art/road/road_asphalt_2lane_s.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/road_asphalt_2lane_s.dds -------------------------------------------------------------------------------- /levels_template/tig/art/road/road_rubber_sticky_d.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/road/road_rubber_sticky_d.dds -------------------------------------------------------------------------------- /levels_template/tig/art/terrains/Overlay_Grass-01.dds: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/levels_template/tig/art/terrains/Overlay_Grass-01.dds -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/better_rate.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/sdc_scissor/sdc_prioritizer/code/r-script/better_rate.pdf -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/medians_APFDc.csv: -------------------------------------------------------------------------------- 1 | benchmark,execution_id.so,execution_id.mo,solution_id.mo 2 | BeamNG.AI.AF1,24,23,134 3 | BeamNG.AI.AF1.5,3,4,73 4 | DeepDriving,25,4,111 5 | -------------------------------------------------------------------------------- /levels_template/tig/info.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "tig", 3 | "description": "Precrime base level", 4 | "previews": ["template_preview.jpg"], 5 | "size": [2048, 2048], 6 | "authors": "Precrime team" 7 | } 8 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/items.level.readme.txt: -------------------------------------------------------------------------------- 1 | {"class":"GroundPlane","persistentId":"8538a723-a048-47be-8a82-0d46c1159338","__parent":"MissionGroup","position":"0 0 0","material":"WarningMaterial","squareSize":54} 2 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/frenetic-sbst21-preprint.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/sdc_scissor/testing_api/test_generators/frenetic/src/frenetic-sbst21-preprint.pdf -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/frenetic-sbst21-preprint.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christianbirchler-org/sdc-scissor/HEAD/sdc_scissor/testing_api/test_generators/frenetic_v/src/frenetic-sbst21-preprint.pdf -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | API 2 | === 3 | .. image:: ../images/sdc-scissor-APIs.jpg 4 | 5 | .. toctree:: 6 | 7 | sdc_scissor_cli 8 | testing_api 9 | feature_extraction_api 10 | simulator_api 11 | machine_learning_api 12 | -------------------------------------------------------------------------------- /docs/api/simulator_api.rst: -------------------------------------------------------------------------------- 1 | Simulator API 2 | ============= 3 | .. automodule:: sdc_scissor.simulator_api.abstract_simulator 4 | .. automodule:: sdc_scissor.simulator_api.simulator_factory 5 | .. automodule:: sdc_scissor.simulator_api.beamng_simulator 6 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/carla_bump.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from sdc_scissor.obstacle_api.bump import Bump 4 | 5 | 6 | class CarlaBump(Bump): 7 | pass 8 | 9 | 10 | if __name__ == "__main__": 11 | logging.info("carla_bump.py") 12 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/carla_tree.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from sdc_scissor.obstacle_api.tree import Tree 4 | 5 | 6 | class CarlaTree(Tree): 7 | pass 8 | 9 | 10 | if __name__ == "__main__": 11 | logging.info("carla_tree.py") 12 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/carla_delineator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from sdc_scissor.obstacle_api.delineator import Delineator 4 | 5 | 6 | class CarlaDelineator(Delineator): 7 | pass 8 | 9 | 10 | if __name__ == "__main__": 11 | logging.info("carla_delineator.py") 12 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/vegetation/items.level.json: -------------------------------------------------------------------------------- 1 | {"class":"ForestWindEmitter","persistentId":"6737fa35-d648-4f48-b789-1d13783838dc","__parent":"vegetation","position":[24,-89,124]} 2 | {"name":"theForest","class":"Forest","persistentId":"4b096347-fd00-402d-bbda-f3b269ead79d","__parent":"vegetation"} 3 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/objectives.m: -------------------------------------------------------------------------------- 1 | function [objs] = objectives(X) 2 | global A Cost 3 | 4 | [~,n] = size(X); 5 | obj1 = 0; 6 | obj2 = 0; 7 | for i=1:n-1 8 | obj1 = obj1 + Cost(X(i)) / i; 9 | obj2 = obj2 + A(X(i), X(i+1)) / i; 10 | end 11 | objs = [obj1; -obj2]; %% we want to maximize 12 | end 13 | -------------------------------------------------------------------------------- /docs/api/testing_api.rst: -------------------------------------------------------------------------------- 1 | Testing API 2 | =========== 3 | .. automodule:: sdc_scissor.testing_api.test 4 | .. automodule:: sdc_scissor.testing_api.test_generator 5 | .. automodule:: sdc_scissor.testing_api.test_loader 6 | .. automodule:: sdc_scissor.testing_api.test_monitor 7 | .. automodule:: sdc_scissor.testing_api.test_runner 8 | -------------------------------------------------------------------------------- /influxdb/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | influxdb: 4 | image: influxdb:latest 5 | container_name: influxdb 6 | ports: 7 | - 8086:8086 8 | volumes: 9 | - influxdb:/var/lib/influxdb2 10 | 11 | volumes: 12 | influxdb: 13 | name: influxdb 14 | external: false 15 | -------------------------------------------------------------------------------- /levels_template/tig/main.decals.json: -------------------------------------------------------------------------------- 1 | { 2 | "header":{ 3 | "name":"DecalData File", 4 | "comments":"// Instances format: rectIdx, size, renderPriority, position.x, position.y, position.z, normal.x, normal.y, normal.z, tangent.x, tangent.y, tangent.z", 5 | "version":1} 6 | , 7 | "instances":{ 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /docs/api/machine_learning_api.rst: -------------------------------------------------------------------------------- 1 | Machine Learning API 2 | ==================== 3 | .. automodule:: sdc_scissor.machine_learning_api.csv_loader 4 | .. automodule:: sdc_scissor.machine_learning_api.model_evaluator 5 | .. automodule:: sdc_scissor.machine_learning_api.predictor 6 | .. automodule:: sdc_scissor.machine_learning_api.cost_effectiveness_evaluator 7 | -------------------------------------------------------------------------------- /sdc_scissor/machine_learning_api/csv_loader.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pandas as pd 4 | 5 | 6 | class CSVLoader: 7 | @staticmethod 8 | def load_dataframe_from_csv(data_path: Path): 9 | """ 10 | 11 | :param data_path: 12 | :return: 13 | """ 14 | dd = pd.read_csv(data_path) 15 | return dd 16 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Ignore valid roads 2 | /valid_roads/ 3 | 4 | # Ignore results 5 | /results/ 6 | /destination/ 7 | 8 | # Ignore simulation data 9 | /simulations/ 10 | 11 | # Ignore Shapely client specific file for pip 12 | /Shapely-* 13 | 14 | /Dockerfile 15 | /.dockerignore 16 | 17 | /.git 18 | /.gitignore 19 | 20 | /**/__pycache__ 21 | 22 | /.vscode 23 | 24 | /documentation 25 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | build: 4 | os: ubuntu-22.04 5 | tools: 6 | python: "3.10" 7 | jobs: 8 | post_create_environment: 9 | - pip install poetry 10 | - poetry config virtualenvs.create false 11 | - poetry install --with docs 12 | - poetry run pyreverse sdc_scissor -o html -d _build/doctrees 13 | 14 | sphinx: 15 | configuration: docs/conf.py 16 | -------------------------------------------------------------------------------- /docs/software_guidebook/software_architecture.md: -------------------------------------------------------------------------------- 1 | # Software Architecture 2 | This section provides an overview of SDC-Scissor's software architecture according to the 3 | [C4 Model](https://c4model.com/). 4 | 5 | ## Context 6 | ![](../c4model/structurizr-1-Diagram1.png) 7 | 8 | ## Containers 9 | ![](../c4model/structurizr-1-Diagram2.png) 10 | 11 | ## Components 12 | ![](../c4model/structurizr-1-Diagram3.png) 13 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/duplicate_elimination.py: -------------------------------------------------------------------------------- 1 | from pymoo.model.duplicate import ElementwiseDuplicateElimination 2 | 3 | 4 | class DuplicateElimination(ElementwiseDuplicateElimination): 5 | """ 6 | Module to remove the same individuals 7 | """ 8 | 9 | def is_equal(self, a, b): 10 | return a.X[0].states == b.X[0].states # remove individuals that are the same 11 | -------------------------------------------------------------------------------- /sample_configs/gen-can-msg.yml: -------------------------------------------------------------------------------- 1 | command: 'gen-can-msg' 2 | options: 3 | strategy: 'random' 4 | canbus: false 5 | can_stdout: true 6 | can_dbc: 'C:\Users\birch\repositories\sdc-scissor\sample_candb\beamng_pipeline_sample.dbc' 7 | can_dbc_map: 'C:\Users\birch\repositories\sdc-scissor\dbc_maps\dbc_map_beamng.json' 8 | can_interface: 'socketcan' 9 | can_channel: 'vcan0' 10 | can_bitrate: 250000 11 | timeout: 8 12 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/plotResults.m: -------------------------------------------------------------------------------- 1 | function [value] = plotResults(X, color) 2 | global A Cost 3 | 4 | [m,n] = size(X); 5 | 6 | xValues = zeros(1, m); 7 | yValues = zeros(1, m); 8 | 9 | xValues(1,1) = 0; 10 | yValues(1,1) = 0; 11 | 12 | value = 0; 13 | for i=2:n 14 | xValues(1,i) = xValues(1,i-1) + Cost(X(i)); 15 | yValues(1,i) = A(X(i-1), X(i)); 16 | end 17 | plot(xValues, yValues, color); 18 | end 19 | -------------------------------------------------------------------------------- /docs/software_guidebook/infrastructure_architecture.md: -------------------------------------------------------------------------------- 1 | # Infrastructure Architecture 2 | This section provides information about the infrastructure architecture needed for SDC-Scissor. 3 | 4 | * **Operating System**: 5 | * Windows 10 6 | * Linux (Limited by external simulator requirements) 7 | * Mac OS (Limited by external simulator requirements) 8 | * **Flavour**: 8G RAM, 40GB Disk 9 | * Dedicated GPU is recommended 10 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = testPrioritization 3 | version = 0.0.1 4 | 5 | [options] 6 | packages = testPrioritization 7 | install_requires = 8 | requests 9 | importlib; python_version == "3.9.1" 10 | 11 | # [mutmut] 12 | # paths_to_mutate = testPrioritization/ 13 | # backup = False 14 | # tests_dir = pd/ 15 | # dict_synonyms = Struct, NamedStruct 16 | 17 | [flake8] 18 | max-line-length = 100 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.6-slim 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y \ 5 | gcc \ 6 | python3-opengl \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | ENV SHELL=/bin/bash 10 | ARG POETRY_VERSION=1.1.13 11 | 12 | RUN pip install "poetry==$POETRY_VERSION" 13 | 14 | WORKDIR /var/project 15 | 16 | COPY pyproject.toml poetry.lock ./ 17 | 18 | RUN poetry install 19 | 20 | COPY . ./ 21 | 22 | ENTRYPOINT ["poetry", "run", "sdc-scissor"] 23 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/utils.R: -------------------------------------------------------------------------------- 1 | fixBenchmarkNames <- function(df){ 2 | df$benchmark <- ifelse(df$benchmark == "DriverAI_Complete", "DeepDriving", df$benchmark) 3 | df$benchmark <- ifelse(df$benchmark == "BeamNG_RF_1_Complete", "BeamNG.AI.AF1", df$benchmark) 4 | df$benchmark <- ifelse(df$benchmark == "BeamNG_RF_1_5_selected", "BeamNG.AI.AF1.5", df$benchmark) 5 | return(df) 6 | } 7 | 8 | 9 | scalar3 <- function(x) {(x-min(x)) / (max(x) - min(x))} 10 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/road_model.py: -------------------------------------------------------------------------------- 1 | from shapely.geometry import LineString 2 | 3 | 4 | class RoadModel: 5 | def __init__(self, road_points: list[list]): 6 | self.coordinates = [(x[0], x[1]) for x in road_points] 7 | self.center_line: LineString = LineString(coordinates=self.coordinates) 8 | self.right_lane = self.center_line.buffer(distance=-5, single_sided=True) 9 | self.ideal_trajectory: LineString = self.center_line.parallel_offset(distance=2.5) 10 | -------------------------------------------------------------------------------- /levels_template/tig/terrain.terrain.json: -------------------------------------------------------------------------------- 1 | { 2 | "binaryFormat" : "version(char), size(unsigned int), heightMap(heightMapSize * heightMapItemSize), layerMap (layerMapSize * layerMapItemSize), materialNames", 3 | "datafile" : "levels/terrain2.ter", 4 | "heightMapItemSize" : 2, 5 | "heightMapSize" : 4194304, 6 | "heightmapImage" : "levels/tig/terrain.terrainheightmap.png", 7 | "layerMapItemSize" : 1, 8 | "layerMapSize" : 4194304, 9 | "materials" : [ "groundmodel_asphalt1" ], 10 | "size" : 2048 11 | } 12 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/fitness.m: -------------------------------------------------------------------------------- 1 | function [AUC] = fitness(X) 2 | global A Cost 3 | 4 | [~,n] = size(X); 5 | AUC = 0; 6 | for i=1:n 7 | % average distance between the the i-th test and the previous ones 8 | %averageDistance = mean(A(X(1:i), X(i))); 9 | if (i == 1) 10 | distance = A(X(i+1), X(i)); 11 | else 12 | distance = A(X(i-1), X(i)); 13 | end 14 | 15 | AUC = AUC + distance / Cost(X(i)) / i ; 16 | end 17 | 18 | AUC = -AUC; %% we want to maximize 19 | end 20 | -------------------------------------------------------------------------------- /tests/sdc_scissor/machine_learning_api/test_csv_loader.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pandas as pd 4 | 5 | from sdc_scissor.machine_learning_api.csv_loader import CSVLoader 6 | 7 | 8 | class TestCSVLoader: 9 | def test_if_loaded_data_is_pandas_dataframe(self, fs): 10 | fs.create_file("test.csv", contents="{'is_test_json' = true}") 11 | actual = CSVLoader.load_dataframe_from_csv(Path("test.csv")) 12 | expected = pd.DataFrame 13 | assert type(actual) is expected 14 | -------------------------------------------------------------------------------- /sdc_scissor/feature_extraction_api/segmentation_strategy.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class SegmentationStrategy(abc.ABC): 5 | """ 6 | Interface for implementing different segmentation strategies. 7 | """ 8 | 9 | @abc.abstractmethod 10 | def extract_segments(self, road_points): 11 | """ 12 | Abstract method to retrieve segments from a road, based on a specific strategy. 13 | 14 | :param road_points: Sequence of coordinates specifying the road. 15 | """ 16 | pass 17 | -------------------------------------------------------------------------------- /docs/api/feature_extraction_api.rst: -------------------------------------------------------------------------------- 1 | Feature Extraction API 2 | ====================== 3 | .. automodule:: sdc_scissor.feature_extraction_api.feature_extraction 4 | .. automodule:: sdc_scissor.feature_extraction_api.road_geometry_calculator 5 | .. automodule:: sdc_scissor.feature_extraction_api.segmentation_strategy 6 | .. automodule:: sdc_scissor.feature_extraction_api.angle_based_strategy 7 | .. automodule:: sdc_scissor.feature_extraction_api.equi_distance_strategy 8 | .. automodule:: sdc_scissor.feature_extraction_api.parameterized_uniform_strategy 9 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/CameraBookmarks/items.level.json: -------------------------------------------------------------------------------- 1 | {"name":"overviewbookmark","internalName":"NewCamera_0","class":"CameraBookmark","persistentId":"87700263-bfbd-48dd-a941-750c49025e6f","__parent":"CameraBookmarks","position":[17.95940017700195,-19.97909927368164,8.283820152282715],"datablock":"CameraBookmarkMarker","isAIControlled":"0","rotationMatrix":[0.6946083307266235,0.7193881273269653,-1.639127731323242e-07,-0.6582382917404175,0.6355648040771484,-0.4034596681594849,-0.2902439832687378,0.2802465558052063,0.9149974584579468]} 2 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/index.rst: -------------------------------------------------------------------------------- 1 | Decision Log 2 | ==================== 3 | This directory contains decision records for SDC-Scissor. 4 | 5 | For new ADRs, please use `adr-template.md `_ as basis. 6 | More information on MADR is available at https://adr.github.io/madr/. 7 | General information about architectural decision records is available at https://adr.github.io/. 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | :glob: 12 | 13 | * 14 | -------------------------------------------------------------------------------- /dbc_maps/dbc_map_carla.json: -------------------------------------------------------------------------------- 1 | { 2 | "wheelspeed": { 3 | "sim_signal_name": "speed", 4 | "min": 0, 5 | "max": 13107, 6 | "default": 0 7 | }, 8 | "throttle": { 9 | "sim_signal_name": "throttle", 10 | "min": 0, 11 | "max": 1, 12 | "default": 0 13 | }, 14 | "brake": { 15 | "sim_signal_name": "brake", 16 | "min": 0, 17 | "max": 1, 18 | "default": 0 19 | }, 20 | "steering": { 21 | "sim_signal_name": "steer", 22 | "min": -655.36, 23 | "max": 655.35, 24 | "default": 0 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dbc_maps/dbc_map_beamng.json: -------------------------------------------------------------------------------- 1 | { 2 | "wheelspeed": { 3 | "sim_signal_name": "wheelspeed", 4 | "min": 0, 5 | "max": 13107, 6 | "default": 0 7 | }, 8 | "throttle": { 9 | "sim_signal_name": "throttle", 10 | "min": 0, 11 | "max": 1, 12 | "default": 0 13 | }, 14 | "brake": { 15 | "sim_signal_name": "brake", 16 | "min": 0, 17 | "max": 1, 18 | "default": 0 19 | }, 20 | "steering": { 21 | "sim_signal_name": "steering", 22 | "min": -655.36, 23 | "max": 655.35, 24 | "default": 0 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/featureSelection.m: -------------------------------------------------------------------------------- 1 | %% Read the dataset 2 | %T = readtable('../datasets/fullroad/BeamNG_AI/BeamNG_RF_1/BeamNG_RF_1_Complete.csv'); 3 | %T = readtable('../datasets/fullroad/BeamNG_AI/BeamNG_RF_1_5/BeamNG_RF_1_5_selected.csv'); 4 | T = readtable('../datasets/fullroad/Driver_AI/DriverAI_Complete.csv'); 5 | TRaw = T(:,1:16); 6 | 7 | mat = TRaw{:,:} 8 | 9 | [idx,scores] = fsulaplacian(mat) 10 | 11 | 12 | %[coeff,score,latent, tsd, variance,mu] = pca(ma) 13 | 14 | %cs = cumsum(variance) 15 | %E = score * coeff(1:16,1:6) 16 | 17 | 18 | %E = score(:,1:6) 19 | -------------------------------------------------------------------------------- /docs/software_guidebook/constraints.md: -------------------------------------------------------------------------------- 1 | # Constraints 2 | This section provides information about the constraints imposed on the development of SDC-Scissor. 3 | 4 | ## Budget 5 | Since SDC-Scissor is built in the context of the EU-Project COSMOS the maintenance of this tool is financed till to the 6 | end of the project. After the COSMOS project there is no warranty for further maintenance. 7 | 8 | ## External Systems 9 | SDC-Scissor relies on several external libraries for performing machine learning tasks. Furthermore, it requires 10 | installation of simulators, e.g., BeamNG.tech, CARLA, etc. 11 | -------------------------------------------------------------------------------- /docs/api/sdc_scissor_cli.rst: -------------------------------------------------------------------------------- 1 | SDC-Scissor CLI 2 | =============== 3 | 4 | .. autofunction:: sdc_scissor.cli.generate_tests 5 | :noindex: 6 | 7 | .. autofunction:: sdc_scissor.cli.label_tests 8 | :noindex: 9 | 10 | .. autofunction:: sdc_scissor.cli.extract_features 11 | :noindex: 12 | 13 | .. autofunction:: sdc_scissor.cli.feature_statistics 14 | :noindex: 15 | 16 | .. autofunction:: sdc_scissor.cli.evaluate_models 17 | :noindex: 18 | 19 | .. autofunction:: sdc_scissor.cli.predict_tests 20 | :noindex: 21 | 22 | .. autofunction:: sdc_scissor.cli.evaluate_cost_effectiveness 23 | :noindex: 24 | -------------------------------------------------------------------------------- /levels_template/tig/art/terrains/materials.cs: -------------------------------------------------------------------------------- 1 | 2 | new TerrainMaterial() 3 | { 4 | diffuseMap = "levels/tig/art/terrains/Overlay_Grass-01"; 5 | detailMap = "levels/tig/art/terrains/Grass-01-D"; 6 | internalName = "groundmodel_asphalt1"; 7 | diffuseSize = "150"; 8 | detailDistance = "50"; 9 | normalMap = "levels/tig/art/terrains/Grass-01-N"; 10 | macroSize = "100"; 11 | macroStrength = "0.4"; 12 | macroDistance = "450"; 13 | macroMap = "levels/tig/art/terrains/Macro_grass"; 14 | detailSize = "3"; 15 | detailStrength = "0.7"; 16 | annotation = "GRASS_THAT_BEHAVES_LIKE_ASPHALT"; 17 | }; 18 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/vsbl.csv: -------------------------------------------------------------------------------- 1 | benchmark,config,config2,VD.magnitude,VD.estimate,wilcox.test.pvalue,avg.ga,avg.bl 2 | BeamNG.AI.AF1,mo-10_feature_GA,Greedy,large,0.8660416666666667,0,0.8090877979166666,0.78693 3 | BeamNG.AI.AF1,mo-10_feature_GA,random,large,1,0,0.8090877979166666,0.4998356666666667 4 | BeamNG.AI.AF1.5,mo-10_feature_GA,Greedy,large,0.9558333333333333,0,0.8041828041666667,0.77053 5 | BeamNG.AI.AF1.5,mo-10_feature_GA,random,large,1,0,0.8041828041666667,0.499838 6 | DeepDriving,mo-10_feature_GA,Greedy,large,0.996875,0,0.8567744125,0.8292 7 | DeepDriving,mo-10_feature_GA,random,large,1,0,0.8567744125,0.4999633333333333 8 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/faultDetection.m: -------------------------------------------------------------------------------- 1 | function [xValues, yValues] = faultDetection(X, T, Cost) 2 | labels = char(table2array(T(:,20))); 3 | 4 | nPoints = length(labels); 5 | faults = zeros(1,nPoints); 6 | for i=1:nPoints 7 | if (contains(labels(i,:),'unsafe')) 8 | faults(1,i) = 1; 9 | end 10 | end 11 | 12 | xValues = zeros(1, nPoints); 13 | yValues = zeros(1, nPoints); 14 | totalFaults = 0; 15 | totlalCost = 0; 16 | 17 | for i=1:nPoints 18 | totlalCost = totlalCost + Cost(X(i)); 19 | totalFaults = totalFaults + faults(X(i)); 20 | 21 | xValues(1,i) = totlalCost; 22 | yValues(1,i) = totalFaults; 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/main.m: -------------------------------------------------------------------------------- 1 | 2 | benchmarks = ["../datasets/fullroad/BeamNG_AI/BeamNG_RF_1/BeamNG_RF_1_Complete.csv" "../datasets/fullroad/BeamNG_AI/BeamNG_RF_1_5/BeamNG_RF_1_5_selected.csv" "../datasets/fullroad/Driver_AI/DriverAI_Complete.csv"]; 3 | configurations = ["10_feature_GA"] 4 | is_hybrid = [false] 5 | for h_status = 1 : length(is_hybrid) 6 | for benchmark_index = 1 : length(benchmarks) 7 | for config_index = 1 : length(configurations) 8 | parfor i = 1:30 9 | runSearch(i,configurations(config_index),benchmarks(benchmark_index),h_status) 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/obstacle_factory.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | from sdc_scissor.obstacle_api.bump import Bump 5 | from sdc_scissor.obstacle_api.delineator import Delineator 6 | from sdc_scissor.obstacle_api.tree import Tree 7 | 8 | 9 | class ObstacleFactory(abc.ABC): 10 | @abc.abstractmethod 11 | def create_bump(self) -> Bump: 12 | pass 13 | 14 | @abc.abstractmethod 15 | def create_delineator(self) -> Delineator: 16 | pass 17 | 18 | @abc.abstractmethod 19 | def create_tree(self) -> Tree: 20 | pass 21 | 22 | 23 | if __name__ == "__main__": 24 | logging.info("abstract_obstacle_factory.py") 25 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/config.py: -------------------------------------------------------------------------------- 1 | ga = {"population": 100, "n_gen": 50, "mut_rate": 0.4, "cross_rate": 1} 2 | 3 | model = { 4 | "speed": 9, # parameter for the simplified car model 5 | "map_size": 200, 6 | "steer_ang": 12, # a parameter for the simplified car model 7 | "min_len": 5, # minimal possible distance in meters 8 | "max_len": 30, # maximal possible disance to go straight in meters 9 | "min_angle": 10, # minimal angle of rotation in degrees 10 | "max_angle": 85, # maximal angle of rotation in degrees 11 | } 12 | 13 | files = {"ga_archive": ".\\GA_archive\\", "tc_img": ".\\TC_img\\", "tc_file": ".\\TC_file\\"} 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature 3 | about: Template for a new Feature 4 | title: '' 5 | labels: enhancement 6 | assignees: ChristianBirchler 7 | 8 | --- 9 | 10 | # User Story 11 | *User Story* 12 | 13 | # Acceptance Criteria 14 | ## Scenario 1: 15 | * Given: 16 | * When: 17 | * (And:) 18 | * Then: 19 | * (Given:) 20 | * (When:) 21 | * (Then:) 22 | 23 | # Definition of Done 24 | * Code is reviewed 25 | * Code is manually tested in dev and test environments 26 | * All code is integrated frequently 27 | * Integration testing is performed 28 | * SonarQube check is run to determine the code metrics 29 | * Refactoring is done 30 | 31 | # Tasks 32 | - [ ] *A concrete task* 33 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/mainGreedy.m: -------------------------------------------------------------------------------- 1 | benchmarks = ["../datasets/fullroad/BeamNG_AI/BeamNG_RF_1/BeamNG_RF_1_Complete.csv" "../datasets/fullroad/BeamNG_AI/BeamNG_RF_1_5/BeamNG_RF_1_5_selected.csv" "../datasets/fullroad/Driver_AI/DriverAI_Complete.csv"]; 2 | 3 | mat = ["benchmark" "config" "APFD" "elapsed_time"]; 4 | for benchmark_index = 1 : length(benchmarks) 5 | tic 6 | APFD = runGreedy("greedy",benchmarks(benchmark_index)) 7 | elapsed_time=toc 8 | [filepath,name,ext] = fileparts(benchmarks(benchmark_index)) 9 | mat = [mat; name "Greedy" APFD elapsed_time] 10 | end 11 | 12 | output_dir = strcat("../data/") 13 | writematrix(mat,strcat(output_dir,"greedy_results.csv")) 14 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/beamng_delineator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from beamngpy import ProceduralCylinder 4 | 5 | from sdc_scissor.obstacle_api.delineator import Delineator 6 | 7 | 8 | class BeamngDelineator(Delineator): 9 | def __init__(self, **kwargs): 10 | super().__init__(**kwargs) 11 | 12 | def get(self): 13 | return ProceduralCylinder( 14 | name="pyCylinder", 15 | pos=(self.x_pos, self.y_pos, self.z_pos), 16 | rot=self.rot, 17 | rot_quat=self.rot_quat, 18 | radius=self.radius, 19 | height=self.height, 20 | ) 21 | 22 | 23 | if __name__ == "__main__": 24 | logging.info("beamng_delineator.py") 25 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/test_case_problem.py: -------------------------------------------------------------------------------- 1 | from pymoo.model.problem import Problem 2 | 3 | 4 | class TestCaseProblem(Problem): 5 | """ 6 | Module to calculate the fitnes of the individuals 7 | """ 8 | 9 | def __init__(self): 10 | super().__init__(n_var=1, n_obj=2, n_constr=1, elementwise_evaluation=True) 11 | 12 | def _evaluate(self, x, out, *args, **kwargs): 13 | s = x[0] 14 | s.get_points() # transform the states into actual points (mutation and crossover operations are performed on states) 15 | s.remove_invalid_cases() 16 | s.eval_fitness() 17 | out["F"] = [s.fitness, s.novelty] 18 | out["G"] = 4 - s.fitness * (-1) 19 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/datasets/fullroad/BeamNG_AI/README.md: -------------------------------------------------------------------------------- 1 | # BeamNG.AI Dataset 2 | 3 | ## Risk Factor (RF) 4 | 5 | The behavior of BeamNG.AI can be parameterized with a so-called “aggression” factor (Risk Factor) which controls the risk the driver is willing to take in order to decrease the time to reach the designated destination. 6 | 7 | As explained by BeamNG.research developers, low aggression values (e.g., 0.7) result in a smooth driving, while high aggression values (e.g., 1.2 and above) result in an edgy driving, which might even “cut corners.” 8 | 9 | We considered for our experiments three different risk factors (1, 1.5, 2), which resulted in drastically different driving behavior from cautious (RF 1) to reckless (RF 2) 10 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.3.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - repo: https://github.com/psf/black 9 | rev: 23.1.0 10 | hooks: 11 | - id: black 12 | args: [--config=pyproject.toml] 13 | - repo: https://github.com/hadialqattan/pycln 14 | rev: v2.1.3 15 | hooks: 16 | - id: pycln 17 | args: [--config=pyproject.toml] 18 | - repo: https://github.com/pycqa/isort 19 | rev: 5.12.0 20 | hooks: 21 | - id: isort 22 | files: "\\.(py)$" 23 | args: [--settings-path=pyproject.toml] 24 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.projectKey=christianbirchler-org_sdc-scissor 2 | sonar.organization=christianbirchler-org 3 | 4 | # This is the name and version displayed in the SonarCloud UI. 5 | #sonar.projectName=sdc-scissor 6 | #sonar.projectVersion=2.0 7 | 8 | sonar.objc.file.suffixes=- 9 | sonar.c.file.suffixes=- 10 | sonar.cpp.file.suffixes=- 11 | 12 | sonar.python.version=3 13 | 14 | sonar.python.coverage.reportPaths=coverage.xml 15 | 16 | # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. 17 | sonar.sources=sdc_scissor 18 | sonar.exclusions=/sdc_scissor/testing_api/test_generators/**,/sdc_scissor/sdc_prioritizer/** 19 | 20 | # Encoding of the source code. Default is default system encoding 21 | #sonar.sourceEncoding=UTF-8 22 | -------------------------------------------------------------------------------- /docs/software_guidebook/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | This is the start of the software guidebook of SDC-Scissor; a cost-effective test selector for simulation-based tests 3 | in self-driving cars software. It includes a summary of the following: 4 | 5 | * The requirements, constraints and principles behind SDC-Scissor. 6 | * The software architecture, including the high-level technology choices and structure of the software. 7 | * The infrastructure architecture and how the software is deployed. 8 | * Operational and support aspects of the website. 9 | 10 | This book is based upon the concept of a software guidebook as described in 11 | [Software Architecture for Developers](https://leanpub.com/software-architecture-for-developers) book. The book is 12 | structured as follows: 13 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/docker_scripts/build-test-prioritization-image.sh: -------------------------------------------------------------------------------- 1 | # Stop and remove the previous docker container 2 | CHECK_CONTAINERS=$(docker container ls | grep 'test-prioritization-container') 3 | if [ -n "$CHECK_CONTAINERS" ]; then 4 | echo "Stopping and removing existing container..." 5 | docker stop test-prioritization-container > /dev/null 6 | docker rm test-prioritization-container > /dev/null 7 | fi 8 | 9 | # Remove previous docker image 10 | CHECK_IMAGES=$(docker images | grep 'test-prioritization-image') 11 | if [ -n "$CHECK_IMAGES" ]; then 12 | docker rmi 'test-prioritization-image' 13 | fi 14 | 15 | # Build the new image from Dockerfile.testPrioritization 16 | docker image build -t test-prioritization-image \ 17 | $(pwd) -f Dockerfile.testPrioritization 18 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/beamng_bump.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from beamngpy import ProceduralBump 4 | 5 | from sdc_scissor.obstacle_api.bump import Bump 6 | 7 | 8 | class BeamngBump(Bump): 9 | def __init__(self, **kwargs): 10 | super().__init__(**kwargs) 11 | 12 | def get(self): 13 | return ProceduralBump( 14 | name="pybump", 15 | pos=(self.x_pos, self.y_pos, self.z_pos), 16 | rot=self.rot, 17 | rot_quat=self.rot_quat, 18 | width=self.width, 19 | length=self.length, 20 | height=self.height, 21 | upper_length=self.upper_length, 22 | upper_width=self.upper_width, 23 | ) 24 | 25 | 26 | if __name__ == "__main__": 27 | logging.info("beamng_bump.py") 28 | -------------------------------------------------------------------------------- /sample_configs/label-tests.yml: -------------------------------------------------------------------------------- 1 | command: 'label-tests' 2 | options: 3 | home: 'C:\Users\birch\Documents\BeamNG.tech.v0.24.0.2\BeamNG.drive-0.24.0.2.13392' 4 | user: 'C:\Users\birch\Documents\BeamNG.drive' 5 | tests: 'C:\Users\birch\repositories\sdc-scissor\sample_tests' 6 | rf: 1.5 7 | oob: 0.3 8 | max_speed: 50 9 | interrupt: false 10 | obstacles: false 11 | bump_dist: 10 12 | delineator_dist: 3 13 | tree_dist: 10 14 | field_of_view: 120 15 | canbus: false 16 | can_stdout: true 17 | can_dbc: 'C:\Users\birch\repositories\sdc-scissor\sample_candb\beamng_pipeline_sample.dbc' 18 | can_dbc_map: 'C:\Users\birch\repositories\sdc-scissor\dbc_maps\dbc_map_beamng.json' 19 | can_interface: 'socketcan' 20 | can_channel: 'vcan0' 21 | can_bitrate: 250000 22 | influxdb_bucket: null 23 | influxdb_org: null 24 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/beamng_tree.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import uuid 3 | 4 | from beamngpy import StaticObject 5 | 6 | from sdc_scissor.obstacle_api.tree import Tree 7 | 8 | 9 | class BeamngTree(Tree): 10 | def __init__(self, **kwargs): 11 | super().__init__(**kwargs) 12 | 13 | def get(self): 14 | name = uuid.uuid4().hex[:3].upper() 15 | name = "pyTree_" + name 16 | 17 | return StaticObject( 18 | name=name, 19 | pos=(self.x_pos, self.y_pos, self.z_pos), 20 | rot=self.rot, 21 | rot_quat=self.rot_quat, 22 | scale=(0.5, 0.5, 0.5), 23 | shape="/levels/east_coast_usa/art/shapes/trees/trees_aspen/tree_aspen_large_a.dae", 24 | ) 25 | 26 | 27 | if __name__ == "__main__": 28 | logging.info("beamng_tree.py") 29 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/utils/frenet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def frenetv_to_cartesian(x0, y0, theta0, ss, kappas): 5 | """ 6 | Trapezoidal integration to compute Cartesian coordinates from given curvature values. 7 | """ 8 | xs = np.zeros(len(kappas)) 9 | ys = np.zeros(len(kappas)) 10 | thetas = np.zeros(len(kappas)) 11 | xs[0] = x0 12 | ys[0] = y0 13 | thetas[0] = theta0 14 | for i in range(thetas.shape[0] - 1): 15 | ss_diff_half = (ss[i + 1] - ss[i]) / 2.0 16 | thetas[i + 1] = thetas[i] + (kappas[i + 1] + kappas[i]) * ss_diff_half 17 | xs[i + 1] = xs[i] + (np.cos(thetas[i + 1]) + np.cos(thetas[i])) * ss_diff_half 18 | ys[i + 1] = ys[i] + (np.sin(thetas[i + 1]) + np.sin(thetas[i])) * ss_diff_half 19 | return (xs, ys) 20 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/0002-config-file-for-cli.md: -------------------------------------------------------------------------------- 1 | # Config File for CLI 2 | * status: accepted 3 | * date: 2022-07-01 4 | * deciders: Christian Birchler 5 | * consulted: - 6 | * informed: - 7 | 8 | ## Context and Problem Statement 9 | The configuration of SDC-Scissor was mainly done over the CLI. Since the tool has many various commands and options it 10 | is not very user-friendly to type all the configs in the console. 11 | 12 | ## Decision Drivers 13 | * Usability 14 | 15 | ## Considered Options 16 | * Use a separate configuration file 17 | * The user configures SDC-Scissor with environment variables 18 | 19 | ## Decision Outcome 20 | 21 | Chosen option: "Use a separate configuration file", because 22 | this enables the possibility to easily share the same setup via one config file that is useful for reproduce experiments. 23 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/delineator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | 5 | class Delineator(abc.ABC): 6 | def __init__( 7 | self, 8 | x_pos=None, 9 | y_pos=None, 10 | z_pos=None, 11 | radius=0.2, 12 | height=1.0, 13 | rot=None, 14 | rot_quat=(0, 0, 0, 1), 15 | obstacle_type="procedural", 16 | ): 17 | super().__init__() 18 | self.x_pos = x_pos 19 | self.y_pos = y_pos 20 | self.z_pos = z_pos 21 | self.radius = radius 22 | self.height = height 23 | self.rot = rot 24 | self.rot_quat = rot_quat 25 | self.obstacle_type = obstacle_type 26 | 27 | @abc.abstractmethod 28 | def get(self): 29 | pass 30 | 31 | 32 | if __name__ == "__main__": 33 | logging.info("delineator.py") 34 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/docker_scripts/run-test-prioritization-container.sh: -------------------------------------------------------------------------------- 1 | # Stop and remove the previous docker container 2 | CHECK_CONTAINERS=$(docker container ls | grep 'test-prioritization-container') 3 | if [ -n "$CHECK_CONTAINERS" ]; then 4 | echo "Stopping and removing existing container..." 5 | docker stop test-prioritization-container > /dev/null 6 | docker rm test-prioritization-container > /dev/null 7 | fi 8 | 9 | 10 | # Run a new docker container. 11 | docker run -dit --name test-prioritization-container \ 12 | --mount type=bind,source="$(pwd)/data",target=/home/user/experiment/data \ 13 | --mount type=bind,source="$(pwd)/testPrioritization",target=/home/user/experiment/testPrioritization \ 14 | test-prioritization-image bash 15 | 16 | 17 | 18 | docker exec -it test-prioritization-container bash -c "pip install --editable /home/user/experiment" 19 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/utils/frenet.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | # TODO: needs improvement in choosing correct number of x, y, theta 5 | def frenet_to_cartesian(x0, y0, theta0, ss, kappas): 6 | """ 7 | Numerical integration with trapezoidal rule to transform curvature samples 8 | given in Frenet frame to points in Cartesian frame 9 | """ 10 | xs = np.zeros(ss.shape[0]) 11 | ys = np.zeros(ss.shape[0]) 12 | thetas = np.zeros(ss.shape[0]) 13 | xs[0] = x0 14 | ys[0] = y0 15 | thetas[0] = theta0 16 | for i in range(thetas.shape[0] - 1): 17 | thetas[i + 1] = thetas[i] + (kappas[i + 1] + kappas[i]) * (ss[i + 1] - ss[i]) / 2.0 18 | xs[i + 1] = xs[i] + (ss[i + 1] - ss[i]) * np.cos(thetas[i]) 19 | ys[i + 1] = ys[i] + (ss[i + 1] - ss[i]) * np.sin(thetas[i]) 20 | return (xs, ys) 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug 3 | about: Template for reporting a bug 4 | title: '' 5 | labels: bug 6 | assignees: ChristianBirchler 7 | 8 | --- 9 | 10 | # Steps to reproduce the bug 11 | *Example:* 12 | 1. Go to ExampleURL.com 13 | 2. Click button A 14 | 3. Select option B 15 | 4. Type "123" 16 | 5. Click Enter 17 | 18 | # Resulting and expected behavior 19 | *Specify what happened compared to what you expected to happen.* 20 | 21 | *Examples:* 22 | * Result of operation "2x2" should be 4. In reality it is 5. 23 | * I get an error "404. Page not found." on ExampleURL.com/blog/bug-report-template. The page with this URL exists and should be accessible. 24 | 25 | # Screen recording or screenshot 26 | *Link to a video or some explanatory screenshots* 27 | 28 | # System information 29 | * OS: 30 | * CPU: 31 | * GPU: 32 | * Software: *e.g., version of SDC-Scissor* 33 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/tree.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | 5 | class Tree(abc.ABC): 6 | def __init__( 7 | self, 8 | x_pos=None, 9 | y_pos=None, 10 | z_pos=None, 11 | radius=0.2, 12 | height=1.0, 13 | rot=None, 14 | rot_quat=(0, 0, 0, 1), 15 | scale=(0.10, 0.10, 0.10), 16 | obstacle_type="static", 17 | ): 18 | super().__init__() 19 | self.x_pos = x_pos 20 | self.y_pos = y_pos 21 | self.z_pos = z_pos 22 | self.radius = radius 23 | self.height = height 24 | self.rot = rot 25 | self.rot_quat = rot_quat 26 | self.scale = scale 27 | self.obstacle_type = obstacle_type 28 | 29 | @abc.abstractmethod 30 | def get(self): 31 | pass 32 | 33 | 34 | if __name__ == "__main__": 35 | logging.info("Tree.py") 36 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/carla_obstacle_factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from sdc_scissor.obstacle_api.bump import Bump 4 | from sdc_scissor.obstacle_api.carla_bump import CarlaBump 5 | from sdc_scissor.obstacle_api.carla_delineator import CarlaDelineator 6 | from sdc_scissor.obstacle_api.carla_tree import CarlaTree 7 | from sdc_scissor.obstacle_api.delineator import Delineator 8 | from sdc_scissor.obstacle_api.obstacle_factory import ObstacleFactory 9 | from sdc_scissor.obstacle_api.tree import Tree 10 | 11 | 12 | class CarlaObstacleFactory(ObstacleFactory): 13 | def create_bump(self) -> Bump: 14 | return CarlaBump() 15 | 16 | def create_delineator(self) -> Delineator: 17 | return CarlaDelineator() 18 | 19 | def create_tree(self) -> Tree: 20 | return CarlaTree() 21 | 22 | 23 | if __name__ == "__main__": 24 | logging.info("carla_obstacle_factory.py") 25 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/beamng_obstacle_factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from sdc_scissor.obstacle_api.beamng_bump import BeamngBump 4 | from sdc_scissor.obstacle_api.beamng_delineator import BeamngDelineator 5 | from sdc_scissor.obstacle_api.beamng_tree import BeamngTree 6 | from sdc_scissor.obstacle_api.bump import Bump 7 | from sdc_scissor.obstacle_api.delineator import Delineator 8 | from sdc_scissor.obstacle_api.obstacle_factory import ObstacleFactory 9 | from sdc_scissor.obstacle_api.tree import Tree 10 | 11 | 12 | class BeamngObstacleFactory(ObstacleFactory): 13 | def create_bump(self) -> Bump: 14 | return BeamngBump() 15 | 16 | def create_delineator(self) -> Delineator: 17 | return BeamngDelineator() 18 | 19 | def create_tree(self) -> Tree: 20 | return BeamngTree() 21 | 22 | 23 | if __name__ == "__main__": 24 | logging.info("beamng_obstacle_factory.py") 25 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/permutationCrossover.m: -------------------------------------------------------------------------------- 1 | function xoverKids = permutationCrossover(parents,options,GenomeLength,~,~,thisPopulation) 2 | 3 | nKids = length(parents)/2; 4 | % Allocate space for the kids 5 | xoverKids = zeros(nKids,GenomeLength); 6 | 7 | % To move through the parents twice as fast as thekids are 8 | % being produced, a separate index for the parents is needed 9 | index = 1; 10 | 11 | for i=1:nKids 12 | % get parents 13 | parent1 = thisPopulation(parents(index),:); 14 | index = index + 1; 15 | parent2 = thisPopulation(parents(index),:); 16 | index = index + 1; 17 | 18 | % cut point is AFTER this index. 19 | cutPoint = randi(length(parent1), 1); 20 | head = parent1(1, 1:cutPoint); 21 | indexes = not(ismember(parent2, head)); 22 | tail = parent2(1, indexes); 23 | 24 | % make one child 25 | xoverKids(i,:) = [ head, tail ]; 26 | 27 | end 28 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /tests/sdc_scissor/feature_extraction_api/test_parameterized_uniform_strategy.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from sdc_scissor.feature_extraction_api.parameterized_uniform_strategy import ParameterizedUniformStrategy 4 | 5 | 6 | class TestParameterizedUniformStrategyTest: 7 | def test_straight_only(self): 8 | strategy = ParameterizedUniformStrategy("2", 0.05) 9 | 10 | road_points = [(x, 0) for x in range(1000)] 11 | 12 | segments = strategy.extract_segments(road_points) 13 | 14 | expected_segments = [(x * 10, x * 10 + 10) for x in range(99)] 15 | expected_segments.append((990, 999)) 16 | assert segments == expected_segments 17 | 18 | def test_road_is_too_short(self): 19 | strategy = ParameterizedUniformStrategy("2", 0.0005) 20 | 21 | road_points = [(x, 0) for x in range(1000)] 22 | 23 | with pytest.raises(Exception): 24 | strategy.extract_segments(road_points) 25 | -------------------------------------------------------------------------------- /sdc_scissor/simulator_api/simulator_factory.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from beamngpy import BeamNGpy 4 | 5 | from sdc_scissor.simulator_api.abstract_simulator import AbstractSimulator 6 | from sdc_scissor.simulator_api.beamng_simulator import BeamNGSimulator 7 | 8 | 9 | class SimulatorFactory: 10 | @staticmethod 11 | def get_beamng_simulator(home, user, rf: float, max_speed: float, fov: int) -> AbstractSimulator: 12 | """ 13 | 14 | :param home: 15 | :param user: 16 | :param rf: 17 | :param max_speed: 18 | :param fov: The field of view for a Camera e.g., 120 19 | :return: 20 | """ 21 | beamng = BeamNGpy(host="localhost", port=64256, home=home, user=user) 22 | beamng_simulator = BeamNGSimulator(beamng=beamng, rf=rf, max_speed=max_speed, fov=fov) 23 | return beamng_simulator 24 | 25 | 26 | if __name__ == "__main__": 27 | logging.info("* simulator_factory.py") 28 | -------------------------------------------------------------------------------- /docs/software_guidebook/quality_attributes.md: -------------------------------------------------------------------------------- 1 | # Quality Attributes 2 | This section provides information about the desired quality attributes (non-functional requirements) of SDC-Scissor. 3 | 4 | ## Performance 5 | Algorithms should, if possible, not implemented in Python due to performance reasons. Usage of third-party libraries 6 | that implemented algorithms in C/C++ should be preferred. 7 | 8 | ## Scalability 9 | Since SDC-Scissor is run on local machines only there is no scalability requirements yet. In case, SDC-Scissor is moving 10 | towards a cloud solution then this section will be updated and a decision log will be created. 11 | 12 | ## Usability 13 | SDC-Scissor should be easy to use by a CLI with descriptive commands. Furthermore, the usage of libraries should be 14 | simple by providing an easy architecture and descriptive classes and methods. 15 | 16 | ## Compatibility 17 | SDC-Scissor should work same for all Windows machines with the same Python version. 18 | -------------------------------------------------------------------------------- /docs/user_documentation/machine_learning.md: -------------------------------------------------------------------------------- 1 | # Test Outcome Prediction 2 | 3 | ![](../images/test-prediction.png) 4 | 5 | ## Evaluate Machine Learning Models 6 | 7 | ```text 8 | Usage: sdc-scissor evaluate-models [OPTIONS] 9 | 10 | Evaluate different machine learning models with a stratified cross 11 | validation approach. 12 | 13 | Options: 14 | --csv PATH Path to road_features.csv 15 | --models-dir PATH Directory to store the trained models 16 | --help 17 | ``` 18 | 19 | ```{eval-rst} 20 | .. autofunction:: sdc_scissor.cli.evaluate_models 21 | ``` 22 | 23 | ## Predict test outcomes before execution 24 | ```text 25 | Usage: sdc-scissor predict-tests [OPTIONS] 26 | 27 | Predict the most likely outcome of a test scenario without executing them in 28 | simulation. 29 | 30 | Options: 31 | -t, --tests PATH 32 | -c, --classifier PATH 33 | --help 34 | ``` 35 | 36 | ```{eval-rst} 37 | .. autofunction:: sdc_scissor.cli.predict_tests 38 | ``` 39 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/testPrioritization/crossover/PMX.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy as np 4 | from pymoo.core.crossover import Crossover 5 | 6 | 7 | def pmx(parent1, parent2): 8 | cutPoint = random.randint(0, len(parent1) - 1) 9 | head = parent1[:cutPoint] 10 | tail = [] 11 | for test_id in parent2: 12 | if test_id not in head: 13 | tail.append(test_id) 14 | tail = np.array(tail) 15 | 16 | offspring = np.concatenate((head, tail), axis=None) 17 | 18 | return offspring 19 | 20 | 21 | class PMXCrossover(Crossover): 22 | def __init__(self, **kwargs): 23 | super().__init__(2, 1, **kwargs) 24 | 25 | def _do(self, problem, X, **kwargs): 26 | _, n_matings, n_var = X.shape 27 | Y = np.full((self.n_offsprings, n_matings, n_var), -1, dtype=int) 28 | 29 | for i in range(n_matings): 30 | a, b = X[:, i, :] 31 | Y[0, i, :] = pmx(a, b) 32 | 33 | return Y 34 | -------------------------------------------------------------------------------- /sample_candb/beamng_pipeline_sample.dbc: -------------------------------------------------------------------------------- 1 | VERSION "" 2 | 3 | 4 | NS_ : 5 | NS_DESC_ 6 | CM_ 7 | BA_DEF_ 8 | BA_ 9 | VAL_ 10 | CAT_DEF_ 11 | CAT_ 12 | FILTER 13 | BA_DEF_DEF_ 14 | EV_DATA_ 15 | ENVVAR_DATA_ 16 | SGTYPE_ 17 | SGTYPE_VAL_ 18 | BA_DEF_SGTYPE_ 19 | BA_SGTYPE_ 20 | SIG_TYPE_REF_ 21 | VAL_TABLE_ 22 | SIG_GROUP_ 23 | SIG_VALTYPE_ 24 | SIGTYPE_VALTYPE_ 25 | BO_TX_BU_ 26 | BA_DEF_REL_ 27 | BA_REL_ 28 | BA_DEF_DEF_REL_ 29 | BU_SG_REL_ 30 | BU_EV_REL_ 31 | BU_BO_REL_ 32 | SG_MUL_VAL_ 33 | 34 | BS_: 35 | 36 | BU_: 37 | 38 | 39 | BO_ 177 sampleFrame2: 4 Vector__XXX 40 | SG_ wheelspeed : 16|16@1+ (0.2,0) [0|13107] "rpm" Vector__XXX 41 | 42 | BO_ 161 sampleFrame1: 7 Vector__XXX 43 | SG_ throttle : 16|16@1+ (0.0001,0) [0|1] "%" Vector__XXX 44 | SG_ brake : 0|16@1+ (0.0001,0) [0|1] "%" Vector__XXX 45 | SG_ steering : 32|17@1- (0.01,0) [-655.36|655.35] "degree" Vector__XXX 46 | 47 | 48 | 49 | BA_DEF_ "BusType" STRING ; 50 | BA_DEF_DEF_ "BusType" "CAN"; 51 | -------------------------------------------------------------------------------- /sdc_scissor/obstacle_api/bump.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | 5 | class Bump(abc.ABC): 6 | def __init__( 7 | self, 8 | x_pos=None, 9 | y_pos=None, 10 | z_pos=None, 11 | width=6, 12 | length=2, 13 | height=0.2, 14 | upper_length=1, 15 | upper_width=2, 16 | rot=None, 17 | rot_quat=(0, 0, 0, 1), 18 | obstacle_type="procedural", 19 | ): 20 | super().__init__() 21 | self.x_pos = x_pos 22 | self.y_pos = y_pos 23 | self.z_pos = z_pos 24 | self.width = width 25 | self.length = length 26 | self.height = height 27 | self.upper_length = upper_length 28 | self.upper_width = upper_width 29 | self.rot = rot 30 | self.rot_quat: tuple = rot_quat 31 | self.obstacle_type = obstacle_type 32 | 33 | @abc.abstractmethod 34 | def get(self): 35 | pass 36 | 37 | 38 | if __name__ == "__main__": 39 | logging.info("bump.py") 40 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/items.level.json: -------------------------------------------------------------------------------- 1 | {"name":"sky_and_sun","class":"SimGroup","persistentId":"43465d7d-0971-4933-89de-c4833b623141","__parent":"MissionGroup"} 2 | {"name":"CameraBookmarks","class":"SimGroup","persistentId":"10c228e4-8d9c-40fb-bdb6-d20febb48221","__parent":"MissionGroup"} 3 | {"name":"vegetation","class":"SimGroup","persistentId":"4f71ae35-2df2-4faf-9297-383b1a62642a","__parent":"MissionGroup"} 4 | {"name":"Water","class":"SimGroup","persistentId":"f0bb0c96-1127-4a30-bce3-cc64d2f856ec","__parent":"MissionGroup"} 5 | {"name":"PlayerDropPoints","class":"SimGroup","persistentId":"ebe56c16-ada2-45a5-8b62-07095074d620","__parent":"MissionGroup","Enabled":"1"} 6 | {"name":"terrain1","class":"TerrainBlock","persistentId":"4e2586a5-7955-49f0-bf35-73613f8cb368","__parent":"MissionGroup","position":[-1023.514038085938,-1114.167846679688,-539.9841918945313],"terrainFile":"levels/tig/terrain.ter"} 7 | {"name":"generated","class":"SimGroup","persistentId":"c545370a-d2b9-4f42-b6fa-578dda84bd81","__parent":"MissionGroup"} 8 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/vsrand.csv: -------------------------------------------------------------------------------- 1 | benchmark,config,VD.magnitude,VD.estimate,wilcox.test.pvalue,avg.ga,avg.rand 2 | BeamNG_RF_1_5_selected,default-GA,large,1,3.0179667984904466e-11,0.6486723333333333,0.5001526666666667 3 | BeamNG_RF_1_Complete,default-GA,large,1,3.0160753198909463e-11,0.6890196666666667,0.49993566666666667 4 | DriverAI_Complete,default-GA,large,1,3.0179667984904466e-11,0.6256363333333333,0.5000763333333333 5 | BeamNG_RF_1_5_selected,10_feature_GA,large,1,3.010407370963094e-11,0.6458696666666667,0.500062 6 | BeamNG_RF_1_Complete,10_feature_GA,large,1,3.012295606683293e-11,0.6885283333333333,0.4997356666666667 7 | DriverAI_Complete,10_feature_GA,large,1,3.0160753198909463e-11,0.6258066666666666,0.499993 8 | BeamNG_RF_1_5_selected,6_feature_GA,large,1,3.0179667984904466e-11,0.6429606666666666,0.49998733333333334 9 | BeamNG_RF_1_Complete,6_feature_GA,large,1,1.6911233892144776e-17,0.6984383333333333,0.499819 10 | DriverAI_Complete,6_feature_GA,large,1,3.014184922807266e-11,0.614009,0.49995466666666666 11 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/generate_test_case_sampling.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from pymoo.model.sampling import Sampling 3 | 4 | import sdc_scissor.testing_api.test_generators.ambiegen.config as cf 5 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.road_gen import RoadGen 6 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.solution import Solution 7 | 8 | 9 | class GenerateTestCaseSampling(Sampling): 10 | 11 | """ 12 | Module to generate the initial population 13 | """ 14 | 15 | def _do(self, problem, n_samples, **kwargs): 16 | generator = RoadGen( 17 | cf.model["map_size"], cf.model["min_len"], cf.model["max_len"], cf.model["min_angle"], cf.model["max_angle"] 18 | ) 19 | X = np.full((n_samples, 1), None, dtype=object) 20 | 21 | for i in range(n_samples): 22 | states = generator.test_case_generate() 23 | s = Solution() 24 | s.states = states 25 | X[i, 0] = s 26 | return X 27 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/testPrioritization/problem/TestPrioritizationMultiObjectiveProblem.py: -------------------------------------------------------------------------------- 1 | import multiprocessing as mp 2 | 3 | from pymoo.core.problem import Problem 4 | 5 | 6 | def ff_eval(x): 7 | global costs 8 | global distances 9 | f1 = 0.0 10 | f2 = 0.0 11 | 12 | for index, test_id in enumerate(x[:-1]): 13 | f1 += costs[test_id] / (index + 1) 14 | f2 -= distances[test_id][x[index + 1]] / (index + 1) 15 | return f1, f2 16 | 17 | 18 | class TestPrioritizationMultiObjectiveProblem(Problem): 19 | generation_counter = 0 20 | 21 | def __init__(self, genome_length, distancesp, costsp): 22 | super().__init__(n_var=genome_length, n_obj=2, n_constr=0, xl=0.0, xu=genome_length) 23 | global costs 24 | costs = costsp 25 | global distances 26 | distances = distancesp 27 | 28 | def _evaluate(self, x, out, *args, **kwargs): 29 | pool1 = mp.Pool(processes=5) 30 | 31 | new_rows2 = pool1.map(ff_eval, x) 32 | pool1.terminate() 33 | out["F"] = new_rows2 34 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.md 2 | :parser: myst_parser.sphinx_ 3 | 4 | Content 5 | ------- 6 | .. toctree:: 7 | :maxdepth: 1 8 | :caption: User Documentation 9 | 10 | user_documentation/quick_start 11 | user_documentation/test_generation 12 | user_documentation/test_execution 13 | user_documentation/feature_extraction 14 | user_documentation/machine_learning 15 | user_documentation/test_prioritization 16 | api/index 17 | 18 | .. toctree:: 19 | :maxdepth: 1 20 | :caption: Software Guidebook 21 | 22 | software_guidebook/introduction 23 | software_guidebook/context 24 | software_guidebook/functional_overview 25 | software_guidebook/quality_attributes 26 | software_guidebook/constraints 27 | software_guidebook/principles 28 | software_guidebook/software_architecture 29 | software_guidebook/infrastructure_architecture 30 | software_guidebook/deployment 31 | software_guidebook/operation_and_support 32 | software_guidebook/decisions/index 33 | contributing 34 | code_of_conduct 35 | -------------------------------------------------------------------------------- /docs/software_guidebook/context.md: -------------------------------------------------------------------------------- 1 | # Context 2 | SDC-Scissor is a tool that let you test self-driving cars more efficiently in simulation. It uses a machine-learning 3 | approach to select only relevant test scenarios so that the testing process is faster. Furthermore, the selected tests 4 | are diverse and try to challenge the car with corner cases. 5 | 6 | ![](../images/sdc-scissor-architecture.jpg) 7 | 8 | The purpose of this tool is to: 9 | 10 | 1. Provide a platform for testing self-driving cars (SDCs) within different simulators 11 | 2. Give access to different regression testing methodologies, e.g., test selection, prioritization and minimization. 12 | 3. Provide easy to use APIs for SDC testers 13 | 14 | ![](../images/sdc-scissor-APIs.jpg) 15 | 16 | ## Users 17 | SDC-Scissor has two types of users: 18 | 19 | 1. SDC testers 20 | 2. Developers of SDC software 21 | 22 | ## External Systems 23 | * BeamNG.tech simulator 24 | * Python 25 | * Various dependencies from [PyPI](https://pypi.org/) 26 | * beamngpy 27 | * click 28 | * Shapely 29 | * scikit-learn 30 | * pymoo 31 | -------------------------------------------------------------------------------- /docs/software_guidebook/principles.md: -------------------------------------------------------------------------------- 1 | # Principles 2 | 3 | ## Research 4 | SDC-Scissor is meant to be used for research in the first place. This means that the architecture of the tool must be 5 | highly modularized so that the development of new testing pipelines is easier and more productive. 6 | 7 | ## Best Practices 8 | The architecture and development of SDC-Scissor should be done by applying best practices such as: 9 | * Use of appropriate design patterns 10 | * Extensively testing on different levels: 11 | * Unit 12 | * System 13 | * Conducting code reviews for each pull request 14 | 15 | ## Continuous Integration 16 | The development of SDC-Scissor integrates the code changes continuously to the `main` branch (ideally multiple times per 17 | day). Features branched will be directly integrated into the `main` branch after approval of the code reviewer. The 18 | merge will trigger a GitHub workflow and executes all the tests. The outcome of the test execution will be reported in 19 | the according badge on top of the `README.md` or on top of the Documentation homepage. 20 | -------------------------------------------------------------------------------- /tests/sdc_scissor/obstacle_api/test_beamng_tree.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from beamngpy import StaticObject 4 | 5 | from sdc_scissor.obstacle_api.beamng_tree import BeamngTree 6 | 7 | 8 | # @pytest.mark.skip() 9 | class TestBeamngTree: 10 | def setup_class(self): 11 | pass 12 | 13 | def teardown_class(self): 14 | pass 15 | 16 | def setup_method(self): 17 | self.beamng_tree = BeamngTree() 18 | 19 | def test_get_new_beamng_tree(self, mocker): 20 | self.beamng_tree.x_pos = 100 21 | self.beamng_tree.y_pos = 100 22 | self.beamng_tree.z_pos = 0 23 | 24 | expected = StaticObject( 25 | name="test", 26 | pos=(100, 100, 0), 27 | rot=0, 28 | rot_quat=0, 29 | scale=(0.25, 0.25, 0.25), 30 | shape="/levels/west_coast_usa/art/shapes/trees/trees_conifer/BBZ_redwood1.dae", 31 | ).pos 32 | 33 | actual = self.beamng_tree.get().pos 34 | assert expected == actual 35 | 36 | 37 | if __name__ == "__main__": 38 | logging.info("test_beamng_tree.py") 39 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/0003-use-conventional-commit-messages.md: -------------------------------------------------------------------------------- 1 | # Use Conventional Commit Messages 2 | * status: accepted 3 | * date: 2022-07-20 4 | * deciders: [Christian Birchler](https://github.com/ChristianBirchler) 5 | 6 | ## Context and Problem Statement 7 | We want to have structured way of commit messages so that communication over them are enhanced among the collaborators. 8 | What convention should we follow? 9 | ## Decision Drivers 10 | * Simplicity 11 | 12 | ## Considered Options 13 | * Conventional Commits: https://www.conventionalcommits.org/en/v1.0.0/ 14 | * Defining own structure 15 | 16 | ## Decision Outcome 17 | 18 | Chosen option: "Conventional Commits", because 19 | * Conventional Commits convention is widely known. 20 | * GitHub can generate better release notes. 21 | * It is easy to apply. 22 | 23 | ### Positive Consequences 24 | * It is easier to cluster commit messages. 25 | * It is easier to identify past changes. 26 | 27 | ### Negative Consequences 28 | * There is some little effort to follow the structure of the convention. 29 | * Collaborators are technically not enforced to follow the convention. 30 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/Dockerfile.testPrioritization: -------------------------------------------------------------------------------- 1 | FROM python:3.9.1 2 | 3 | RUN adduser --disabled-password --gecos '' user 4 | USER user 5 | 6 | RUN mkdir ~/experiment 7 | 8 | 9 | # set work directory 10 | WORKDIR /home/user/experiment 11 | 12 | # Set virtual environment 13 | ENV VIRTUAL_ENV=venv 14 | RUN python3 -m venv $VIRTUAL_ENV 15 | ENV PATH="$VIRTUAL_ENV/bin:$PATH" 16 | 17 | 18 | # Install dependencies for setting up virtual env 19 | RUN python3 -m pip install --upgrade pip virtualenv 20 | RUN pip install --upgrade pip setuptools wheel 21 | USER root 22 | RUN apt-get update 23 | RUN apt install -y libgl1-mesa-glx 24 | USER user 25 | 26 | # Copy files required for setting up virtual env 27 | COPY setup.py /home/user/experiment 28 | COPY setup.cfg /home/user/experiment 29 | COPY requirements.txt /home/user/experiment 30 | 31 | 32 | # Copy the module 33 | # COPY /testPrioritization /home/user/experiment/testPrioritization 34 | 35 | # Copy inputs directory 36 | COPY /datasets /home/user/experiment/datasets 37 | 38 | # Install dependencies 39 | RUN pip install -r requirements.txt 40 | # setup the project 41 | # RUN pip install --editable /home/user/experiment 42 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 ERATO MMSD 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic_v/src/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 ERATO MMSD 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/initialPopulation.m: -------------------------------------------------------------------------------- 1 | function population = initialPopulation(GenomeLength,FitnessFcn,options) 2 | global Cost H BM; 3 | 4 | totalPopulation = sum(options.PopulationSize); 5 | population = zeros(totalPopulation, GenomeLength); 6 | 7 | if H 8 | permutations_dir = "../data/greedy-permutations/"; 9 | permutation = readmatrix(strcat(permutations_dir,BM,".csv")); 10 | population(1,:) = permutation; 11 | for i=2:totalPopulation/2 12 | child = permutation; 13 | % indexes of the element to permute 14 | mutationPoint1 = randi(GenomeLength, 1); 15 | mutationPoint2 = randi(GenomeLength, 1); 16 | 17 | % permutaton 18 | temp = child(mutationPoint1); 19 | child(mutationPoint1) = child(mutationPoint2); 20 | child(mutationPoint2) = temp; 21 | population(i,:) = child; 22 | end 23 | 24 | for i=(totalPopulation/2)+1:totalPopulation-1 25 | population(i,:) = randperm(GenomeLength); 26 | end 27 | 28 | else 29 | for i=1:totalPopulation-1 30 | population(i,:) = randperm(GenomeLength); 31 | end 32 | end 33 | 34 | [~, indexes] = sort(Cost, 'ascend'); 35 | population(totalPopulation,:) = indexes; 36 | end 37 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/runGreedy.m: -------------------------------------------------------------------------------- 1 | function [APFD_g] = runGreedy(configuration,csv_file) 2 | 3 | csv_file 4 | T = readtable(csv_file); 5 | Cost = table2array(T(:,19)); 6 | Features = table2array(T(:,1:16)); 7 | Differences = pdist(Features,"seuclidean"); 8 | Differences = squareform(Differences); 9 | 10 | 11 | 12 | maximum = max(max(Differences)); 13 | [x,y] = find(Differences==maximum); 14 | 15 | firstTest=x(1); 16 | secondTest=x(2); 17 | permutation_size = size(Features,1); 18 | 19 | permutation = [firstTest,secondTest]; 20 | 21 | 22 | for index = 3:permutation_size 23 | max_avg=0; 24 | max_index=0; 25 | for c = 1:permutation_size 26 | if ~ismember(c, permutation) 27 | dists=0; 28 | for already_selected_test_index = 1:(index-1) 29 | dists= dists + Differences(c,permutation(already_selected_test_index)); 30 | end 31 | avg = dists / Cost(c) ; 32 | if avg > max_avg 33 | max_avg = avg; 34 | max_index = c; 35 | end 36 | end 37 | end 38 | % Now, we add the detected max 39 | permutation(index) = max_index; 40 | end 41 | 42 | 43 | 44 | 45 | [a,b] = faultDetection(permutation, T, Cost); 46 | APFD_g = trapz(a, b) / max(a) / max(b); 47 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/datasets/fullroad/BeamNG_AI/BeamNG_RF_1_5/filter.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | 4 | selected_rows = [] 5 | remove = False 6 | 7 | final_csv_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "BeamNG_RF_1_5_selected.csv") 8 | with open(final_csv_dir, "r") as file: 9 | reader = csv.reader(file) 10 | for row in reader: 11 | if row[19] == "safe": 12 | selected_rows.append(row) 13 | else: 14 | if not remove: 15 | selected_rows.append(row) 16 | 17 | remove = not remove 18 | 19 | print(len(selected_rows)) 20 | 21 | final_csv_file = open(final_csv_dir, "wb") 22 | final_csv_writer = csv.writer(final_csv_file) 23 | 24 | fields = [ 25 | "direct_distance", 26 | "road_distance", 27 | "num_l_turns", 28 | "num_r_turns", 29 | "num_straights", 30 | "median_angle", 31 | "total_angle", 32 | "mean_angle", 33 | "std_angle", 34 | "max_angle", 35 | "min_angle", 36 | "median_pivot_off", 37 | "mean_pivot_off", 38 | "std_pivot_off", 39 | "max_pivot_off", 40 | "min_pivot_off", 41 | "start_time", 42 | "end_time", 43 | "duration_seconds", 44 | "safety", 45 | ] 46 | final_csv_writer.writerow(fields) 47 | 48 | for row in selected_rows: 49 | print(row) 50 | final_csv_writer.writerow(row) 51 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/0001-refactor-complex-sdc-scissor-v1.md: -------------------------------------------------------------------------------- 1 | # Refactor Complex SDC-Scissor v1.0 2 | * status: accepted 3 | * date: 2022-03-* 4 | * deciders: Christian Birchler 5 | * consulted: Sebastiano Panichella 6 | * informed: Sebastiano Panichella, Alessio Gambi, Sajad Khatiri 7 | 8 | ## Context and Problem Statement 9 | SDC-Scissor v1.0 is heavily based on the complex 10 | [SBST tool competition platform](https://github.com/se2p/tool-competition-av). To add another simulator or other 11 | test generators is difficult due to high coupling and high technical debt. 12 | 13 | 14 | ## Decision Drivers 15 | * providing reusable code 16 | * improve code quality 17 | 18 | ## Considered Options 19 | 1. Refactor the system and modularize the code base 20 | 2. Stick with the tool competition platform 21 | 22 | ## Decision Outcome 23 | Chosen option: Option 1, because 24 | it is the only option for having a sustainable platform for future research that allows to integrate easier new code 25 | to the repository without breaking existing functionality. 26 | 27 | 28 | ### Positive Consequences 29 | It improves the quality of the system and allows us to do more efficient research on SDCs. 30 | 31 | 32 | ### Negative Consequences 33 | Time of 3 work weeks are required to reengineer the system and to implement and to test. 34 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/testPrioritization/mutation/HybridMut.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import numpy as np 4 | from pymoo.core.mutation import Mutation 5 | 6 | 7 | class HybridMut(Mutation): 8 | def __init__(self, prob=1.0): 9 | super().__init__() 10 | self.prob = prob 11 | 12 | def _do(self, problem, X, **kwargs): 13 | X = X[: int(len(X) / 2)] 14 | Y = X.copy() 15 | 16 | for child in Y: 17 | mutation_points = [] 18 | while len(mutation_points) < 2: 19 | temp_point = random.randint(0, len(child) - 1) 20 | if temp_point not in mutation_points: 21 | mutation_points.append(temp_point) 22 | mutation_selector_prob = random.uniform(0, 1) 23 | if mutation_selector_prob <= 0.33: 24 | temp = child[mutation_points[0]] 25 | child[mutation_points[0]] = child[mutation_points[1]] 26 | child[mutation_points[1]] = temp 27 | 28 | elif mutation_selector_prob <= 0.66: 29 | min_index = min(mutation_points) 30 | max_index = max(mutation_points) 31 | 32 | child[min_index : max_index + 1] = np.array(list(reversed(child[min_index : max_index + 1]))) 33 | 34 | else: 35 | temp = child[mutation_points[0]] 36 | child = np.delete(child, mutation_points[0]) 37 | child = np.insert(child, mutation_points[1], temp) 38 | 39 | return Y 40 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/python/make-single-csv.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # dirs 4 | codes_directory = os.path.dirname(os.path.abspath(__file__)) 5 | root_directory = os.path.join(codes_directory, "..", "..") 6 | data_directory = os.path.join(root_directory, "data") 7 | 8 | # benchmarks, configs, and number of execution repeats used in this experiment 9 | benchmarks = ["BeamNG_RF_1_Complete", "BeamNG_RF_1_5_selected", "DriverAI_Complete"] 10 | configurations = ["default-GA", "10_feature_GA", "6_feature_GA"] 11 | number_of_runs = 30 12 | 13 | # Open the output csv 14 | out_csv_address = os.path.join(data_directory, "results.csv") 15 | out_csv = open(out_csv_address, "w") 16 | out_csv.write("benchmark,config,execution_id,best_fitness_value,APFD,elapsed_time,avg_rand_APFD,std_rand_APFD") 17 | out_csv.write("\n") 18 | out_csv.close() 19 | 20 | out_csv = open(out_csv_address, "a") 21 | for benchmark in benchmarks: 22 | for config in configurations: 23 | for run in range(1, number_of_runs + 1): 24 | current_data_directory = os.path.join(data_directory, benchmark, config, str(run)) 25 | current_results_csv = os.path.join(current_data_directory, "results.csv") 26 | 27 | f = open(current_results_csv) 28 | lines = f.readlines() 29 | # pop the title line 30 | lines.pop(0) 31 | 32 | for interesting_line in lines: 33 | prepared_line_to_save = benchmark + "," + interesting_line 34 | out_csv.write(prepared_line_to_save) 35 | 36 | out_csv.close() 37 | -------------------------------------------------------------------------------- /docs/user_documentation/test_generation.md: -------------------------------------------------------------------------------- 1 | # Test Generation 2 | SDC-Scissor uses state-of-the-art test generators from the previous editions of the 3 | SBST tool competition. The test generators of SDC-Scissor produce sequences of road points, 4 | which represent road points. By interpolating between those road points the actual roads are 5 | created. The following figure illustrates the concept of generating a test case for SDCs 6 | with SDC-Scissor. 7 | 8 | ![](../images/sdc-test-case.png) 9 | 10 | The test cases that specifies the road points are stored as JSON files in a provided directory. 11 | With those specified test cases in JSON files you you can run the simulations in the next step. 12 | 13 | The following command generates the test cases and stores them as JSON files in a given directory: 14 | 15 | ````text 16 | Usage: sdc-scissor generate-tests [OPTIONS] 17 | 18 | Generate tests (road specifications) for self-driving cars. 19 | 20 | Options: 21 | -c, --count INTEGER Number of tests the generator should produce 22 | (invalid roads inclusive) 23 | -k, --keep / --no-keep Keep the invalid roads produced by the test 24 | generator 25 | -d, --destination PATH Output directory to store the generated tests 26 | -t, --tool TEXT 27 | --help 28 | ```` 29 | 30 | ## Options 31 | The command `generate-tests` comes with several options. 32 | Those options are mainly about to configure the test generation process and how to persist the test specifications. 33 | 34 | ```{eval-rst} 35 | .. autofunction:: sdc_scissor.cli.generate_tests 36 | ``` 37 | -------------------------------------------------------------------------------- /tests/sdc_scissor/machine_learning_api/test_predictor.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from sdc_scissor.machine_learning_api.csv_loader import CSVLoader 5 | from sdc_scissor.machine_learning_api.model_evaluator import ModelEvaluator 6 | from sdc_scissor.machine_learning_api.predictor import Predictor 7 | from sdc_scissor.testing_api.test_loader import TestLoader 8 | from sdc_scissor.testing_api.test_validator import SimpleTestValidator 9 | 10 | 11 | class TestPredictor: 12 | def setup_class(self): 13 | self.test_dir = Path(os.path.dirname(__file__)).parent.parent.parent / "sample_tests" 14 | self.test_data_csv_path = Path(os.path.dirname(__file__)) / "test_road_features.csv" 15 | 16 | def test_prediction_with_train_data(self, fs): 17 | fs_test_dir = Path("/sample_tests") 18 | fs.add_real_directory(source_path=self.test_dir, read_only=False, target_path=fs_test_dir) 19 | fs.add_real_file(source_path=self.test_data_csv_path, target_path=fs_test_dir / "test_road_features.csv") 20 | 21 | data_path = fs_test_dir / "test_road_features.csv" 22 | dd = CSVLoader.load_dataframe_from_csv(data_path) 23 | 24 | test_loader = TestLoader(tests_dir=fs_test_dir, test_validator=SimpleTestValidator()) 25 | model_evaluator = ModelEvaluator(data_frame=dd, label="safety") 26 | fs.makedirs("models") 27 | model_evaluator.model_evaluation_with_balanced_training() 28 | model_evaluator.save_models(Path("models")) 29 | test_predictor = Predictor(test_loader=test_loader, joblib_classifier=Path("models/decision_tree.joblib")) 30 | test_predictor.predict() 31 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: "CD" 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | strategy: 12 | fail-fast: false 13 | matrix: 14 | python-version: ["3.10"] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Set up Python 18 | uses: actions/setup-python@v3 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | python -m pip install poetry 25 | poetry install 26 | - name: Build package 27 | run: | 28 | poetry build 29 | - uses: marvinpinto/action-automatic-releases@v1.2.1 30 | if: ${{ contains(github.ref_name, 'rc') || contains(github.ref_name, 'a') || contains(github.ref_name, 'b') }} 31 | with: 32 | repo_token: ${{ secrets.GITHUB_TOKEN }} 33 | prerelease: true 34 | files: | 35 | ./LICENSE.md 36 | ./dist/*.whl 37 | ./dist/*.tar.gz 38 | - uses: marvinpinto/action-automatic-releases@v1.2.1 39 | if: ${{ !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'a') && !contains(github.ref_name, 'b') }} 40 | with: 41 | repo_token: ${{ secrets.GITHUB_TOKEN }} 42 | prerelease: false 43 | files: | 44 | ./LICENSE.md 45 | ./dist/*.whl 46 | ./dist/*.tar.gz 47 | - name: Publish package 48 | run: | 49 | poetry publish -u ${{ secrets.PYPI_USERNAME }} -p ${{ secrets.PYPI_PASSWORD }} 50 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/0000-use-markdown-any-decision-records.md: -------------------------------------------------------------------------------- 1 | # Use Markdown Any Decision Records 2 | * status: accepted 3 | * date: 2022-06-13 4 | * deciders: [Christian Birchler](https://github.com/ChristianBirchler) 5 | * consulted: Patrick Baumgartner 6 | * informed: Sebastiano Panichella 7 | 8 | ## Context and Problem Statement 9 | We want to record any decisions made in this project independent whether decisions concern the architecture ("architectural decision record"), the code, or other fields. 10 | Which format and structure should these records follow? 11 | 12 | ## Considered Options 13 | * [MADR](https://adr.github.io/madr/) 3.0.0 – The Markdown Any Decision Records 14 | * [Michael Nygard's template](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions) – The first incarnation of the term "ADR" 15 | * [Sustainable Architectural Decisions](https://www.infoq.com/articles/sustainable-architectural-design-decisions) – The Y-Statements 16 | * Other templates listed at 17 | * Formless – No conventions for file format and structure 18 | 19 | ## Decision Outcome 20 | Chosen option: "MADR 3.0.0", because 21 | 22 | * Implicit assumptions should be made explicit. 23 | Design documentation is important to enable people understanding the decisions later on. 24 | See also [A rational design process: How and why to fake it](https://doi.org/10.1109/TSE.1986.6312940). 25 | * MADR allows for structured capturing of any decision. 26 | * The MADR format is lean and fits our development style. 27 | * The MADR structure is comprehensible and facilitates usage & maintenance. 28 | * The MADR project is vivid. 29 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/permutationMutation.m: -------------------------------------------------------------------------------- 1 | function mutationChildren = permutationMutation(parents,~,GenomeLength,~,state,~,thisPopulation,mutationRate) 2 | state.Generation 3 | nVar = length(thisPopulation(parents(1),:)); 4 | 5 | if nargin < 8 || isempty(mutationRate) 6 | mutationRate = 1/nVar; % default mutation rate 7 | end 8 | 9 | mutationChildren = zeros(length(parents),GenomeLength); 10 | for i=1:length(parents) 11 | child = thisPopulation(parents(i),:); 12 | 13 | for j=1:round(nVar * mutationRate) 14 | % indexes of the element to permute 15 | mutationPoint1 = randi(nVar, 1); 16 | mutationPoint2 = randi(nVar-1, 1); 17 | 18 | prob = rand(); 19 | 20 | % SWAP mutation 21 | if prob <= 0.33 22 | temp = child(mutationPoint1); 23 | child(mutationPoint1) = child(mutationPoint2); 24 | child(mutationPoint2) = temp; 25 | else 26 | % INVERT mutation 27 | if prob <= 0.66 28 | min_index = min(mutationPoint1, mutationPoint2); 29 | max_index = max(mutationPoint1, mutationPoint2); 30 | child(min_index:max_index) = child(max_index:-1:min_index); 31 | else 32 | % INSERT mutation 33 | temp = child(mutationPoint1); 34 | child(mutationPoint1) = []; 35 | child = [child(1:mutationPoint2), temp, child(mutationPoint2+1:nVar-1)]; 36 | end 37 | end 38 | end 39 | 40 | % storing the child 41 | mutationChildren(i,:) = child; 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "CI" 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | ci: 11 | if: github.repository == 'christianbirchler-org/sdc-scissor' 12 | runs-on: ubuntu-22.04 13 | steps: 14 | - name: Copy repository 15 | uses: actions/checkout@v3 16 | - name: Push to COSMOS repository 17 | env: 18 | gitlab_url: ${{ secrets.GITLAB_URL }} 19 | gitlab_token: ${{ secrets.GITLAB_TOKEN }} 20 | run: | 21 | echo Starting to push repo to gitlab 22 | git config user.name "Christian Birchler" 23 | git config user.email "birchler.chr@gmail.com" 24 | git fetch --unshallow 25 | git pull 26 | git remote set-url origin https://access-token:${gitlab_token}@${gitlab_url} 27 | git push origin main --force 28 | git push origin main --tag 29 | - name: Install Poetry 30 | run: pipx install poetry 31 | - name: Install tox 32 | run: pipx install tox 33 | - name: Set up Python with caching 34 | uses: actions/setup-python@v4 35 | with: 36 | python-version: '3.10' 37 | cache: 'poetry' 38 | - name: Install dependencies 39 | run: | 40 | sudo apt install python3-opengl 41 | poetry install 42 | - name: Check black code style 43 | run: | 44 | poetry run black -C -l 120 --check . 45 | - name: Test with pytest 46 | run: | 47 | tox 48 | - name: SonarCloud Scan 49 | uses: sonarsource/sonarcloud-github-action@master 50 | env: 51 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any 52 | SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 53 | -------------------------------------------------------------------------------- /sdc_scissor/config.py: -------------------------------------------------------------------------------- 1 | class _Config: 2 | def __init__(self): 3 | self.config = None 4 | 5 | @property 6 | def BEAMNG_HOME(self): 7 | return self.config["home"] 8 | 9 | @property 10 | def BEAMNG_USER(self): 11 | return self.config["user"] 12 | 13 | @property 14 | def RISK_FACTOR(self): 15 | return self.config["rf"] 16 | 17 | @property 18 | def MAX_SPEED(self): 19 | return self.config["max_speed"] 20 | 21 | @property 22 | def FIELD_OF_VIEW(self): 23 | return self.config["field_of_view"] 24 | 25 | @property 26 | def HAS_CAN_BUS(self): 27 | return self.config["canbus"] 28 | 29 | @property 30 | def CAN_STDOUT(self): 31 | return self.config["can_stdout"] 32 | 33 | @property 34 | def CAN_DBC_PATH(self): 35 | return self.config["can_dbc"] 36 | 37 | @property 38 | def CAN_DBC_MAP_PATH(self): 39 | return self.config["can_dbc_map"] 40 | 41 | @property 42 | def CAN_INTERFACE(self): 43 | return self.config["can_interface"] 44 | 45 | @property 46 | def CAN_CHANNEL(self): 47 | return self.config["can_channel"] 48 | 49 | @property 50 | def CAN_BITRATE(self): 51 | return self.config["can_bitrate"] 52 | 53 | @property 54 | def INFLUXDB_BUCKET(self): 55 | return self.config["influxdb_bucket"] 56 | 57 | @property 58 | def INFLUXDB_ORG(self): 59 | return self.config["influxdb_org"] 60 | 61 | @property 62 | def EXECUTION_START_TIME(self): 63 | return self.config["execution_start_date_time"] 64 | 65 | @property 66 | def CURRENT_TEST_ID(self): 67 | return self.config["current_test_id"] 68 | 69 | 70 | CONFIG = _Config() 71 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # This CITATION.cff file was generated with cffinit. 2 | # Visit https://bit.ly/cffinit to generate yours today! 3 | 4 | cff-version: 1.2.0 5 | title: SDC-Scissor 6 | message: >- 7 | A Tool for Cost-effective Simulation-based Test Selection 8 | in Self-driving Cars Software 9 | type: software 10 | authors: 11 | - given-names: Christian 12 | family-names: Birchler 13 | email: birc@zhaw.ch 14 | affiliation: Zurich University of Applied Sciences 15 | orcid: 'https://orcid.org/0000-0003-3987-0276' 16 | - given-names: Nicolas 17 | family-names: Ganz 18 | email: gann@zhaw.ch 19 | affiliation: Zurich University of Applied Sciences 20 | orcid: 'https://orcid.org/0000-0002-4165-0275' 21 | - given-names: Sajad 22 | family-names: Khatiri 23 | email: mazr@zhaw.ch 24 | affiliation: Zurich University of Applied Sciences 25 | orcid: 'https://orcid.org/0000-0003-0354-9747' 26 | - given-names: Alessio 27 | family-names: Gambi 28 | email: alessio.gambi@fh-krems.ac.at 29 | orcid: 'https://orcid.org/0000-0002-0132-6497' 30 | affiliation: University of Passau 31 | - given-names: Sebastiano 32 | family-names: Panichella 33 | email: panc@zhaw.ch 34 | affiliation: Zurich University of Applied Sciences 35 | orcid: 'https://orcid.org/0000-0003-4120-626X' 36 | identifiers: 37 | - type: doi 38 | value: 10.1016/j.scico.2023.102926 39 | description: >- 40 | Original Software Publication at the Science of 41 | Computer Programming journal 42 | - type: doi 43 | value: 10.1109/SANER53432.2022.00030 44 | description: Tool Demo Paper at SANER 2022 45 | repository-code: 'https://github.com/ChristianBirchler/sdc-scissor' 46 | url: 'https://sdc-scissor.readthedocs.io/' 47 | license: GPL-3.0 48 | -------------------------------------------------------------------------------- /tests/sdc_scissor/testing_api/test_test_loader.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | 3 | import pytest 4 | 5 | from sdc_scissor.testing_api.test_loader import TestLoader 6 | from sdc_scissor.testing_api.test_validator import NoIntersectionValidator, SimpleTestValidator 7 | 8 | 9 | class TestTestLoader: 10 | def test_empty_test_folder_should_have_no_next_test(self, mocker): 11 | empty_test_dir_mock = mocker.patch("pathlib.Path") 12 | tv = NoIntersectionValidator(SimpleTestValidator()) 13 | test_loader = TestLoader(tests_dir=empty_test_dir_mock, test_validator=tv) 14 | has_next = test_loader.has_next() 15 | assert not has_next 16 | 17 | def test_empty_test_folder_should_have_no_next_test_and_throw_exception_on_next(self, mocker): 18 | empty_test_dir_mock = mocker.patch("pathlib.Path") 19 | tv = NoIntersectionValidator(SimpleTestValidator()) 20 | test_loader = TestLoader(tests_dir=empty_test_dir_mock, test_validator=tv) 21 | with pytest.raises(Exception): 22 | _ = test_loader.next() 23 | 24 | def test_non_empty_test_folder_should_have_next_test(self, mocker, fs): 25 | fs.makedir(r"./tests") 26 | fs.makedir(r"./tests/unlabeled") 27 | fs.create_file(r"./tests/test.json", contents=r'{"key": 0}') 28 | fs.create_file(r"./tests/no-match.json", contents=r'{"key": 0}') 29 | fs.create_file(r"./tests/unlabeled/test.json", contents=r'{"key": 0}') 30 | non_empty_test_dir_mock = pathlib.Path("./tests") 31 | tv = NoIntersectionValidator(SimpleTestValidator()) 32 | test_loader = TestLoader(tests_dir=non_empty_test_dir_mock, test_validator=tv) 33 | test_loader.test_paths = [mocker.stub(), mocker.stub()] 34 | has_next = test_loader.has_next() 35 | assert has_next 36 | -------------------------------------------------------------------------------- /docs/c4model/workspace.dsl: -------------------------------------------------------------------------------- 1 | workspace { 2 | model { 3 | user = person "User" 4 | sdcScissor = softwareSystem "SDC-Scissor" { 5 | python = container "Python" { 6 | cli = component "CLI" 7 | simulatorAPI = component "Simulator API" 8 | testingAPI = component "Testing API" 9 | obstacleAPI = component "Obstacle API" 10 | mlAPI = component "Machine Learning API" 11 | featureExtractionAPI = component "Feature Extraction API" 12 | 13 | sklearn = component "sklearn" 14 | beamngpy = component "beamngpy" 15 | carla = component "carla" 16 | } 17 | beamngSimulator = container "BeamNG Simulator" 18 | carlaSimulator = container "CARLA Simulator" 19 | } 20 | 21 | 22 | user -> cli "uses" 23 | cli -> testingAPI "uses" 24 | cli -> mlAPI "uses" 25 | cli -> featureExtractionAPI "uses" 26 | testingAPI -> simulatorAPI "uses" 27 | testingAPI -> obstacleAPI "uses" 28 | mlAPI -> sklearn "uses" 29 | user -> sdcScissor "uses" 30 | python -> beamngSimulator "needs" 31 | python -> carlaSimulator "needs" 32 | simulatorAPI -> beamngpy "uses" 33 | simulatorAPI -> carla "uses" 34 | beamngpy -> beamngSimulator "controls" 35 | carla -> carlaSimulator "controls" 36 | } 37 | 38 | views { 39 | systemContext sdcScissor "Diagram1" { 40 | include * 41 | autoLayout 42 | } 43 | 44 | container sdcScissor "Diagram2" { 45 | include * 46 | autoLayout 47 | } 48 | 49 | component python "Diagram3" { 50 | include * 51 | autoLayout 52 | } 53 | 54 | theme default 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /docs/software_guidebook/operation_and_support.md: -------------------------------------------------------------------------------- 1 | # Operation and Support 2 | This section provides information about the operational and support aspects of SDC-Scissor. 3 | 4 | 5 | ## Usage 6 | For SDC-Scissor v2.0 you can use the tests in the `sample_tests` directory. 7 | 8 | The tool can be used with the following two commands: 9 | 10 | * PyPI: `sdc-scissor [COMMAND] [OPTIONS]` 11 | * Docker: `docker run --volume "$(pwd)/destination:/var/project/destination" --rm sdc-scissor [COMMAND] [OPTIONS]` (this will write all files written to `/var/project/destination` to the local folder `destination`) 12 | * Poetry: `poetry run sdc-scissor [COMMAND] [OPTIONS]` 13 | 14 | There are multiple commands to use. 15 | For simplifying the documentation only the command and their options are described. 16 | 17 | * Generation of tests: 18 | * `generate-tests` 19 | * Extract the features into a CSV file: 20 | * `extract-features` 21 | * Automated labeling of Tests: 22 | * `label-tests` 23 | * *Note:* This only works locally with BeamNG.research installed 24 | * Model evaluation: 25 | * `evaluate-models` 26 | * Evaluate the cost effectiveness of the models: 27 | * `evaluate_cost_effectiveness` 28 | * Test outcome prediction: 29 | * `predict-tests` 30 | 31 | The possible parameters are always documented with `--help`. 32 | 33 | ## Community 34 | We want to set up a community of researchers, developers, and users of SDCs. To enhance the research on these topics, we 35 | use the platform on [GitHub Discussions](https://github.com/ChristianBirchler/sdc-scissor/discussions) as the main 36 | communication channel to discuss everything related on SDCs and SDC-Scissor. Feel free to join the discussions if you 37 | want to discuss new potential features that can be implemented or research ideas. Everyone is welcome! 38 | 39 | For more detailed information about contributing to SDC-Scissor, please refer to the [Contributing](../contributing.rst) 40 | page. 41 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/table.R: -------------------------------------------------------------------------------- 1 | generate_APFD_table <- function(table_data,ga_name){ 2 | 3 | outputFile <- paste0("../../data/tables/ga-vs-baseline.tex") 4 | unlink(outputFile) 5 | 6 | # Redirect cat outputs to file 7 | sink(outputFile, append = TRUE, split = TRUE) 8 | cat("\\begin{tabular}{ l l | c c cl | c c c}\n") 9 | cat("\\hline", "\n") 10 | cat("\\textbf{Project}", "&", 11 | "\\textbf{GA Config.}", "&", 12 | "\\multicolumn{3}{c|}{\\textbf{Vs. Random}}", "&", 13 | "\\multicolumn{3}{c}{\\textbf{Vs. Greedy}}") 14 | cat("\\hline", "\n") 15 | cat(" \\\\", "\n") 16 | cat(" ", "&", 17 | " ", "&", 18 | "$\\hat{A}_{12}$", "&", 19 | "p", "&", 20 | "Magnitude", "&", 21 | "$\\hat{A}_{12}$", "&", 22 | "p", "&", 23 | "Magnitude") 24 | cat(" \\\\", "\n") 25 | 26 | benchmarks <- unique(greedy_df$benchmark) 27 | for(ben in benchmarks){ 28 | config <- ga_name 29 | temp <- table_data %>% 30 | filter(benchmark == ben & config.baseline == "random") 31 | 32 | rand_estimate <- formatC(temp$VD.estimate[1], digits=1, format="f", big.mark = ',') 33 | rand_p <- formatC(temp$wilcox.test.pvalue[1], format="g", big.mark = ',') 34 | rand_mag <- temp$VD.magnitude[1] 35 | 36 | temp <- table_data %>% 37 | filter(benchmark == ben & config.baseline == "Greedy") 38 | 39 | greedy_estimate <- formatC(temp$VD.estimate[1], digits=1, format="f", big.mark = ',') 40 | greedy_p <- formatC(temp$wilcox.test.pvalue[1], format="g", big.mark = ',') 41 | greedy_mag <- temp$VD.magnitude[1] 42 | 43 | cat(ben, "&", 44 | config, "&", 45 | rand_estimate, "&", 46 | rand_p, "&", 47 | rand_mag, "&", 48 | greedy_estimate, "&", 49 | greedy_p, "&", 50 | greedy_mag) 51 | 52 | cat(" \\\\", "\n") 53 | } 54 | cat("\\hline", "\n") 55 | cat("\\end{tabular}") 56 | 57 | # Restore cat outputs to console 58 | sink() 59 | } 60 | -------------------------------------------------------------------------------- /sdc_scissor/simulator_api/abstract_simulator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class AbstractSimulator(abc.ABC): 5 | def __init__(self): 6 | """ """ 7 | super().__init__() 8 | 9 | @abc.abstractmethod 10 | def open(self): 11 | """ 12 | Start the simulator in a separate process and enable interprocess communication by opening a socket. 13 | """ 14 | pass 15 | 16 | @abc.abstractmethod 17 | def close(self): 18 | """ 19 | Quit the simulator process and close the socket. 20 | """ 21 | pass 22 | 23 | @abc.abstractmethod 24 | def create_new_instance(self): 25 | """ 26 | Restart the simulator process. 27 | """ 28 | pass 29 | 30 | @abc.abstractmethod 31 | def stop_scenario(self): 32 | """ 33 | Stop the execution of an ongoing scenario in simulation. 34 | """ 35 | pass 36 | 37 | @abc.abstractmethod 38 | def start_scenario(self): 39 | """ 40 | Start the execution of a scenario in simulation. 41 | """ 42 | pass 43 | 44 | @abc.abstractmethod 45 | def load_scenario(self, test, scenario, obstacles: list): 46 | """ 47 | Prepare the simulator for a specific test scenario. 48 | 49 | :param test: An object defining the test scenario, i.e. vehicle setup, road, etc. 50 | :param scenario: Scenario object. 51 | :param obstacles: A list with obstacles to place in the virtual environment. 52 | """ 53 | pass 54 | 55 | @abc.abstractmethod 56 | def update_car(self): 57 | """ 58 | Retrieve the current sensor data of the vehicle in a running test that is executing in simulation within another process. 59 | """ 60 | pass 61 | 62 | @abc.abstractmethod 63 | def get_car_position(self): 64 | """ """ 65 | pass 66 | 67 | @abc.abstractmethod 68 | def get_sensor_data(self): 69 | pass 70 | -------------------------------------------------------------------------------- /sdc_scissor/can_api/can_msg_generator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import math 3 | import random 4 | 5 | import can 6 | import cantools.database 7 | 8 | from sdc_scissor.config import CONFIG 9 | 10 | 11 | class CANMessage: 12 | def __init__(self, **kwargs): 13 | self.__msg = can.Message(**kwargs) 14 | 15 | def __repr__(self): 16 | return can.Message.__repr__(self.__msg) 17 | 18 | def __str__(self): 19 | return can.Message.__str__(self.__msg) 20 | 21 | 22 | class ICANGenerationStrategy(abc.ABC): 23 | @abc.abstractmethod 24 | def generate(self) -> CANMessage: 25 | pass 26 | 27 | 28 | def _get_random_valid_can_message(dbc: cantools.database.Database) -> CANMessage: 29 | random_msg_template: cantools.database.Message = random.choice(dbc.messages) 30 | random_msg_signals_template = random_msg_template.signals 31 | 32 | random_can_msg_data = {} 33 | for signal_template in random_msg_signals_template: 34 | random_can_msg_data[signal_template.name] = random.randrange( 35 | math.ceil(signal_template.minimum), math.floor(signal_template.maximum) 36 | ) 37 | 38 | encoded_random_can_data = random_msg_template.encode(random_can_msg_data) 39 | can_message = CANMessage(arbitration_id=random_msg_template.frame_id, data=encoded_random_can_data) 40 | return can_message 41 | 42 | 43 | class RandomCANMessageGeneration(ICANGenerationStrategy): 44 | def __init__(self): 45 | self.__msg_pool = [] 46 | self.__can_dbc = CONFIG.CAN_DBC_PATH 47 | 48 | def generate(self) -> CANMessage: 49 | dbc = cantools.database.load_file(CONFIG.CAN_DBC_PATH) 50 | can_message: CANMessage = _get_random_valid_can_message(dbc) 51 | return can_message 52 | 53 | 54 | class CANMessageGenerator: 55 | def __init__(self, strategy: ICANGenerationStrategy): 56 | self.generation_strategy = strategy 57 | 58 | def generate(self) -> CANMessage: 59 | return self.generation_strategy.generate() 60 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/plot.R: -------------------------------------------------------------------------------- 1 | 2 | 3 | plotBoxPlot_APFD <- function(plot_df,l){ 4 | p <- ggplot(plot_df, aes(x=factor(config,levels = c("MO-SDC-Prioritizer","SO-SDC-Prioritizer","Greedy","random")), y=APFD, fill=factor(config))) + 5 | scale_colour_grey(start = 0, end = .9) + 6 | geom_boxplot() + 7 | xlab("") + 8 | ylab(l) + 9 | guides(fill=FALSE) + 10 | facet_grid(. ~ benchmark, margins = TRUE) + theme(axis.text.x = element_text(angle = 90, size = 12, face = "bold", margin = margin(t = 10, r = 0, b = 0, l = 0)), axis.text.y = element_text(size = 10, face="bold"), strip.text.x = element_text(size = 10, face="bold")) + 11 | stat_summary(fun.y=mean, geom="point", shape=23, size=2, fill = 'white') 12 | return(p) 13 | } 14 | 15 | 16 | plotBoxPlot_time <- function(ga_time_df,l){ 17 | p <- ggplot(ga_time_df, aes(x=factor(config,levels = unique(ga_time_df$config)), y=elapsed_time, fill=factor(config))) + 18 | scale_colour_grey(start = 0, end = .9) + 19 | geom_boxplot() + 20 | xlab("") + 21 | ylab(l) + 22 | guides(fill=FALSE) + 23 | facet_grid(. ~ benchmark, margins = TRUE) + theme(axis.text.x = element_text(angle = 90, size = 12, face = "bold", margin = margin(t = 10, r = 0, b = 0, l = 0)), axis.text.y = element_text(size = 10, face="bold"), strip.text.x = element_text(size = 10, face="bold")) + 24 | stat_summary(fun.y=mean, geom="point", shape=23, size=2, fill = 'white') 25 | return(p) 26 | } 27 | 28 | 29 | plotBoxPlot_pareto<- function(df){ 30 | p <- ggplot(df, aes(x = factor(benchmark,levels = unique(df$benchmark)), y = rate)) + 31 | geom_boxplot() + 32 | stat_summary(fun.y=mean, geom="point", shape=23, size=2, fill = 'white') + 33 | xlab("") + 34 | ylab("Better than greedy non-dominated solutions") + 35 | theme(axis.text.x = element_text(angle = 0, size = 12, face = "bold", margin = margin(t = 10, r = 0, b = 0, l = 0)), axis.text.y = element_text(size = 10, face="bold"), strip.text.x = element_text(size = 10, face="bold")) 36 | return(p) 37 | } 38 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_plotter.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | import matplotlib.pyplot as plt 4 | 5 | from sdc_scissor.testing_api.road_model import RoadModel 6 | 7 | 8 | class TestPlotter(abc.ABC): 9 | def plot(self, *args): 10 | pass 11 | 12 | 13 | class NullTestPlotter(TestPlotter): 14 | pass 15 | 16 | 17 | class TestPlotter(TestPlotter): 18 | def __init__(self): 19 | pass 20 | 21 | def plot(self, road_model: RoadModel): 22 | x_center_line = [i[0] for i in road_model.coordinates] 23 | y_center_line = [i[1] for i in road_model.coordinates] 24 | right_line = road_model.center_line.parallel_offset(side="right", distance=5.0) 25 | left_line = road_model.center_line.parallel_offset(side="left", distance=5.0) 26 | 27 | x_right_line = [] 28 | y_right_line = [] 29 | for i in range(0, round(right_line.length)): 30 | interpolated_point_right_line = right_line.interpolate(i) 31 | x_right_line.append(interpolated_point_right_line.x) 32 | y_right_line.append(interpolated_point_right_line.y) 33 | 34 | x_left_line = [] 35 | y_left_line = [] 36 | for i in range(0, round(left_line.length)): 37 | interpolated_point_left_line = left_line.interpolate(i) 38 | x_left_line.append(interpolated_point_left_line.x) 39 | y_left_line.append(interpolated_point_left_line.y) 40 | 41 | fig, ax = plt.subplots(figsize=(5, 5)) 42 | ax.plot(x_center_line, y_center_line, color="#ebba34", linewidth=2.0, label="center line") 43 | ax.plot(x_left_line, y_left_line, color="black", linewidth=2.0, label="left line") 44 | ax.plot(x_right_line, y_right_line, color="black", linewidth=2.0, label="right line") 45 | ax.plot([x_center_line[0]], [y_center_line[0]], "o", color="green", label="start") 46 | ax.plot([x_center_line[-1]], [y_center_line[-1]], "*", color="red", label="end") 47 | ax.legend() 48 | ax.set_xlabel("x") 49 | ax.set_ylabel("y") 50 | 51 | plt.show() 52 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | ## Feature Request 3 | When contributing to this repository, please first discuss the change you wish to make on our 4 | [GitHub Discussions](https://github.com/ChristianBirchler/sdc-scissor/discussions). 5 | If the feature request is approved then a new issue is opened to keep track 6 | of the development of the feature. The issue should follow the provided `Feature` issue template. 7 | In case you want to implement the feature by yourself then you can do it by forking the repository 8 | and create a pull request to the `main` branch. A pull request is only accepted if it adheres to 9 | the pull request process and all checks of the CI pipeline have passed. 10 | 11 | In case you are a direct contributor with access to this repository please note that we follow a 12 | [Scaled Trunk-Based Development](https://trunkbaseddevelopment.com/#scaled-trunk-based-development) 13 | branching strategy. Rather having dedicated release branches we tag the releases directly on the 14 | trunk (`main` branch). By tagging a release a dedicated CD pipeline is triggered that publishes 15 | the release automatically on PyPI. 16 | 17 | ### Pull Request Process 18 | 1. Fork the SDC-Scissor repository 19 | 2. Implement your feature (**NOTE**: Follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) commit message style!) 20 | 3. Provide test implementations 21 | - Unit tests 22 | - Integration tests 23 | - System tests 24 | 4. Format your code with `poetry run black -C -l 120 .` command 25 | 5. Open a pull request to SDC-Scissor's `main` branch 26 | 6. Follow the instructions of the code reviewer 27 | 28 | ## Bug Report 29 | In case you want to report a bug then open directly a new issue with the `Bug` issue template. 30 | Please fill all the sections in the provided template so that the resolvement of the bug can be 31 | done as efficient as possible. 32 | 33 | Please note we have a [Code of Conduct](https://github.com/ChristianBirchler/sdc-scissor/blob/main/CODE_OF_CONDUCT.md), 34 | please follow it in all your interactions with the project. 35 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/r-script/gavsga.csv: -------------------------------------------------------------------------------- 1 | benchmark,config.alg1,config.alg2,VD.magnitude,VD.estimate,wilcox.test.pvalue,avg.alg1,avg.alg2,VD.estimate.category 2 | BeamNG.AI.AF1,MO-GA.6F,MO-GA.FF,small,0.3826001953125,0,0.8027397062500004,0.8108647041666666,< 0.5 3 | BeamNG.AI.AF1,MO-GA.6F,MO-SDC-Prioritizer,small,0.4139510416666667,0,0.8027397062500004,0.8090877979166663,< 0.5 4 | BeamNG.AI.AF1,MO-GA.FF,MO-GA.6F,small,0.6173998046875,0,0.8108647041666662,0.8027397062499988,> 0.5 5 | BeamNG.AI.AF1,MO-GA.FF,MO-SDC-Prioritizer,negligible,0.5338394314236111,0,0.8108647041666662,0.8090877979166663,> 0.5 6 | BeamNG.AI.AF1,MO-SDC-Prioritizer,MO-GA.6F,small,0.5860489583333334,0,0.8090877979166675,0.8027397062499988,> 0.5 7 | BeamNG.AI.AF1,MO-SDC-Prioritizer,MO-GA.FF,negligible,0.4661605685763889,0,0.8090877979166675,0.8108647041666666,< 0.5 8 | BeamNG.AI.AF1.5,MO-GA.6F,MO-GA.FF,small,0.3930776909722222,0,0.8004425062499997,0.8057606791666666,< 0.5 9 | BeamNG.AI.AF1.5,MO-GA.6F,MO-SDC-Prioritizer,negligible,0.43365128038194445,0,0.8004425062499997,0.8041828041666657,< 0.5 10 | BeamNG.AI.AF1.5,MO-GA.FF,MO-GA.6F,small,0.6069223090277778,0,0.8057606791666663,0.8004425062499997,> 0.5 11 | BeamNG.AI.AF1.5,MO-GA.FF,MO-SDC-Prioritizer,negligible,0.5482263020833333,0,0.8057606791666663,0.8041828041666657,> 0.5 12 | BeamNG.AI.AF1.5,MO-SDC-Prioritizer,MO-GA.6F,negligible,0.5663487196180556,0,0.8041828041666662,0.8004425062499997,> 0.5 13 | BeamNG.AI.AF1.5,MO-SDC-Prioritizer,MO-GA.FF,negligible,0.45177369791666666,0,0.8041828041666662,0.8057606791666666,< 0.5 14 | DeepDriving,MO-GA.6F,MO-GA.FF,small,0.40118962673611114,0,0.8544256999999991,0.8567714958333332,< 0.5 15 | DeepDriving,MO-GA.6F,MO-SDC-Prioritizer,small,0.4182814236111111,0,0.8544256999999991,0.8567744124999997,< 0.5 16 | DeepDriving,MO-GA.FF,MO-GA.6F,small,0.5988103732638889,0,0.8567714958333335,0.8544256999999988,> 0.5 17 | DeepDriving,MO-GA.FF,MO-SDC-Prioritizer,negligible,0.5218762152777778,0,0.8567714958333335,0.8567744124999997,> 0.5 18 | DeepDriving,MO-SDC-Prioritizer,MO-GA.6F,small,0.5817185763888889,0,0.8567744124999999,0.8544256999999988,> 0.5 19 | DeepDriving,MO-SDC-Prioritizer,MO-GA.FF,negligible,0.4781237847222222,0,0.8567744124999999,0.8567714958333332,< 0.5 20 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "SDC-Scissor" 3 | version = "2.2.3" 4 | description = "A cost-effective test selection for self-driving cars in virtual environments" 5 | authors = ["Christian Birchler "] 6 | license = "GPL-3.0-or-later" 7 | readme = "README.md" 8 | repository = "https://github.com/ChristianBirchler/sdc-scissor" 9 | documentation = "https://sdc-scissor.readthedocs.io/" 10 | packages = [ 11 | {include = "sdc_scissor"} 12 | ] 13 | 14 | [tool.poetry.scripts] 15 | sdc-scissor = "sdc_scissor.cli:cli" 16 | 17 | [tool.poetry.dependencies] 18 | python = ">=3.10,<3.11" 19 | click = "^8.0.4" 20 | beamngpy = "=1.22" 21 | pandas = "^1.4.1" 22 | Shapely = "^1.8.1" 23 | scikit-learn = "^1.0.2" 24 | pymoo = "=0.4.2.2" 25 | PyYAML = "^6.0" 26 | icontract = "^2.6.2" 27 | cantools = "^38.0.1" 28 | influxdb-client = "^1.36.1" 29 | python-dotenv = "^1.0.0" 30 | 31 | [tool.poetry.group.dev.dependencies] 32 | parameterized = "^0.8.1" 33 | pytest = "^7.1.2" 34 | coverage = "^6.3.2" 35 | pytest-mock = "^3.7.0" 36 | black = "^22.3.0" 37 | notebook = "^6.4.12" 38 | pyfakefs = "^4.6.2" 39 | pynguin = "^0.27.0" 40 | pre-commit = "^3.0.4" 41 | tox = "^4.6.4" 42 | 43 | [tool.poetry.group.docs.dependencies] 44 | Sphinx = "^4.5.0" 45 | myst-parser = "^0.18.0" 46 | sphinx-rtd-theme = "^1.0.0" 47 | sphinxcontrib-mermaid = "^0.8.1" 48 | pylint = "^2.16.2" 49 | sphinxcontrib-youtube = "^1.2.0" 50 | 51 | [build-system] 52 | requires = ["poetry-core>=1.0.0"] 53 | build-backend = "poetry.core.masonry.api" 54 | 55 | [tool.black] 56 | line-length = 120 57 | 58 | [tool.pycln] 59 | all = true 60 | 61 | [tool.isort] 62 | line_length = 120 63 | multi_line_output = 3 64 | include_trailing_comma = true 65 | force_grid_wrap = 0 66 | use_parentheses = true 67 | ensure_newline_before_comments = true 68 | 69 | [tool.tox] 70 | legacy_tox_ini = """ 71 | [tox] 72 | skipsdist = true 73 | envlist = py310 74 | 75 | [testenv] 76 | allowlist_externals = poetry 77 | commands = 78 | poetry run coverage run --branch -m pytest tests 79 | poetry run coverage xml 80 | 81 | [coverage:run] 82 | relative_files = True 83 | source = sdc_scissor/ 84 | branch = True 85 | 86 | """ 87 | -------------------------------------------------------------------------------- /docs/software_guidebook/deployment.md: -------------------------------------------------------------------------------- 1 | # Deployment 2 | This section provides information about the mapping between the 3 | [software architecture](https://sdc-scissor.readthedocs.io/en/latest/software_guidebook/software_architecture.html) 4 | and the 5 | [infrastructure architecture](https://sdc-scissor.readthedocs.io/en/latest/software_guidebook/infrastructure_architecture.html). 6 | 7 | ## Simulator 8 | To use the full potential of SDC-Scissor you should follow the installation guidelines of the desired simulator provided 9 | on the according website of the simulator's provider. 10 | 11 | ## Installing SDC-Scissor 12 | The tool can be run locally using Python and Pip to install SDC-Scissor `pip install sdc-scissor`. Another way to run the tool is to clone the repository and install the tool with [Poetry](https://python-poetry.org/docs/) or with 13 | [Docker](https://docs.docker.com/get-docker/). If you want to run the tool with Docker then you can not use the `label-tests` command since the BeamNG simulator is Windows based and therefore not compatible with Docker. 14 | 15 | ## Requirements 16 | * BeamNG 17 | * When running the simulations a working installation of [BeamNG.tech version 0.24](https://beamng.tech) is required. 18 | * Additionally, this simulation cannot be run in a Docker container but must run locally. 19 | * `Python 3.9` 20 | * Note: `SDC-Scissor` is tested with this version of python only 21 | * [Pip](https://pypi.org/project/sdc-scissor/) 22 | * [Poetry](https://python-poetry.org/docs/) 23 | * [Docker](https://docs.docker.com/get-docker/) 24 | 25 | ## BeamNG 26 | For using SDC-Scissor with the BeamNG.tech simulator you can obtain an academic license from BeamNG from their 27 | [website](https://register.beamng.tech/). After installing BeamNG.tech you need to replace the `tig` level it in the 28 | user directory of BeamNG.tech (`C:\Users\myaccount\Documents\BeamNG.drive\0.24\levels`) wth the one from this repository 29 | (`levels_template/tig`). 30 | 31 | ## Installation commands 32 | To install the ``SDC-Scissor``, use one of the following approaches: 33 | 34 | * Pip: `pip install sdc-scissor` 35 | 36 | Or after cloning the SDC-Scissor repository: 37 | * Docker: `docker build --tag sdc-scissor .` 38 | * Poetry: `poetry install` 39 | -------------------------------------------------------------------------------- /tests/sdc_scissor/testing_api/test_test_validator.py: -------------------------------------------------------------------------------- 1 | from sdc_scissor.testing_api.test_validator import ( 2 | NoIntersectionValidator, 3 | NoTooSharpTurnsValidator, 4 | SimpleTestValidator, 5 | Test, 6 | ) 7 | 8 | 9 | class TestTestValidator: 10 | def test_simple_test_validator(self, mocker): 11 | test = mocker.patch("sdc_scissor.testing_api.test_validator.Test") 12 | tv = SimpleTestValidator() 13 | expected = True 14 | actual = tv.validate(test) 15 | assert actual == expected 16 | 17 | def test_no_intersection_decorator_on_valid_test(self): 18 | test = Test(0, road_points=[[10, 10], [15, 15], [100, 100], [150, 150]], test_outcome=None, test_duration=None) 19 | tv = NoIntersectionValidator(SimpleTestValidator()) 20 | expected = True 21 | actual = tv.validate(test) 22 | assert actual == expected 23 | 24 | def test_no_intersection_decorator_on_invalid_test(self): 25 | test = Test( 26 | 0, 27 | road_points=[[10, 10], [100, 10], [150, 50], [100, 100], [60, 60], [20, 0], [20, -10]], 28 | test_outcome=None, 29 | test_duration=None, 30 | ) 31 | tv = NoIntersectionValidator(SimpleTestValidator()) 32 | expected = False 33 | actual = tv.validate(test) 34 | assert actual == expected 35 | 36 | def test_no_sharp_turns_decorator_on_invalid_test(self): 37 | test = Test( 38 | 0, 39 | road_points=[[10, 10], [100, 10], [145, 10], [148, 10], [150, 10], [150, 11], [150, 12], [150, 13]], 40 | test_outcome=None, 41 | test_duration=None, 42 | ) 43 | tv = NoTooSharpTurnsValidator(SimpleTestValidator()) 44 | expected = False 45 | actual = tv.validate(test) 46 | assert actual == expected 47 | 48 | def test_no_sharp_turns_decorator_on_valid_test(self): 49 | test = Test( 50 | 0, road_points=[[10, 10], [100, 10], [145, 10], [148, 10], [150, 10]], test_outcome=None, test_duration=None 51 | ) 52 | tv = NoTooSharpTurnsValidator(SimpleTestValidator()) 53 | expected = True 54 | actual = tv.validate(test) 55 | assert actual == expected 56 | -------------------------------------------------------------------------------- /tests/sdc_scissor/simulator_api/test_beamng_simulator.py: -------------------------------------------------------------------------------- 1 | from sdc_scissor.simulator_api.beamng_simulator import BeamNGSimulator 2 | 3 | 4 | # @pytest.mark.skip() 5 | class TestBeamNGSimulator: 6 | def test_open_beamng(self, mocker): 7 | beamng_mock = mocker.patch("beamngpy.BeamNGpy") 8 | beamng_mock.open.return_value = None 9 | 10 | beamng_simulator = BeamNGSimulator(beamng=beamng_mock, rf=1.5, max_speed=120, fov=120) 11 | beamng_simulator.open() 12 | 13 | def test_close_beamng(self, mocker): 14 | beamng_mock = mocker.patch("beamngpy.BeamNGpy") 15 | 16 | beamng_simulator = BeamNGSimulator(beamng=beamng_mock, rf=1.5, max_speed=120, fov=120) 17 | beamng_simulator.close() 18 | 19 | def test_create_new_instance(self, mocker, fs): 20 | fs.makedir("./home") 21 | fs.create_file("./home/Bin64/BeamNG.drive.x64.exe") 22 | fs.makedir("./user") 23 | 24 | beamng_mock = mocker.patch("beamngpy.BeamNGpy") 25 | beamng_mock.home = "./home" 26 | beamng_mock.user = "./user" 27 | 28 | beamng_simulator = BeamNGSimulator(beamng=beamng_mock, rf=1.5, max_speed=120, fov=120) 29 | beamng_simulator.create_new_instance() 30 | 31 | def test_load_scenario(self, mocker, fs): 32 | test_mock = mocker.patch("sdc_scissor.testing_api.test.Test") 33 | test_mock.interpolated_road_points = [[1, 1], [100, 100]] 34 | 35 | mock_bump = mocker.patch("sdc_scissor.obstacle_api.bump.Bump") 36 | mock_bump.obstacle_type = "procedural" 37 | mock_tree = mocker.patch("sdc_scissor.obstacle_api.tree.Tree") 38 | mock_tree.obstacle_type = "static" 39 | 40 | mock_obstacles = [mock_bump, mock_tree] 41 | 42 | mock_scenario = mocker.patch("beamngpy.Scenario") 43 | 44 | fs.makedir("./home") 45 | fs.create_file("./home/Bin64/BeamNG.drive.x64.exe") 46 | fs.makedir("./user") 47 | 48 | beamng_mock = mocker.patch("beamngpy.BeamNGpy") 49 | beamng_mock.home = "./home" 50 | beamng_mock.user = "./user" 51 | 52 | beamng_simulator = BeamNGSimulator(beamng=beamng_mock, rf=1.5, max_speed=120, fov=120) 53 | beamng_simulator.load_scenario(test_mock, mock_scenario, mock_obstacles) 54 | -------------------------------------------------------------------------------- /tests/sdc_scissor/can_api/test_can_bus_handler.py: -------------------------------------------------------------------------------- 1 | import random 2 | from pathlib import Path 3 | 4 | from sdc_scissor.can_api.can_bus_handler import CanBusHandler 5 | from sdc_scissor.can_api.can_output import NoCANBusOutput 6 | from sdc_scissor.config import CONFIG 7 | 8 | 9 | class TestCanBusHandler: 10 | def setup_class(self): 11 | pass 12 | 13 | def test_data_to_stdout_transmission(self, mocker): 14 | candb_file_path = Path(__file__).parent.parent.parent.parent / "sample_candb" / "beamng_pipeline_sample.dbc" 15 | can_dbc_map_path = Path(__file__).parent.parent.parent.parent / "dbc_maps" / "dbc_map_beamng.json" 16 | CONFIG.config = { 17 | "canbus": False, 18 | "can_dbc": candb_file_path, 19 | "can_dbc_map": can_dbc_map_path, 20 | "can_interface": "", 21 | "can_bitrate": "", 22 | } 23 | output_handler = NoCANBusOutput() 24 | can_bus_handler = CanBusHandler(output_handler=output_handler) 25 | 26 | data = {"wheelspeed": 50} 27 | can_bus_handler.transmit_sensor_data_to_can_bus(data=data) 28 | 29 | def test_encoded_and_decoded_CAN_wheelspeed_messages_should_be_equal(self, mocker): 30 | candb_file_path = Path(__file__).parent.parent.parent.parent / "sample_candb" / "beamng_pipeline_sample.dbc" 31 | can_dbc_map_path = Path(__file__).parent.parent.parent.parent / "dbc_maps" / "dbc_map_beamng.json" 32 | CONFIG.config = { 33 | "canbus": False, 34 | "can_dbc": candb_file_path, 35 | "can_dbc_map": can_dbc_map_path, 36 | "can_interface": "", 37 | "can_bitrate": "", 38 | } 39 | output_handler = NoCANBusOutput() 40 | can_bus_handler = CanBusHandler(output_handler=output_handler) 41 | db = can_bus_handler.db 42 | for _ in range(9000): 43 | wheelspeed = random.randint(0, 9000) 44 | data = {"wheelspeed": wheelspeed} 45 | for msg_specs in db.messages: 46 | if not msg_specs.frame_id == 177: 47 | continue 48 | encoded_msg = db.encode_message(msg_specs.frame_id, data=data) 49 | decoded_msg = db.decode_message(msg_specs.frame_id, data=encoded_msg) 50 | print(decoded_msg) 51 | assert decoded_msg == data 52 | -------------------------------------------------------------------------------- /tests/sdc_scissor/pynguin/test_sdc_scissor_obstacle_api_beamng_delineator.py: -------------------------------------------------------------------------------- 1 | # Automatically generated by Pynguin. 2 | import pytest 3 | 4 | import sdc_scissor.obstacle_api.beamng_delineator as module_0 5 | 6 | 7 | # @pytest.mark.skip() 8 | def test_case_0(): 9 | beamng_delineator_0 = module_0.BeamngDelineator() 10 | assert ( 11 | f"{type(beamng_delineator_0).__module__}.{type(beamng_delineator_0).__qualname__}" 12 | == "sdc_scissor.obstacle_api.beamng_delineator.BeamngDelineator" 13 | ) 14 | assert beamng_delineator_0.x_pos is None 15 | assert beamng_delineator_0.y_pos is None 16 | assert beamng_delineator_0.z_pos is None 17 | assert beamng_delineator_0.radius == pytest.approx(0.2, abs=0.01, rel=0.01) 18 | assert beamng_delineator_0.height == pytest.approx(1.0, abs=0.01, rel=0.01) 19 | assert beamng_delineator_0.rot is None 20 | assert beamng_delineator_0.rot_quat == (0, 0, 0, 1) 21 | assert beamng_delineator_0.obstacle_type == "procedural" 22 | 23 | 24 | # @pytest.mark.skip() 25 | def test_case_1(): 26 | beamng_delineator_0 = module_0.BeamngDelineator() 27 | assert ( 28 | f"{type(beamng_delineator_0).__module__}.{type(beamng_delineator_0).__qualname__}" 29 | == "sdc_scissor.obstacle_api.beamng_delineator.BeamngDelineator" 30 | ) 31 | assert beamng_delineator_0.x_pos is None 32 | assert beamng_delineator_0.y_pos is None 33 | assert beamng_delineator_0.z_pos is None 34 | assert beamng_delineator_0.radius == pytest.approx(0.2, abs=0.01, rel=0.01) 35 | assert beamng_delineator_0.height == pytest.approx(1.0, abs=0.01, rel=0.01) 36 | assert beamng_delineator_0.rot is None 37 | assert beamng_delineator_0.rot_quat == (0, 0, 0, 1) 38 | assert beamng_delineator_0.obstacle_type == "procedural" 39 | var_0 = beamng_delineator_0.get() 40 | assert f"{type(var_0).__module__}.{type(var_0).__qualname__}" == "beamngpy.scenario.ProceduralCylinder" 41 | assert var_0.id == "pyCylinder" 42 | assert var_0.name == "pyCylinder" 43 | assert var_0.type == "ProceduralMesh" 44 | assert var_0.pos == (None, None, None) 45 | assert var_0.rot == (0, 0, 0, 1) 46 | assert var_0.scale == (1, 1, 1) 47 | assert var_0.opts == {} 48 | assert var_0.children == [] 49 | assert var_0.material is None 50 | assert var_0.radius == pytest.approx(0.2, abs=0.01, rel=0.01) 51 | assert var_0.height == pytest.approx(1.0, abs=0.01, rel=0.01) 52 | -------------------------------------------------------------------------------- /levels_template/tig/art/road/materials.cs: -------------------------------------------------------------------------------- 1 | 2 | singleton Material(tig_road_rubber_sticky) 3 | { 4 | mapTo = "tig_road_rubber_sticky"; 5 | diffuseMap[0] = "levels/tig/art/road/road_asphalt_2lane_d.dds"; 6 | doubleSided = "0"; 7 | translucentBlendOp = "LerpAlpha"; 8 | normalMap[0] = "levels/tig/art/road/road_asphalt_2lane_n.dds"; 9 | specularPower[0] = "1"; 10 | useAnisotropic[0] = "1"; 11 | materialTag0 = "RoadAndPath"; 12 | materialTag1 = "beamng"; 13 | specularMap[0] = "levels/tig/art/road/road_asphalt_2lane_s.dds"; 14 | reflectivityMap[0] = "levels/tig/art/road/road_rubber_sticky_d.dds"; 15 | cubemap = "global_cubemap_metalblurred"; 16 | translucent = "1"; 17 | translucentZWrite = "1"; 18 | alphaTest = "0"; 19 | alphaRef = "255"; 20 | castShadows = "0"; 21 | specularStrength[0] = "0"; 22 | }; 23 | 24 | 25 | 26 | 27 | singleton Material(tig_line_white) 28 | { 29 | mapTo = "tig_line_white"; 30 | doubleSided = "0"; 31 | translucentBlendOp = "LerpAlpha"; 32 | normalMap[0] = "levels/tig/art/road/line_white_n.dds"; 33 | specularPower[0] = "1"; 34 | useAnisotropic[0] = "1"; 35 | materialTag0 = "RoadAndPath"; 36 | materialTag1 = "beamng"; 37 | //cubemap = "cubemap_road_sky_reflection"; 38 | //specularMap[0] = "levels/tig/art/road/line_white_s.dds"; 39 | translucent = "1"; 40 | translucentZWrite = "1"; 41 | alphaTest = "0"; 42 | alphaRef = "255"; 43 | castShadows = "0"; 44 | specularStrength[0] = "0"; 45 | colorMap[0] = "levels/tig/art/road/line_white_d.dds"; 46 | annotation = "SOLID_LINE"; 47 | specularStrength0 = "0"; 48 | specularColor0 = "1 1 1 1"; 49 | materialTag2 = "driver_training"; 50 | }; 51 | 52 | singleton Material(tig_line_yellow) 53 | { 54 | mapTo = "tig_line_yellow"; 55 | doubleSided = "0"; 56 | translucentBlendOp = "LerpAlpha"; 57 | normalMap[0] = "levels/tig/art/road/line_white_n.dds"; 58 | specularPower[0] = "1"; 59 | useAnisotropic[0] = "1"; 60 | materialTag0 = "RoadAndPath"; 61 | materialTag1 = "beamng"; 62 | //cubemap = "cubemap_road_sky_reflection"; 63 | //specularMap[0] = "levels/tig/art/road/line_yellowblack_s.dds"; 64 | translucent = "1"; 65 | translucentZWrite = "1"; 66 | alphaTest = "0"; 67 | alphaRef = "255"; 68 | castShadows = "0"; 69 | specularStrength[0] = "0"; 70 | annotation = "SOLID_LINE"; 71 | colorMap[0] = "levels/tig/art/road/line_yellow_d.dds"; 72 | specularStrength0 = "0"; 73 | }; 74 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/testPrioritization/README.md: -------------------------------------------------------------------------------- 1 | # Python Test Prioritizer 2 | This tool gets the information regarding the tests written for self-driving cars and performs an evolutionary algorithm to deliver a test execution order that captures more unsafe behavior with less test execution time. 3 | 4 | ## Getting Started 5 | 6 | 1- Clone this repository. 7 | 8 | 2- Build the docker image: 9 | ```bash 10 | . docker_scripts/build-test-prioritization-image.sh 11 | ``` 12 | 13 | 3- Run the docker container: 14 | ```bash 15 | . docker_scripts/run-test-prioritization-container.sh 16 | ``` 17 | 18 | 4- Run the Test Prioritizer: 19 | 20 | ```bash 21 | docker exec -it test-prioritization-container bash -c "python testPrioritization/run.py [population size] [number of generations]" 22 | ``` 23 | This script has minimum __3__ and maximum __5__ input parameters. We will explain each of these arguments here: 24 | 25 | ARG1) __dataset CSV file:__ The address to one of the CSV files including all of the features in a set of tests. 26 | 27 | ARG2) __execution name:__ Each execution should have a unique name. The prioritizer stores all of the outputs under a directory with this name. 28 | 29 | ARG3) __output directory:__ The prioritizer stores all of the outputs in this directory. If the given directory does not exist, the script will create it. The processor will create a subdirectory with the given `` in the given ``. 30 | 31 | ARG4) __population size__ (optional)__:__ This argument indicates the number of solutions that the genetic algorithm produces in each generation. This value is set to 100 by default. 32 | 33 | ARG5) __number of generations__ (optional)__:__ This argument indicates the search budget of the prioritizer. This value is set to 2000. It means that The search process stops after generating the 2000th generation of test orders by default. 34 | ### Outputs 35 | The search process saves the Pareto front as `plot.png` in the output directory. It also saves the best-generated test order as `solution.txt` in the same directory. 36 | ### Example 37 | For example, the following command will execute the test prioritization for `BeamNG_RF_1_Complete` dataset with a population size of 50 and a search budget of 1000 generations. The output of this process will be saved under `data/results/config1`: 38 | 39 | 40 | ```bash 41 | docker exec -it test-prioritization-container bash -c "python testPrioritization/run.py datasets/fullroad/BeamNG_AI/BeamNG_RF_1/BeamNG_RF_1_Complete.csv 'config1' data/results/ 50 1000" 42 | ``` 43 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from pathlib import Path 4 | 5 | import numpy as np 6 | from scipy.interpolate import splev, splprep 7 | 8 | 9 | class Test: 10 | def __init__(self, test_id, road_points: list[list], test_outcome, test_duration=None): 11 | """ 12 | Class representing a test case. 13 | 14 | :param test_id: Unique test identifier 15 | :param road_points: Road points defining the test 16 | :param test_outcome: Outcome of the test execution 17 | :param test_duration: The duration of the test execution 18 | """ 19 | self.test_id = test_id 20 | self.is_valid = None 21 | self.test_outcome = test_outcome 22 | self.predicted_test_outcome = None 23 | self.test_duration = test_duration 24 | self.road_points = road_points 25 | self.interpolated_road_points = self.__interpolate(road_points) 26 | self.simulation_data = [] 27 | 28 | def save_as_json(self, file_path: Path): 29 | """ 30 | Save the test as a JSON file. 31 | 32 | :param file_path: File path to save the test as a json file 33 | """ 34 | logging.debug("* save_as_json") 35 | logging.info("Save test as a JSON file") 36 | nr_whitespaces_for_indentation_in_json_file = 2 37 | with open(file_path, "w") as fp: 38 | test_dict = vars(self) 39 | json.dump(test_dict, fp, indent=nr_whitespaces_for_indentation_in_json_file) 40 | 41 | @staticmethod 42 | def __interpolate(road_points: list[list]): 43 | """ 44 | 45 | :param road_points: 46 | :return: 47 | """ 48 | logging.debug("* __interpolate") 49 | road_matrix = np.array(road_points) 50 | x = road_matrix[:, 0] 51 | y = road_matrix[:, 1] 52 | num_nodes = 50 53 | 54 | if len(x) == 2: 55 | # With two points the only option is a straight segment 56 | k = 1 57 | elif len(y) == 3: 58 | # With three points we use an arc, using linear interpolation will result in invalid road tests 59 | k = 2 60 | else: 61 | # Otheriwse, use cubic splines 62 | k = 3 63 | 64 | pos_tck, *_pos_u = splprep([x, y], s=0, k=k) 65 | step_size = 1 / num_nodes 66 | unew = np.arange(0, 1 + step_size, step_size) 67 | x_new, y_new = splev(unew, pos_tck) 68 | new_road_points = np.column_stack((x_new, y_new)).tolist() 69 | 70 | return new_road_points 71 | 72 | 73 | if __name__ == "__main__": 74 | logging.info("* test.py") 75 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | 16 | sys.path.insert(0, os.path.abspath("..")) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = "SDC-Scissor" 22 | copyright = "2022-2023, Christian Birchler" 23 | author = "Christian Birchler" 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = "v2.0" 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ["sphinx.ext.autodoc", "myst_parser", "sphinxcontrib.mermaid", "sphinxcontrib.youtube"] 35 | autodoc_default_options = { 36 | "member-order": "bysource", 37 | "special-members": "__init__", 38 | "undoc-members": True, 39 | "members": True, 40 | } 41 | 42 | github_url = "https://github.com/ChristianBirchler/sdc-scissor" 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ["_templates"] 46 | 47 | # List of patterns, relative to source directory, that match files and 48 | # directories to ignore when looking for source files. 49 | # This pattern also affects html_static_path and html_extra_path. 50 | exclude_patterns = ["software_guidebook/decisions/adr-template.md"] 51 | 52 | 53 | # -- Options for HTML output ------------------------------------------------- 54 | 55 | # The theme to use for HTML and HTML Help pages. See the documentation for 56 | # a list of builtin themes. 57 | # 58 | html_theme = "sphinx_rtd_theme" 59 | 60 | html_theme_options = {"analytics_id": "G-EF6L7M0P5T", "titles_only": True, "sticky_navigation": False} 61 | 62 | # Add any paths that contain custom static files (such as style sheets) here, 63 | # relative to this directory. They are copied after the builtin static files, 64 | # so a file named "default.css" will overwrite the builtin "default.css". 65 | # html_static_path = ['_static'] 66 | 67 | epub_show_urls = "footnote" 68 | -------------------------------------------------------------------------------- /sdc_scissor/machine_learning_api/predictor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pathlib import Path 3 | 4 | import joblib 5 | import numpy as np 6 | import pandas as pd 7 | 8 | from sdc_scissor.feature_extraction_api.angle_based_strategy import AngleBasedStrategy 9 | from sdc_scissor.feature_extraction_api.feature_extraction import FeatureExtractor, RoadFeatures 10 | from sdc_scissor.testing_api.test_loader import TestLoader 11 | 12 | 13 | class Predictor: 14 | def __init__(self, test_loader: TestLoader, joblib_classifier: Path, label="safety"): 15 | """ 16 | 17 | :param test_loader: 18 | :param joblib_classifier: 19 | :param label: 20 | """ 21 | self.test_loader = test_loader 22 | self.__classifier = joblib.load(joblib_classifier) 23 | self.feature_extractor = FeatureExtractor(segmentation_strategy=AngleBasedStrategy()) 24 | self.label = label 25 | self.X_model_attributes = [ 26 | "direct_distance", 27 | "max_angle", 28 | "max_pivot_off", 29 | "mean_angle", 30 | "mean_pivot_off", 31 | "median_angle", 32 | "median_pivot_off", 33 | "min_angle", 34 | "min_pivot_off", 35 | "num_l_turns", 36 | "num_r_turns", 37 | "num_straights", 38 | "road_distance", 39 | "std_angle", 40 | "std_pivot_off", 41 | "total_angle", 42 | "mean_road_diversity", 43 | "full_road_diversity", 44 | ] 45 | 46 | def predict(self): 47 | """ 48 | Predict the outcome of the tests and saves is as a json property to the test file. 49 | """ 50 | logging.info("predict") 51 | while self.test_loader.has_next(): 52 | test, test_path = self.test_loader.next() 53 | road_features: RoadFeatures = self.feature_extractor.extract_features(test) 54 | road_features_dict = road_features.to_dict() 55 | road_features_df = pd.DataFrame(road_features_dict) 56 | X: np.ndarray = road_features_df[self.X_model_attributes].to_numpy() 57 | y_pred: int = self.__classifier.predict(X)[0] 58 | if y_pred == 1: 59 | test.predicted_test_outcome = "FAIL" 60 | elif y_pred == 0: 61 | test.predicted_test_outcome = "PASS" 62 | else: 63 | logging.warning("Prediction failed!") 64 | raise Exception("Prediction failed!") 65 | logging.info("predicted outcome: {}".format(test.predicted_test_outcome)) 66 | test.save_as_json(file_path=test_path) 67 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/solution.py: -------------------------------------------------------------------------------- 1 | import sdc_scissor.testing_api.test_generators.ambiegen.config as cf 2 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.car_road import Map 3 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.vehicle import Car 4 | 5 | # from code_pipeline.beamng_executor import BeamngExecutor 6 | # from code_pipeline.tests_generation import RoadTestFactory 7 | 8 | 9 | class Solution: 10 | 11 | """ 12 | This is a class to represent one individual of the genetic algorithm 13 | """ 14 | 15 | def __init__(self): 16 | self.road_points = [] 17 | self.states = {} 18 | self.car = Car(cf.model["speed"], cf.model["steer_ang"], cf.model["map_size"]) 19 | self.road_builder = Map(cf.model["map_size"]) 20 | self.fitness = 0 21 | self.car_path = [] 22 | self.novelty = 0 23 | self.intp_points = [] 24 | self.too_sharp = 0 25 | self.just_fitness = 0 26 | 27 | def eval_fitness(self): 28 | road = self.road_points 29 | if not road: # if no road points were calculated yet 30 | self.get_points() 31 | self.remove_invalid_cases() 32 | road = self.road_points 33 | 34 | if len(self.road_points) <= 2: 35 | self.fitness = 0 36 | else: 37 | self.intp_points = self.car.interpolate_road(road) 38 | self.fitness, self.car_path = self.car.execute_road(self.intp_points) 39 | 40 | return 41 | 42 | # def car_model_fit(self): 43 | 44 | # the_executor = BeamngExecutor(cf.model["map_size"]) 45 | 46 | # the_test = RoadTestFactory.create_road_test(self.road_points) 47 | 48 | # fit = the_executor._eval_tc(the_test) 49 | 50 | # return fit 51 | 52 | def get_points(self): 53 | self.road_points = self.road_builder.get_points_from_states(self.states) 54 | 55 | def remove_invalid_cases(self): 56 | self.states, self.road_points = self.road_builder.remove_invalid_cases(self.road_points, self.states) 57 | 58 | def calc_novelty(self, old, new): 59 | novelty = 0 60 | difference = abs(len(new) - len(old)) / 2 61 | novelty += difference 62 | if len(new) <= len(old): 63 | shorter = new 64 | else: 65 | shorter = old 66 | for tc in shorter: 67 | if old[tc]["state"] == new[tc]["state"]: 68 | value_list = [old[tc]["value"], new[tc]["value"]] 69 | ratio = max(value_list) / min(value_list) 70 | if ratio >= 2: 71 | novelty += 0.5 72 | else: 73 | novelty += 1 74 | return -novelty 75 | 76 | @property 77 | def n_states(self): 78 | return len(self.states) 79 | -------------------------------------------------------------------------------- /docs/user_documentation/test_prioritization.md: -------------------------------------------------------------------------------- 1 | # Test Prioritization 2 | 3 | This tool gets the information regarding the tests written for self-driving cars and performs an evolutionary algorithm to deliver a test execution order that captures more unsafe behavior with less test execution time. 4 | 5 | ```{eval-rst} 6 | .. youtube:: Mo9bgtNkeeM 7 | :width: 695 8 | :height: 480 9 | ``` 10 | 11 | ## Getting Started 12 | 13 | 1. Clone this repository and go the `sdc_prioritizer` directory: 14 | ```bash 15 | git clone https://github.com/christianbirchler-org/sdc-scissor.git 16 | cd sdc-scissor/sdc_scissor/sdc_prioritizer 17 | ``` 18 | 2. Build the docker image: 19 | ```bash 20 | . docker_scripts/build-test-prioritization-image.sh 21 | ``` 22 | 3. Run the docker container: 23 | ```bash 24 | . docker_scripts/run-test-prioritization-container.sh 25 | ``` 26 | 4. Run the Test Prioritizer: 27 | 28 | ```bash 29 | docker exec -it test-prioritization-container bash -c "python testPrioritization/run.py [population size] [number of generations]" 30 | ``` 31 | This script has minimum __3__ and maximum __5__ input parameters. We will explain each of these arguments here: 32 | 33 | - ARG1) __dataset CSV file:__ The address to one of the CSV files including all of the features in a set of tests. 34 | - ARG2) __execution name:__ Each execution should have a unique name. The prioritizer stores all of the outputs under a directory with this name. 35 | - ARG3) __output directory:__ The prioritizer stores all of the outputs in this directory. If the given directory does not exist, the script will create it. The processor will create a subdirectory with the given `` in the given ``. 36 | - ARG4) __population size__ (optional)__:__ This argument indicates the number of solutions that the genetic algorithm produces in each generation. This value is set to 100 by default. 37 | - ARG5) __number of generations__ (optional)__:__ This argument indicates the search budget of the prioritizer. This value is set to 2000. It means that The search process stops after generating the 2000th generation of test orders by default. 38 | 39 | ### Outputs 40 | The search process saves the Pareto front as `plot.png` in the output directory. It also saves the best-generated test order as `solution.txt` in the same directory. 41 | 42 | ### Example 43 | For example, the following command will execute the test prioritization for `BeamNG_RF_1_Complete` dataset with a population size of 50 and a search budget of 1000 generations. The output of this process will be saved under `data/results/config1`: 44 | 45 | ```bash 46 | docker exec -it test-prioritization-container bash -c "python testPrioritization/run.py datasets/fullroad/BeamNG_AI/BeamNG_RF_1/BeamNG_RF_1_Complete.csv 'config1' data/results/ 50 1000" 47 | ``` 48 | -------------------------------------------------------------------------------- /sdc_scissor/feature_extraction_api/parameterized_uniform_strategy.py: -------------------------------------------------------------------------------- 1 | from .road_geometry_calculator import RoadGeometryCalculator 2 | from .segmentation_strategy import SegmentationStrategy 3 | 4 | seg_lengths_dict = {"1": 50, "1.5": 30, "2": 10} 5 | 6 | 7 | class ParameterizedUniformStrategy(SegmentationStrategy): 8 | """ 9 | This class segments the road based on parameters like risk factor and 10 | fullroad length. 11 | """ 12 | 13 | def __init__(self, risk_factor: str, max_seg_length_to_full_road): 14 | """ 15 | 16 | :param risk_factor: Risk factor of the driving AI. 17 | :param max_seg_length_to_full_road: Relative size of the longest allowed segment compared to the full road length. 18 | """ 19 | # dependend on the risk factor 20 | self.__seg_length_in_meters = seg_lengths_dict[risk_factor] 21 | # dependend on the length of the full road 22 | self.__max_seg_length_in_meters = None 23 | 24 | self.__max_seg_length_to_full_road = max_seg_length_to_full_road 25 | 26 | self.__road_geometry_calculator = RoadGeometryCalculator() 27 | 28 | # TODO: formalize the clear segmentaion process 29 | def extract_segments(self, road_points): 30 | """ 31 | Returns a list of tuples that contain start end end indexes of road 32 | segments. 33 | 34 | :param road_points: List of coordinates specifying the road. 35 | :return: List of start and end indexes defining the start and end road points of segments. 36 | """ 37 | road_length = self.__road_geometry_calculator.get_road_length(road_points) 38 | 39 | self.__max_seg_length_in_meters = road_length * self.__max_seg_length_to_full_road 40 | 41 | if self.__seg_length_in_meters > self.__max_seg_length_in_meters: 42 | raise Exception("Road is too short for the required segment length.") 43 | 44 | # iterate over road points until we reach the desired segment length 45 | end = 0 46 | start = 0 47 | current_segment_length = 0 48 | segment_indexes = [] 49 | for i in range(1, len(road_points)): 50 | current_elementary_segment = road_points[i - 1 : i + 1] 51 | current_segment_length += self.__road_geometry_calculator.get_road_length(current_elementary_segment) 52 | 53 | if current_segment_length > self.__max_seg_length_in_meters: 54 | raise Exception("Max segment reached.") 55 | 56 | # segment reached its required length 57 | if (current_segment_length >= self.__seg_length_in_meters) or (i == len(road_points) - 1): 58 | current_segment_length = 0 59 | end = i 60 | segment_indexes.append((start, end)) 61 | start = i # set start index for the next segment 62 | 63 | return segment_indexes 64 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/test_case_crossover.py: -------------------------------------------------------------------------------- 1 | import random as rm 2 | 3 | import numpy as np 4 | from pymoo.model.crossover import Crossover 5 | 6 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.solution import Solution 7 | 8 | 9 | class TestCaseCrossover(Crossover): 10 | """ 11 | Module to perform the crossover 12 | """ 13 | 14 | def __init__(self, cross_rate): 15 | super().__init__(2, 2) 16 | self.cross_rate = cross_rate 17 | 18 | def _do(self, problem, X, **kwargs): 19 | # The input of has the following shape (n_parents, n_matings, n_var) 20 | _, n_matings, n_var = X.shape 21 | 22 | # The output owith the shape (n_offsprings, n_matings, n_var) 23 | # Because there the number of parents and offsprings are equal it keeps the shape of X 24 | Y = np.full_like(X, None, dtype=object) 25 | 26 | # for each mating provided 27 | 28 | for k in range(n_matings): 29 | r = np.random.random() 30 | 31 | s_a, s_b = X[0, k, 0], X[1, k, 0] 32 | 33 | # s_a.get_points() 34 | # s_a.remove_invalid_cases() 35 | # s_b.get_points() 36 | # s_b.remove_invalid_cases() 37 | 38 | if r < self.cross_rate: 39 | tc_a = s_a.states 40 | tc_b = s_b.states 41 | 42 | if len(tc_a) < len(tc_b): 43 | crossover_point = rm.randint(1, len(tc_a) - 1) 44 | elif len(tc_b) < len(tc_a): 45 | crossover_point = rm.randint(1, len(tc_b) - 1) 46 | else: 47 | crossover_point = rm.randint(1, len(tc_a) - 1) 48 | 49 | if s_a.n_states > 2 and s_b.n_states > 2: 50 | offa = {} 51 | offb = {} 52 | 53 | # one point crossover 54 | for i in range(0, crossover_point): 55 | offa["st" + str(i)] = tc_a["st" + str(i)] 56 | offb["st" + str(i)] = tc_b["st" + str(i)] 57 | for m in range(crossover_point, len(tc_b)): 58 | offa["st" + str(m)] = tc_b["st" + str(m)] 59 | for n in range(crossover_point, len(tc_a)): 60 | offb["st" + str(n)] = tc_a["st" + str(n)] 61 | 62 | off_a = Solution() 63 | off_b = Solution() 64 | 65 | off_a.states = offa 66 | off_b.states = offb 67 | 68 | else: 69 | print("Not enough states!") 70 | 71 | off_a.novelty = off_a.calc_novelty(tc_a, off_a.states) 72 | 73 | off_b.novelty = off_b.calc_novelty(tc_b, off_b.states) 74 | 75 | Y[0, k, 0], Y[1, k, 0] = off_a, off_b 76 | 77 | else: 78 | Y[0, k, 0], Y[1, k, 0] = s_a, s_b 79 | 80 | return Y 81 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/frenetic/src/generators/base_frenet_generator.py: -------------------------------------------------------------------------------- 1 | import logging as log 2 | 3 | import numpy as np 4 | 5 | import sdc_scissor.testing_api.test_generators.frenetic.src.utils.frenet as frenet 6 | from sdc_scissor.testing_api.test_generators.frenetic.src.generators.base_generator import BaseGenerator 7 | 8 | 9 | class BaseFrenetGenerator(BaseGenerator): 10 | def __init__(self, time_budget=None, executor=None, map_size=None, margin=10, strict_father=False): 11 | # Margin size w.r.t the map 12 | self.margin = margin 13 | self.recent_count = 0 14 | super().__init__(time_budget=time_budget, executor=executor, map_size=map_size, strict_father=strict_father) 15 | 16 | def kappas_to_road_points(self, kappas, frenet_step=10, theta0=1.57): 17 | """ 18 | Args: 19 | kappas: list of kappa values 20 | frenet_step: The distance between to points. 21 | theta0: The initial angle of the line. (1.57 == 90 degrees) 22 | Returns: 23 | road points in cartesian coordinates 24 | """ 25 | # Using the bottom center of the map. 26 | y0 = self.margin 27 | x0 = self.map_size / 2 28 | ss = np.arange(y0, (len(kappas) * frenet_step), frenet_step) 29 | 30 | # Transforming the frenet points to cartesian 31 | (xs, ys) = frenet.frenet_to_cartesian(x0, y0, theta0, ss, kappas) 32 | road_points = self.reframe_road(xs, ys) 33 | return road_points 34 | 35 | def execute_frenet_test(self, kappas, method="random", frenet_step=10, theta0=1.57, parent_info={}, extra_info={}): 36 | extra_info["kappas"] = kappas 37 | road_points = self.kappas_to_road_points(kappas, frenet_step=frenet_step, theta0=theta0) 38 | return road_points 39 | # if road_points: 40 | # self.recent_count += 1 41 | # return self.execute_test(road_points, method=method, parent_info=parent_info, extra_info=extra_info) 42 | # else: 43 | # return 'CANNOT_REFRAME', None 44 | 45 | def reframe_road(self, xs, ys): 46 | """ 47 | Args: 48 | xs: cartesian x coordinates 49 | ys: cartesian y coordinates 50 | Returns: 51 | A representation of the road that fits the map size (when possible). 52 | """ 53 | min_xs = min(xs) 54 | min_ys = min(ys) 55 | road_width = self.margin # TODO: How to get the exact road width? 56 | if (max(xs) - min_xs + road_width > self.map_size - self.margin) or ( 57 | max(ys) - min_ys + road_width > self.map_size - self.margin 58 | ): 59 | log.debug("Skip: Road won't fit") 60 | return None 61 | # TODO: Fail the entire test and start over 62 | xs = list(map(lambda x: x - min_xs + road_width, xs)) 63 | ys = list(map(lambda y: y - min_ys + road_width, ys)) 64 | return list(zip(xs, ys)) 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore model evaluations 2 | *result.txt 3 | 4 | # Ignore valid roads 5 | valid_roads/ 6 | 7 | # Ignore results 8 | results/ 9 | 10 | # Ignore simulation data 11 | simulations/ 12 | 13 | # Ignore Shapely client specific file for pip 14 | Shapely-* 15 | 16 | 17 | # Byte-compiled / optimized / DLL files 18 | __pycache__/ 19 | *.py[cod] 20 | *$py.class 21 | 22 | # C extensions 23 | *.so 24 | 25 | # Distribution / packaging 26 | .Python 27 | build/ 28 | develop-eggs/ 29 | dist/ 30 | downloads/ 31 | eggs/ 32 | .eggs/ 33 | lib/ 34 | lib64/ 35 | parts/ 36 | sdist/ 37 | var/ 38 | wheels/ 39 | pip-wheel-metadata/ 40 | share/python-wheels/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | MANIFEST 45 | 46 | # PyInstaller 47 | # Usually these files are written by a python script from a template 48 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 49 | *.manifest 50 | *.spec 51 | 52 | # Installer logs 53 | pip-log.txt 54 | pip-delete-this-directory.txt 55 | 56 | # Unit test / coverage reports 57 | htmlcov/ 58 | .tox/ 59 | .nox/ 60 | .coverage 61 | .coverage.* 62 | .cache 63 | nosetests.xml 64 | coverage.xml 65 | *.cover 66 | *.py,cover 67 | .hypothesis/ 68 | .pytest_cache/ 69 | 70 | # Translations 71 | *.mo 72 | *.pot 73 | 74 | # Django stuff: 75 | *.log 76 | local_settings.py 77 | db.sqlite3 78 | db.sqlite3-journal 79 | 80 | # Flask stuff: 81 | instance/ 82 | .webassets-cache 83 | 84 | # Scrapy stuff: 85 | .scrapy 86 | 87 | # Sphinx documentation 88 | docs/_build/ 89 | 90 | # PyBuilder 91 | target/ 92 | 93 | # Jupyter Notebook 94 | .ipynb_checkpoints 95 | 96 | # IPython 97 | profile_default/ 98 | ipython_config.py 99 | 100 | # pyenv 101 | .python-version 102 | 103 | # pipenv 104 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 105 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 106 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 107 | # install all needed dependencies. 108 | #Pipfile.lock 109 | 110 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 111 | __pypackages__/ 112 | 113 | # Celery stuff 114 | celerybeat-schedule 115 | celerybeat.pid 116 | 117 | # SageMath parsed files 118 | *.sage.py 119 | 120 | # Environments 121 | .env 122 | .venv 123 | env/ 124 | venv/ 125 | ENV/ 126 | env.bak/ 127 | venv.bak/ 128 | 129 | # Spyder project settings 130 | .spyderproject 131 | .spyproject 132 | 133 | # Rope project settings 134 | .ropeproject 135 | 136 | # mkdocs documentation 137 | /site 138 | 139 | # mypy 140 | .mypy_cache/ 141 | .dmypy.json 142 | dmypy.json 143 | 144 | # Pyre type checker 145 | .pyre/ 146 | 147 | # persited models 148 | *.joblib 149 | 150 | # PyCharm 151 | .idea/ 152 | 153 | /destination/ 154 | /analysis.ipynb 155 | /.vscode/ 156 | /trained_models/ 157 | /debug_scenario.py 158 | /docs/c4model/.structurizr/ 159 | -------------------------------------------------------------------------------- /tests/sdc_scissor/pynguin/test_sdc_scissor_obstacle_api_beamng_bump.py: -------------------------------------------------------------------------------- 1 | # Automatically generated by Pynguin. 2 | import pytest 3 | 4 | import sdc_scissor.obstacle_api.beamng_bump as module_0 5 | 6 | 7 | # @pytest.mark.skip() 8 | def test_case_0(): 9 | beamng_bump_0 = module_0.BeamngBump() 10 | assert ( 11 | f"{type(beamng_bump_0).__module__}.{type(beamng_bump_0).__qualname__}" 12 | == "sdc_scissor.obstacle_api.beamng_bump.BeamngBump" 13 | ) 14 | assert beamng_bump_0.x_pos is None 15 | assert beamng_bump_0.y_pos is None 16 | assert beamng_bump_0.z_pos is None 17 | assert beamng_bump_0.width == 6 18 | assert beamng_bump_0.length == 2 19 | assert beamng_bump_0.height == pytest.approx(0.2, abs=0.01, rel=0.01) 20 | assert beamng_bump_0.upper_length == 1 21 | assert beamng_bump_0.upper_width == 2 22 | assert beamng_bump_0.rot is None 23 | assert beamng_bump_0.rot_quat == (0, 0, 0, 1) 24 | assert beamng_bump_0.obstacle_type == "procedural" 25 | 26 | 27 | # @pytest.mark.skip() 28 | def test_case_1(): 29 | beamng_bump_0 = module_0.BeamngBump() 30 | assert ( 31 | f"{type(beamng_bump_0).__module__}.{type(beamng_bump_0).__qualname__}" 32 | == "sdc_scissor.obstacle_api.beamng_bump.BeamngBump" 33 | ) 34 | assert beamng_bump_0.x_pos is None 35 | assert beamng_bump_0.y_pos is None 36 | assert beamng_bump_0.z_pos is None 37 | assert beamng_bump_0.width == 6 38 | assert beamng_bump_0.length == 2 39 | assert beamng_bump_0.height == pytest.approx(0.2, abs=0.01, rel=0.01) 40 | assert beamng_bump_0.upper_length == 1 41 | assert beamng_bump_0.upper_width == 2 42 | assert beamng_bump_0.rot is None 43 | assert beamng_bump_0.rot_quat == (0, 0, 0, 1) 44 | assert beamng_bump_0.obstacle_type == "procedural" 45 | beamng_bump_1 = module_0.BeamngBump() 46 | assert beamng_bump_1.x_pos is None 47 | assert beamng_bump_1.y_pos is None 48 | assert beamng_bump_1.z_pos is None 49 | assert beamng_bump_1.width == 6 50 | assert beamng_bump_1.length == 2 51 | assert beamng_bump_1.height == pytest.approx(0.2, abs=0.01, rel=0.01) 52 | assert beamng_bump_1.upper_length == 1 53 | assert beamng_bump_1.upper_width == 2 54 | assert beamng_bump_1.rot is None 55 | assert beamng_bump_1.rot_quat == (0, 0, 0, 1) 56 | assert beamng_bump_1.obstacle_type == "procedural" 57 | var_0 = beamng_bump_0.get() 58 | assert f"{type(var_0).__module__}.{type(var_0).__qualname__}" == "beamngpy.scenario.ProceduralBump" 59 | assert var_0.id == "pybump" 60 | assert var_0.name == "pybump" 61 | assert var_0.type == "ProceduralMesh" 62 | assert var_0.pos == (None, None, None) 63 | assert var_0.rot == (0, 0, 0, 1) 64 | assert var_0.scale == (1, 1, 1) 65 | assert var_0.opts == {} 66 | assert var_0.children == [] 67 | assert var_0.material is None 68 | assert var_0.width == 6 69 | assert var_0.length == 2 70 | assert var_0.height == pytest.approx(0.2, abs=0.01, rel=0.01) 71 | assert var_0.upper_length == 1 72 | assert var_0.upper_width == 2 73 | var_1 = beamng_bump_1.get() 74 | assert var_1.id == "pybump" 75 | assert var_1.name == "pybump" 76 | -------------------------------------------------------------------------------- /docs/software_guidebook/decisions/adr-template.md: -------------------------------------------------------------------------------- 1 | # {short title of solved problem and solution} 2 | * status: {proposed | rejected | accepted | deprecated | … | superseded by [ADR-0000](0000-use-markdown-any-decision-records.md)} 3 | * date: {YYYY-MM-DD when the decision was last updated} 4 | * deciders: {list everyone involved in the decision} 5 | * consulted: {list everyone whose opinions are sought (typically subject-matter experts); and with whom there is a two-way communication} 6 | * informed: {list everyone who is kept up-to-date on progress; and with whom there is a one-way communication} 7 | 8 | ## Context and Problem Statement 9 | 10 | {Describe the context and problem statement, e.g., in free form using two to three sentences or in the form of an illustrative story. 11 | You may want to articulate the problem in form of a question and add links to collaboration boards or issue management systems.} 12 | 13 | 14 | ## Decision Drivers 15 | 16 | * {decision driver 1, e.g., a force, facing concern, …} 17 | * {decision driver 2, e.g., a force, facing concern, …} 18 | * … 19 | 20 | ## Considered Options 21 | 22 | * {title of option 1} 23 | * {title of option 2} 24 | * {title of option 3} 25 | * … 26 | 27 | ## Decision Outcome 28 | 29 | Chosen option: "{title of option 1}", because 30 | {justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force {force} | … | comes out best (see below)}. 31 | 32 | 33 | ### Positive Consequences 34 | 35 | * {e.g., improvement of one or more desired qualities, …} 36 | * … 37 | 38 | 39 | ### Negative Consequences 40 | 41 | * {e.g., compromising one or more desired qualities, …} 42 | * … 43 | 44 | 45 | ## Validation 46 | 47 | {describe how the implementation of/compliance with the ADR is validated. E.g., by a review or an ArchUnit test} 48 | 49 | 50 | ## Pros and Cons of the Options 51 | 52 | ### {title of option 1} 53 | 54 | 55 | {example | description | pointer to more information | …} 56 | 57 | * Good, because {argument a} 58 | * Good, because {argument b} 59 | 60 | * Neutral, because {argument c} 61 | * Bad, because {argument d} 62 | * … 63 | 64 | ### {title of other option} 65 | 66 | {example | description | pointer to more information | …} 67 | 68 | * Good, because {argument a} 69 | * Good, because {argument b} 70 | * Neutral, because {argument c} 71 | * Bad, because {argument d} 72 | * … 73 | 74 | 75 | ## More Information 76 | 77 | {You might want to provide additional evidence/confidence for the decision outcome here and/or 78 | document the team agreement on the decision and/or 79 | define when this decision when and how the decision should be realized and if/when it should be re-visited and/or 80 | how the decision is validated. 81 | Links to other decisions and resources might here appear as well.} 82 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/sky_and_sun/items.level.json: -------------------------------------------------------------------------------- 1 | {"name":"tod","class":"TimeOfDay","persistentId":"49a3be50-5147-474c-bb92-22b37cb07e18","__parent":"sky_and_sun","position":[36.62279891967773,-62.50979995727539,15.32040023803711],"axisTilt":10,"azimuthOverride":0,"play":false,"startTime":0.1000000014901161,"time":0.1000000014901161} 2 | {"name":"theLevelInfo","class":"LevelInfo","persistentId":"9db07344-2f59-4f04-8117-07d04b08e71e","__parent":"sky_and_sun","Enabled":"1","canvasClearColor":[1,1,1,255],"fogAtmosphereHeight":800,"fogColor":[0.7411760091781616,0.8156859874725342,0.9254900217056274,1],"fogDensity":0.0006000000284984708,"globalEnviromentMap":"BNG_Sky_02_cubemap","gravity":-9.810000419616699,"soundAmbience":"AudioAmbienceDefault","visibleDistance":4500} 3 | {"name":"sunsky","class":"ScatterSky","persistentId":"7d6eb1cf-4a73-4780-a87f-9fcacc227ced","__parent":"sky_and_sun","position":[70.39710235595703,4.539209842681885,73.99230194091797],"ambientScale":[0.5450980067253113,0.5450980067253113,0.549019992351532,1],"azimuth":286.6984558105469,"colorize":[0.4235289990901947,0.6078429818153381,0.8666669726371765,1],"colorizeAmount":2,"elevation":52.818603515625,"exposure":15,"fadeStartDistance":1000,"flareScale":5,"flareType":"BNG_Sunflare_2","fogScale":[0.6941180229187012,0.8549020290374756,0.992156982421875,1],"lastSplitTerrainOnly":true,"logWeight":0.9900000095367432,"mieScattering":0.000607529713306576,"moonLightColor":[0.1960780024528503,0.1960780024528503,0.1960780024528503,1],"moonMat":"Moon_Glow_Mat","moonScale":0.02999999932944775,"nightColor":[0.02352939918637276,0.02352939918637276,0.02352939918637276,1],"nightCubemap":"BNG_Sky_Space","nightFogColor":[0.003921569790691137,0.003921569790691137,0.003921569790691137,1],"overDarkFactor":[40000,8000,1800,650],"rayleighScattering":0.009999999776482582,"shadowDarkenColor":[0,0,0,0],"shadowDistance":1600,"shadowSoftness":0.2000000029802322,"skyBrightness":11,"sunScale":[0.9960780143737793,0.9019610285758972,0.8313729763031006,1],"texSize":1024,"useNightCubemap":true} 4 | {"name":"clouds","class":"CloudLayer","persistentId":"ba1f1451-1006-47f8-a30a-658946b01714","__parent":"sky_and_sun","position":[547.2169799804688,-452.9719848632813,609.7449951171875],"Textures":[{"texScale":1.5,"texSpeed":0.002000000094994903},{"texDirection":[0.800000011920929,0.2000000029802322],"texScale":3,"texSpeed":0.02500000037252903},{"texDirection":[0.2000000029802322,0.5],"texScale":4,"texSpeed":0.03500000014901161}],"baseColor":[0.9960780143737793,0.9960780143737793,0.9960780143737793,0.9960780143737793],"coverage":0.07999999821186066,"exposure":1.299999952316284,"height":3,"texture":"levels/jungle_rock_island/art/skies/SkyNormals_05.dds","windSpeed":0.2000000029802322} 5 | {"name":"clouds1","class":"CloudLayer","persistentId":"541e37ca-096c-40e2-aac3-3cd898df7419","__parent":"sky_and_sun","position":[-1278.56005859375,-1211.329956054688,-186.7310028076172],"Textures":[{"texSpeed":0.002000000094994903},{"texDirection":[0.800000011920929,0.2000000029802322],"texScale":2,"texSpeed":0.02500000037252903},{"texDirection":[0.2000000029802322,0.5],"texScale":0.5,"texSpeed":0.03500000014901161}],"baseColor":[0.9960780143737793,0.9960780143737793,0.9960780143737793,0.9960780143737793],"cloneOrigin":"clouds","coverage":0.2000000029802322,"exposure":1.299999952316284,"height":7,"texture":"levels/jungle_rock_island/art/skies/SkyNormals_05.dds","windSpeed":0.2000000029802322} 6 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_loader.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import re 5 | from pathlib import Path 6 | 7 | from sdc_scissor.testing_api.test import Test 8 | from sdc_scissor.testing_api.test_validator import TestValidator 9 | 10 | 11 | class TestLoader: 12 | def __init__(self, tests_dir: Path, test_validator: TestValidator): 13 | """ 14 | 15 | :param tests_dir: 16 | """ 17 | self.tests_dir: Path = tests_dir 18 | self.test_validator: TestValidator = test_validator 19 | self.test_paths: list[Path] = [] 20 | self.__set_test_paths(self.test_paths) 21 | 22 | def __set_test_paths(self, tests_paths: list): 23 | """ 24 | 25 | :param tests_paths: 26 | """ 27 | logging.debug("* _test_loader_gen") 28 | pattern: str = r".*test.*.json" 29 | for root, dirs, files in os.walk(self.tests_dir): 30 | if "unlabeled" in str(root): 31 | continue 32 | for file in files: 33 | if re.fullmatch(pattern, file): 34 | full_path = Path(root) / file 35 | with open(full_path) as fp: 36 | test_json: dict = json.load(fp) 37 | is_valid = test_json.get("is_valid", True) 38 | if is_valid: 39 | tests_paths.append(full_path) 40 | 41 | def has_next(self) -> bool: 42 | """ 43 | 44 | :return: 45 | """ 46 | return len(self.test_paths) > 0 47 | 48 | def next(self) -> tuple[Test, Path]: 49 | """ 50 | 51 | :return: 52 | """ 53 | if not self.has_next(): 54 | logging.warning("There are no remaining tests!") 55 | raise Exception("There are no remaining tests!") 56 | 57 | test_path: Path = self.test_paths.pop() 58 | test: Test = self.__load_test_from_path(test_path) 59 | return test, test_path 60 | 61 | def __load_test_from_path(self, test_path: Path): 62 | """ 63 | 64 | :param test_path: 65 | :return: 66 | """ 67 | logging.debug(str(test_path)) 68 | with open(test_path, "r") as fp: 69 | test_json: dict = json.load(fp) 70 | 71 | road_points = test_json.get("interpolated_road_points", None) 72 | if not road_points: 73 | road_points = test_json.get("interpolated_points", None) 74 | if not road_points: 75 | road_points = test_json.get("road_points", None) 76 | if road_points is None: 77 | raise Exception("No road points") 78 | test_outcome = test_json.get("test_outcome", None) 79 | sim_time = test_json.get("test_duration", None) 80 | if not sim_time: 81 | sim_time = test_json.get("simulation_time", None) 82 | 83 | id_pattern = r"(.*test.*)" 84 | logging.debug("test_path: {}".format(str(test_path))) 85 | match_obj = re.match(pattern=id_pattern, string=str(test_path)) 86 | test_id = match_obj.group(1) 87 | logging.debug("test_id: {}".format(test_id)) 88 | logging.debug("road_points: {}".format(road_points)) 89 | 90 | test = Test(test_id=test_id, road_points=road_points, test_outcome=test_outcome, test_duration=sim_time) 91 | self.test_validator.validate(test) 92 | 93 | return test 94 | -------------------------------------------------------------------------------- /tests/sdc_scissor/testing_api/test_test_monitor.py: -------------------------------------------------------------------------------- 1 | from sdc_scissor.config import CONFIG 2 | from sdc_scissor.testing_api.test_monitor import TestMonitor 3 | 4 | 5 | class TestTestMonitor: 6 | def setup_class(self): 7 | CONFIG.config = {"canbus": False, "can_dbc": "", "can_dbc_map": "", "can_interface": "", "can_bitrate": ""} 8 | 9 | def teardown_class(self): 10 | pass 11 | 12 | def setup_method(self): 13 | # can_bus_handler = CanBusHandler(NoCANBusOutput()) 14 | self.test_monitor = TestMonitor(simulator=None, oob=None, can_bus_handler=None) 15 | self.test_monitor.start_timer() 16 | 17 | def test_is_car_moving_car_stays_on_same_position(self, mocker): 18 | self.test_monitor.test = mocker.patch("sdc_scissor.testing_api.test.Test") 19 | simulation_data_entry_t0 = {"time": 0, "position": (0, 0, 0)} 20 | simulation_data_entry_t1 = {"time": 11, "position": (0, 0, 0)} 21 | self.test_monitor.test.simulation_data = [simulation_data_entry_t0, simulation_data_entry_t1] 22 | expected = False 23 | actual = self.test_monitor.is_car_moving() 24 | assert expected == actual 25 | 26 | def test_is_car_moving_car_stays_not_on_same_position(self, mocker): 27 | self.test_monitor.test = mocker.patch("sdc_scissor.testing_api.test.Test") 28 | simulation_data_entry_t0 = {"time": 0, "position": (0, 0, 0)} 29 | simulation_data_entry_t1 = {"time": 5, "position": (10, 10, 0)} 30 | self.test_monitor.test.simulation_data = [simulation_data_entry_t0, simulation_data_entry_t1] 31 | expected = True 32 | actual = self.test_monitor.is_car_moving() 33 | assert expected == actual 34 | 35 | def test_is_car_moving_not_enough_sim_data_yet_to_make_decision(self, mocker): 36 | self.test_monitor.test = mocker.patch("sdc_scissor.testing_api.test.Test") 37 | simulation_data_entry_t0 = {"time": 0, "position": (0, 0, 0)} 38 | simulation_data_entry_t1 = {"time": 0.1, "position": (0, 0, 0)} 39 | self.test_monitor.test.simulation_data = [simulation_data_entry_t0, simulation_data_entry_t1] 40 | expected = True 41 | actual = self.test_monitor.is_car_moving() 42 | assert expected == actual 43 | 44 | def test_check_verify_if_test_is_finished(self, mocker): 45 | mock_simulator = mocker.patch("sdc_scissor.simulator_api.abstract_simulator.AbstractSimulator") 46 | mock_simulator.update_car.return_value = None 47 | mock_simulator.get_car_position.return_value = 0, 0, 0 48 | 49 | class _TestSensorData: 50 | def __init__(self): 51 | self._data = 1 52 | 53 | test_sensor_data = _TestSensorData() 54 | mock_simulator.get_sensor_data.return_value = test_sensor_data 55 | self.test_monitor.simulator = mock_simulator 56 | 57 | mock_test = mocker.patch("sdc_scissor.testing_api.test.Test") 58 | mock_test.interpolated_road_points = [[0, 0, 0], [1, 1, 0]] 59 | self.test_monitor.test = mock_test 60 | 61 | mock_can_bus_handler = mocker.patch("sdc_scissor.can_api.can_bus_handler.CanBusHandler") 62 | mock_can_bus_handler.transmit_sensor_data_to_can_bus.return_value = None 63 | self.test_monitor.cbh = mock_can_bus_handler 64 | 65 | self.test_monitor.process_car_state(interrupt_on_failure=True) 66 | expected = True 67 | actual = self.test_monitor.is_test_finished 68 | assert expected == actual 69 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_validator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | from shapely.geometry import LineString, MultiLineString 4 | 5 | from sdc_scissor.feature_extraction_api.road_geometry_calculator import RoadGeometryCalculator 6 | from sdc_scissor.testing_api.test import Test 7 | 8 | 9 | class TestIsNotValidException(Exception): 10 | pass 11 | 12 | 13 | class TestValidator(abc.ABC): 14 | @abc.abstractmethod 15 | def validate(self, test: Test) -> bool: 16 | pass 17 | 18 | 19 | class SimpleTestValidator(TestValidator): 20 | def validate(self, test: Test) -> bool: 21 | test.is_valid = True 22 | return True 23 | 24 | 25 | class TestValidatorDecorator(TestValidator): 26 | def __init__(self, wrappee: TestValidator): 27 | self.wrappee: TestValidator = wrappee 28 | 29 | def validate(self, test: Test) -> bool: 30 | return self.wrappee.validate(test) 31 | 32 | 33 | class MakeTestInvalidValidator(TestValidatorDecorator): 34 | def __init__(self, wrappee: TestValidator): 35 | super().__init__(wrappee) 36 | 37 | def validate(self, test: Test) -> bool: 38 | self.wrappee.validate(test) 39 | test.is_valid = False 40 | return False 41 | 42 | 43 | class NoIntersectionValidator(TestValidatorDecorator): 44 | def __init__(self, wrappee: TestValidator): 45 | super().__init__(wrappee) 46 | 47 | def validate(self, test: Test) -> bool: 48 | self.wrappee.validate(test) 49 | if not test.is_valid: 50 | return False 51 | 52 | road_points_line_string: LineString = LineString( 53 | coordinates=[(node[0], node[1]) for node in test.interpolated_road_points] 54 | ) 55 | left_bound_line_string = road_points_line_string.parallel_offset(distance=5, side="left") 56 | right_bound_line_string = road_points_line_string.parallel_offset(distance=5, side="right") 57 | 58 | if left_bound_line_string.geom_type != "LineString" or right_bound_line_string.geom_type != "LineString": 59 | test.is_valid = False 60 | return test.is_valid 61 | 62 | road_lines: MultiLineString = MultiLineString( 63 | (left_bound_line_string, road_points_line_string, right_bound_line_string) 64 | ) 65 | 66 | test.is_valid = True if road_lines.is_simple else False 67 | return test.is_valid 68 | 69 | 70 | class NoTooSharpTurnsValidator(TestValidatorDecorator): 71 | def __init__(self, wrappee, angle_threshold=1.0): 72 | super().__init__(wrappee) 73 | self.__road_geometry_calculator = RoadGeometryCalculator() 74 | self.angle_threshold = angle_threshold 75 | 76 | def validate(self, test: Test) -> bool: 77 | self.wrappee.validate(test) 78 | 79 | p0 = test.interpolated_road_points[0][0], test.interpolated_road_points[0][1] 80 | p1 = test.interpolated_road_points[1][0], test.interpolated_road_points[1][1] 81 | previous_direction = self.__road_geometry_calculator.get_direction(p0, p1) 82 | previous_point = p1 83 | 84 | for current_point in test.interpolated_road_points[2:]: 85 | current_direction = self.__road_geometry_calculator.get_direction(previous_point, current_point) 86 | angle = self.__road_geometry_calculator.get_angle(previous_direction, current_direction) 87 | if angle > self.angle_threshold: 88 | return False 89 | previous_point, previous_direction = current_point, current_direction 90 | 91 | return True 92 | -------------------------------------------------------------------------------- /docs/software_guidebook/functional_overview.md: -------------------------------------------------------------------------------- 1 | # Functional Overview 2 | SDC-Scissor provides a variety of functionalities that users can leverage for their purposes. The main high-level 3 | functionalities are described in this section. If you have questions about more the low-level functionalities then you 4 | can also ask the community on 5 | [Slack](https://join.slack.com/t/sdc-scissor/shared_invite/zt-1aikrj1uu-Dz0a9BE1AQ4GQp4A3Bm5og) or 6 | [Discord](https://discord.gg/AaYnBS6s7E). 7 | 8 | ## Virtual Environments 9 | SDC-Scissor is capabable in running the same test in different simulators. The architecture allows the use of identical 10 | interfaces for different external simulators, e.g., [BeamNG.tech](https://beamng.tech/) and [CARLA](https://carla.org/). 11 | 12 | 13 | ## State-of-the-Art Test Generators 14 | The use of the recent test generators from the [SBST Workshop](https://sbst22.github.io/) allows the user of SDC-Scissor 15 | to generate diverse and meaningful tests automatically without ccreating them manually. The SBST workschop is an annual 16 | event co-located with the 17 | [ACM/IEEE International Conference on Software Engineering (ICSE)](https://conf.researchr.org/home/icse-2022), the top 18 | conference in the field of academic software engineering. 19 | 20 | ## Regression Testing 21 | The goal of regression testing is to detect faults in a software system as early as possible in the testing process 22 | after a change was pushed to the repository. According to [1], regression testing consists of three components: (i) selection, 23 | (ii) prioritization, and (iii) minimization. 24 | 25 | ### Test Selection 26 | SDC-Scissor predicts the outcome of tests before executing them. This allows us in the context of regression testing to 27 | select only those tests that are predicted to fail. By disregarding the tests that will likely fail we can speed up the 28 | testing process as shown in [2]. 29 | 30 | ### Test Prioritization 31 | Detecting faults early in the testing process is achieved by applying a single-objective and a multi-objective genetic 32 | algorithms as described in [3]. The goal is to find a permutation that runs first the most diverse and less costly 33 | tests. 34 | 35 | ### Test Minimization *(not implemented)* 36 | SDC-Scissor is able to shrink a test case only to the relevant part. The relevant part is the cause of a failure. The 37 | goal of minimizing a test scenario is to save time in simulation by omitting the parts of a scenario that are seen as 38 | safe for the car. 39 | 40 | ## SDC-Scissor Library 41 | Testers of SDCs can use SDC-Scissor as a library. It supports the building process of building different testing 42 | pipelines according to the user's needs. 43 | 44 | ## References 45 | * [1] S. Yoo, M. Harman, "Regression testing minimization, selection and prioritization: a survey", 46 | *Software: Testing, Verification and Reliability*, vol. 22, no. 2, pp. 67-120, Mar. 2012, doi: 10.1002/stvr.430 47 | * [2] C. Birchler, N. Ganz, S. Khatiri, A. Gambi, S. Panichella, 48 | "Cost-effective Simulation-based Test Selection in Self-driving Cars Software with SDC-Scissor", 49 | *2022 IEEE International Conference on Software Analysis, Evolution and Reengineering (SANER)*, 50 | Honolulu, HI, USA (online), Mar. 15-18 2022, pp. 164-168, doi: 10.1109/SANER53432.2022.00030 51 | * [3] C. Birchler, S. Khatiri, P. Derakhshanfar, S. Panichella, A. Panichella, 52 | "Single and Multi-objective Test Cases Prioritization for Self-driving Cars in Virtual Environments", 53 | *ACM Transactions on Software Engineering and Methodology (TOSEM)*, Just Accepted, doi: 10.1145/3533818 54 | -------------------------------------------------------------------------------- /sdc_scissor/feature_extraction_api/road_geometry_calculator.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | 4 | class RoadGeometryCalculator: 5 | def extract_turn_angles(self, road_points): 6 | """ 7 | Extract angles of raod points and ad them to the instance variable. 8 | 9 | :param road_points: Points that define the road in the test scenario. 10 | :return: Angles in degrees of road's turns defined by the road points. 11 | """ 12 | angles = [] 13 | # iterate over "all" road points 14 | for i in range(2, len(road_points)): 15 | # calculate angle between previous direction and vector from 16 | # previous point to the current one 17 | 18 | point_before = road_points[i - 2] 19 | mid_point = road_points[i - 1] 20 | point_after = road_points[i] 21 | 22 | prev_direction = self.get_direction(point_before, mid_point) 23 | current_direction = self.get_direction(mid_point, point_after) 24 | 25 | turn_angle = self.get_angle(prev_direction, current_direction) 26 | angles.append(turn_angle) 27 | 28 | return angles 29 | 30 | @staticmethod 31 | def get_angle(first_vec, second_vec): 32 | """ 33 | Returns the angle in degrees between the first and second vector. 34 | A left turn as positive angles whereas right turns have negatives. 35 | 36 | :param first_vec: First 2D vector 37 | :param second_vec: Second 3D vector 38 | :return: Angle between the vectors in degrees 39 | """ 40 | a1, a2 = first_vec[0], first_vec[1] 41 | b1, b2 = second_vec[0], second_vec[1] 42 | 43 | angle_in_radians = math.atan2(b2, b1) - math.atan2(a2, a1) 44 | angle_in_degrees = math.degrees(angle_in_radians) 45 | 46 | return angle_in_degrees 47 | 48 | @staticmethod 49 | def get_distance_between(first_point, second_point) -> float: 50 | """ 51 | Calculate Euclidean distance between two points. 52 | 53 | :param first_point: (x,y) coordinates of first point 54 | :param second_point: (x,y) coordinates of second point 55 | :return: Euclidean distance between the points 56 | """ 57 | a1, a2 = first_point[0], first_point[1] 58 | b1, b2 = second_point[0], second_point[1] 59 | c1, c2 = (b1 - a1, b2 - a2) 60 | distance = math.sqrt(c1**2 + c2**2) 61 | return distance 62 | 63 | @staticmethod 64 | def get_direction(first_point, second_point): 65 | """ 66 | Get the direction vector from the first point to the second point. 67 | 68 | :param first_point: (x,y) coordinates of first point 69 | :param second_point: (x,y) coordinates of second point 70 | :return: Return the difference 2D vector (second_point-first_point) 71 | """ 72 | return second_point[0] - first_point[0], second_point[1] - first_point[1] 73 | 74 | @staticmethod 75 | def get_road_length(road_points): 76 | """ 77 | Compute the length of the road. 78 | 79 | :param road_points: List of road points defining the road 80 | :return: Length of the road 81 | """ 82 | nr_of_road_points = len(road_points) 83 | 84 | road_length = 0 85 | for i in range(1, nr_of_road_points): 86 | a1, a2 = road_points[i - 1][0], road_points[i - 1][1] 87 | b1, b2 = road_points[i][0], road_points[i][1] 88 | 89 | c1, c2 = (b1 - a1, b2 - a2) 90 | 91 | road_length += math.sqrt(c1**2 + c2**2) 92 | 93 | return road_length 94 | -------------------------------------------------------------------------------- /levels_template/tig/main/MissionGroup/generated/items.level.json-readme.txt: -------------------------------------------------------------------------------- 1 | This is an example of file that describe the customized objects into the 3d level 2 | 3 | {"name": "street_1", "class": "DecalRoad", "breakAngle": 180, "distanceFade": [1000, 1000], "drivability": 1, "material": "tig_road_rubber_sticky", "overObjects": true, "persistentId": "d62cdb76-3384-4620-9d3d-134b21968f32", "__parent": "generated", "position": [0, 0.0, -28, 8], "textureLength": 2.5, "nodes": [[0, 0.0, -28, 8], [5, 4.207354924039483, -28, 8], [10, 4.546487134128409, -28, 8], [15, 0.7056000402993361, -28, 8], [20, -3.7840124765396412, -28, 8], [25, -4.794621373315692, -28, 8], [30, -1.3970774909946293, -28, 8], [35, 3.2849329935939453, -28, 8], [40, 4.946791233116909, -28, 8], [45, 2.060592426208783, -28, 8], [50, -2.7201055544468487, -28, 8], [55, -4.9999510327535175, -28, 8], [60, -2.6828645900021746, -28, 8], [65, 2.1008351841332047, -28, 8], [70, 4.953036778474352, -28, 8], [75, 3.251439200785584, -28, 8], [80, -1.4395165833253265, -28, 8], [85, -4.806987459397784, -28, 8], [90, -3.7549362338583805, -28, 8], [95, 0.7493860483147617, -28, 8], [100, 4.564726253638138, -28, 8], [105, 4.18327819268028, -28, 8], [110, -0.04425654645201938, -28, 8], [115, -4.231102020875853, -28, 8], [120, -4.527891810033119, -28, 8], [125, -0.6617587504888651, -28, 8], [130, 3.8127922523980136, -28, 8], [135, 4.781879642022515, -28, 8], [140, 1.3545289415393453, -28, 8], [145, -3.318169421064838, -28, 8], [150, -4.940158120464309, -28, 8], [155, -2.0201882266153253, -28, 8], [160, 2.757133406208453, -28, 8], [165, 4.999559300536336, -28, 8], [170, 2.645413430600119, -28, 8], [175, -2.140913347480755, -28, 8], [180, -4.9588942672155785, -28, 8], [185, -3.2176906667849976, -28, 8], [190, 1.4818428935469266, -28, 8], [195, 4.818976931420439, -28, 8], [200, 3.725565802396744, -28, 8], [205, -0.7931133440235449, -28, 8], [210, -4.582607739578169, -28, 8], [215, -4.158873713142992, -28, 8], [220, 0.08850962552706788, -28, 8], [225, 4.254517622670592, -28, 8], [230, 4.508941738244046, -28, 8], [235, 0.61786561372612, -28, 8], [240, -3.841273306618334, -28, 8], [245, -4.768763263797359, -28, 8], [250, -1.3118742685196438, -28, 8], [255, 3.3511458792168733, -28, 8], [260, 4.933137960202426, -28, 8], [265, 1.9796257509091708, -28, 8], [270, -2.793945244258081, -28, 8], [275, -4.9987758667931, -28, 8], [280, -2.607755010434559, -28, 8], [285, 2.1808237762391247, -28, 8], [290, 4.964363240422686, -28, 8], [295, 3.1836900356956894, -28, 8], [300, -1.5240531055110833, -28, 8], [305, -4.8305888500419645, -28, 8], [310, -3.695903483246114, -28, 8], [315, 0.8367785015140345, -28, 8], [320, 4.600130190983953, -28, 8], [325, 4.134143397450517, -28, 8], [330, -0.13275577011983397, -28, 8], [335, -4.277599894876611, -28, 8], [340, -4.489638403446456, -28, 8], [345, -0.5739240689159362, -28, 8], [350, 3.8694534077894454, -28, 8], [355, 4.755273266271873, -28, 8], [360, 1.2691168138101814, -28, 8], [365, -3.383859784436538, -28, 8], [370, -4.925731302341237, -28, 8], [375, -1.9389081770471521, -28, 8], [380, 2.8305381844909014, -28, 8], [385, 4.997600792903657, -28, 8], [390, 2.569892279937676, -28, 8], [395, -2.220563343537542, -28, 8], [400, -4.969443269616876, -28, 8], [405, -3.1494399713722694, -28, 8], [410, 1.5661439121654257, -28, 8], [415, 4.841822305500926, -28, 8], [420, 3.665951600366461, -28, 8], [425, -0.8803780997429355, -28, 8], [430, -4.617292235020299, -28, 8], [435, -4.109089183154112, -28, 8], [440, 0.1769915136683034, -28, 8], [445, 4.300347029062267, -28, 8], [450, 4.4699833180027895, -28, 8], [455, 0.5299375587557843, -28, 8], [460, -3.8973303480790236, -28, 8], [465, -4.741410706349736, -28, 8], [470, -1.2262599273382717, -28, 8], [475, 3.4163085736806047, -28, 8], [480, 4.917938727171725, -28, 8], [485, 1.8980386951376085, -28, 8], [490, -2.8669093599521145, -28, 8], [495, -4.996034170931768, -28, 8]]} 4 | -------------------------------------------------------------------------------- /docs/user_documentation/quick_start.md: -------------------------------------------------------------------------------- 1 | # Quick Start 2 | ## Requirements 3 | * Windows 10 4 | * BeamNG.tech v0.24.0.2 5 | * Python 3.9 6 | 7 | ````{note} 8 | SDC-Scissor work currently only with BeamNG.tech v0.24.0.2! 9 | ```` 10 | 11 | You can install SDC-Scissor from PyPI or manually by downloading it from GitHub and install it with Poetry. 12 | ## PyPI 13 | ````shell 14 | pip install sdc-scissor 15 | sdc-scissor -c your_config.yml 16 | ```` 17 | 18 | ## Manual installation 19 | Clone the repository: 20 | ````shell 21 | git clone https://github.com/ChristianBirchler/sdc-scissor.git 22 | ```` 23 | 24 | Install the dependencies: 25 | ````shell 26 | cd sdc-scissor 27 | pip install poetry 28 | poetry install 29 | ```` 30 | 31 | ## Install BeamNG.tech 32 | For using SDC-Scissor with the BeamNG.tech simulator you can obtain an academic license from BeamNG from their 33 | [website](https://register.beamng.tech/). After installing BeamNG.tech you need to replace the `tig` level it in the 34 | user directory of BeamNG.tech (`C:\Users\myaccount\Documents\BeamNG.drive\0.24\levels`) wth the one from this repository 35 | (`levels_template/tig`). 36 | 37 | ## TEASER 38 | You can run SDC-Scissor with a CAN bus feature called TEASER. Look [here](test_execution.md) for more details. 39 | 40 | ## Run the tool 41 | 42 | A sample pipeline can be found with the following DOI: [10.24433/CO.1187310.v2](https://doi.org/10.24433/CO.1187310.v2) 43 | 44 | ### Sequence Diagram 45 | ```{mermaid} 46 | sequenceDiagram 47 | actor User 48 | participant SDC-Scissor 49 | participant BeamNG.tech 50 | User ->> SDC-Scissor: Generate tests 51 | activate SDC-Scissor 52 | SDC-Scissor -->> User: Tests stored 53 | deactivate SDC-Scissor 54 | User ->> SDC-Scissor: Label tests 55 | activate BeamNG.tech 56 | loop for test in test suite 57 | SDC-Scissor ->> SDC-Scissor: Run test 58 | SDC-Scissor ->> BeamNG.tech: Run test in simulator 59 | end 60 | deactivate BeamNG.tech 61 | SDC-Scissor -->> User: Tests with outcome stored 62 | User ->> SDC-Scissor: Extract features 63 | SDC-Scissor -->> User: CSV file with features stored 64 | User ->> SDC-Scissor: Evaluate models 65 | SDC-Scissor -->> User: Trained models stored 66 | User ->> SDC-Scissor: Predict test outcome 67 | SDC-Scissor -->> User: Predicted test outcomes stored 68 | User ->> SDC-Scissor: Run only selected tests 69 | activate BeamNG.tech 70 | loop for test in selected tests 71 | SDC-Scissor ->> SDC-Scissor: Run test 72 | SDC-Scissor ->> BeamNG.tech: Run test in simulator 73 | end 74 | deactivate BeamNG.tech 75 | SDC-Scissor -->> User: Test outcomes stored 76 | ``` 77 | 78 | ````shell 79 | poetry run sdc-scissor -c sample_configs/label-tests.yml 80 | ```` 81 | 82 | The [Software Guidebook](../software_guidebook/introduction.md) provides more explanations about the 83 | [installation](../software_guidebook/deployment.md) and [usage](../software_guidebook/operation_and_support.md) of 84 | SDC-Scissor. 85 | 86 | ## Demo 87 | ```{eval-rst} 88 | .. youtube:: -PYwf4b_Oi4 89 | :width: 695 90 | :height: 480 91 | ``` 92 | 93 | ````{note} 94 | The following demo was made for SDC-Scissor v1.0. A demo video for the recent version is coming soon. 95 | ```` 96 | ```{eval-rst} 97 | .. youtube:: Cn8p648KnfQ 98 | :width: 695 99 | :height: 480 100 | ``` 101 | 102 | ## Data 103 | In the repository is a directory `sample_tests` with some simple sample test cases. These tests are mainly used for 104 | development purposes. 105 | 106 | The data used for the demo and evaluation of SDC-Scissor v1.0 we made available on Zenodo: 107 | [![Zenodo](https://zenodo.org/badge/DOI/10.5281/zenodo.5903161.svg)](https://doi.org/10.5281/zenodo.5903161) 108 | ````{note} 109 | The used simulator was BeamNG.research which is deprecated and not compatible anymore with SDC-Scissor. 110 | ```` 111 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/ambiegen_generator.py: -------------------------------------------------------------------------------- 1 | # from code_pipeline.tests_generation import RoadTestFactory 2 | import logging as log 3 | import time 4 | 5 | from pymoo.algorithms.nsga2 import NSGA2 6 | from pymoo.configuration import Configuration 7 | from pymoo.optimize import minimize 8 | 9 | Configuration.show_compile_hint = False 10 | 11 | import sdc_scissor.testing_api.test_generators.ambiegen.config as cf 12 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.duplicate_elimination import DuplicateElimination 13 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.generate_test_case_sampling import GenerateTestCaseSampling 14 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.test_case_crossover import TestCaseCrossover 15 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.test_case_mutation import TestCaseMutation 16 | from sdc_scissor.testing_api.test_generators.ambiegen.Utils.test_case_problem import TestCaseProblem 17 | 18 | 19 | class CustomAmbieGenGenerator: 20 | """ 21 | This test generator creates road points using affine tratsformations to vectors. 22 | Initially generated test cases are optimized by NSGA2 algorithm with two objectives: 23 | fault revealing power and diversity. We use a simplified model of a vehicle to 24 | estimate the fault revealing power (as the maximum deviation from the road center). 25 | We use 100 generations and 100 population size. In each iteration of the generator 26 | the Pareto optimal solutions are provided and executed. Then the algorithm is launched again. 27 | 28 | Sample usage: 29 | sdc-scissor generate-tests -c 30 -k -t "ambiegen" -d "results" 30 | """ 31 | 32 | def __init__(self, time_budget=None, executor=None, map_size=200, **kwargs): 33 | self.map_size = kwargs.get("map_size", map_size) 34 | self.executor = executor 35 | self.count = kwargs.get("count", None) 36 | 37 | def start(self): 38 | """ 39 | In this function the algorithm is launched and 40 | the Pareto optimal solutions are returned 41 | """ 42 | 43 | log.info("Test generation ambiegen.") 44 | algorithm = NSGA2( 45 | n_offsprings=50, 46 | pop_size=cf.ga["population"], 47 | sampling=GenerateTestCaseSampling(), 48 | crossover=TestCaseCrossover(cf.ga["cross_rate"]), 49 | mutation=TestCaseMutation(cf.ga["mut_rate"]), 50 | eliminate_duplicates=DuplicateElimination(), 51 | ) 52 | 53 | generated_tests_count = 0 54 | test_suite = [] 55 | start = time.time() 56 | tests_per_run = 10 57 | while generated_tests_count < self.count: 58 | t = int(time.time() * 1000) 59 | seed = ( 60 | ((t & 0xFF000000) >> 24) + ((t & 0x00FF0000) >> 8) + ((t & 0x0000FF00) << 8) + ((t & 0x000000FF) << 24) 61 | ) 62 | res = minimize( 63 | TestCaseProblem(), 64 | algorithm, 65 | ("n_gen", cf.ga["n_gen"]), 66 | seed=seed, 67 | verbose=False, 68 | save_history=True, 69 | eliminate_duplicates=True, 70 | ) 71 | 72 | # print("Best solution found: \nF = %s" % (res.F)) 73 | gen = len(res.history) - 1 74 | test_cases = [] 75 | i = 0 76 | 77 | while i < tests_per_run and generated_tests_count + i < self.count: 78 | result = res.history[gen].pop.get("X")[i] 79 | 80 | road_points = result[0].intp_points 81 | test_cases.append(road_points) 82 | i += 1 83 | 84 | generated_tests_count += len(test_cases) 85 | test_suite.extend(test_cases) 86 | 87 | # log.info(f"Generated {len(test_suite)} tests in {time.time() - start} seconds.") 88 | return test_suite 89 | -------------------------------------------------------------------------------- /sdc_scissor/sdc_prioritizer/code/runSearch.m: -------------------------------------------------------------------------------- 1 | function [] = runSearch(execution_id,configuration,csv_file, h_status) 2 | %RUNSEARCH Executes a search process according to the given parameters 3 | % global variables for light-weight fitness function evaluation 4 | global A Cost Features H BM; 5 | 6 | 7 | %% Read the dataset 8 | T = readtable(csv_file); 9 | Cost = table2array(T(:,19)); % test execution time is column 19 10 | Features = table2array(T(:,1:16)); % ignore the time stamp (columns 17, 18) 11 | H = h_status; 12 | 13 | %% We normalize all the feautes (min-max normalizaton) 14 | [m, n] = size(Features); 15 | % for i = 1:n 16 | % Features(:,i) = (Features(:,i) - min(Features(:,i)))/ (max(Features(:,i)) - min(Features(:,i))); 17 | % end 18 | %% 19 | 20 | if ~strcmp("default-GA",configuration) 21 | [coeff,scores,latent, tsd, variance,mu] = pca(Features); 22 | if strcmp("10_feature_GA",configuration) 23 | Features = scores(:,1:10); 24 | else 25 | Features = scores(:,1:6); 26 | end 27 | end 28 | 29 | [pathstr,name,ext] = fileparts(csv_file); 30 | BM = name 31 | %% We compute the pairwse (Euclidean) distance between test cases 32 | Differences = pdist(Features,"seuclidean"); 33 | Differences = squareform(Differences); 34 | A = Differences; % this variable is used in the fitness function evaluation 35 | 36 | %% Genetic Algoritm (Chromosomes = test permutations) 37 | options = optimoptions('ga'); 38 | options = optimoptions(options,'PopulationType', 'custom'); 39 | options = optimoptions(options,'PopulationSize', 100); 40 | options = optimoptions(options,'MaxGenerations', 4000); 41 | options = optimoptions(options,'CreationFcn', @initialPopulation); 42 | options = optimoptions(options,'SelectionFcn', @selectionroulette); 43 | options = optimoptions(options,'CrossoverFcn', @permutationCrossover); 44 | options = optimoptions(options,'MutationFcn', @permutationMutation); 45 | options = optimoptions(options,'FunctionTolerance', 0); 46 | options = optimoptions(options,'MaxStallGenerations', 100); 47 | options = optimoptions(options,'Display', 'off'); 48 | options = optimoptions(options,'PlotFcn', { @gaplotbestf, @gaplotscorediversity }); 49 | tic 50 | [x,fval,exitflag,output,population,score] = ga(@fitness,m,[],[],[],[],[],[],[],[],options); 51 | elapsed_time=toc 52 | % x is the optimum permutation of test executions detected by ga 53 | % fval is the fitness value of x 54 | % population is the final population generated by ga 55 | % score contains the fitness values of the individuals in the last population 56 | 57 | %% Comparison in Fault detection 58 | 59 | [a,b] = faultDetection(x, T, Cost); 60 | % a(1,i) is execution cost for running first i tests 61 | % b(1,i) is the number of detected faults after running first i tests 62 | plot(a, b, 'r'); 63 | hold on 64 | 65 | APFD_c = trapz(a, b) / max(a) / max(b) 66 | %% Save search information as a csv file 67 | 68 | 69 | output_dir = strcat("../data/",name,"/",configuration,"/",int2str(execution_id),"/"); 70 | mkdir (output_dir) 71 | 72 | 73 | 74 | 75 | writematrix(x,strcat(output_dir,"best-permutation.csv")) 76 | 77 | rand_APFD = zeros(1,500); 78 | %% Random permutations (baseline) 79 | for i=1:500 80 | [a,b] = faultDetection(randperm(m), T, Cost); 81 | plot(a, b, 'b'); 82 | rand_APFD(1,i) = trapz(a, b) / max(a) / max(b); 83 | end 84 | 85 | writematrix(rand_APFD,strcat(output_dir,"rands.csv")) 86 | 87 | avg_rand_APFD = mean(rand_APFD,2) 88 | std_rand_APFD = std(rand_APFD) 89 | 90 | mat = ["config" "execution_id" "best_fitness_value" "APFD" "elapsed_time" "avg_rand_APFD" "std_rand_APFD";configuration execution_id fval APFD_c elapsed_time avg_rand_APFD std_rand_APFD]; 91 | writematrix(mat,strcat(output_dir,"results.csv")) 92 | 93 | title('Fault detection capability') 94 | xlabel('Execution cost (s)') 95 | ylabel('Number of failures') 96 | 97 | hold off 98 | exportgraphics(gcf,strcat(output_dir,"plot.png")) 99 | end 100 | -------------------------------------------------------------------------------- /sdc_scissor/testing_api/test_generators/ambiegen/Utils/test_case_mutation.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | import numpy as np 4 | from pymoo.model.mutation import Mutation 5 | 6 | import sdc_scissor.testing_api.test_generators.ambiegen.config as cf 7 | 8 | 9 | class TestCaseMutation(Mutation): 10 | """ 11 | Module to perform the mutation 12 | """ 13 | 14 | def __init__(self, mut_rate): 15 | super().__init__() 16 | self.mut_rate = mut_rate 17 | 18 | def _do(self, problem, X, **kwargs): 19 | for i in range(len(X)): 20 | r = np.random.random() 21 | s = X[i, 0] 22 | # with a probabilty of 40% - change the order of characters 23 | if r < self.mut_rate: # cf.ga["mut_rate"]: 24 | sn = copy.deepcopy(s) 25 | 26 | sn.get_points() 27 | sn.remove_invalid_cases() 28 | 29 | wr = np.random.random() 30 | child = sn.states 31 | old_points = sn.road_points 32 | old_states = child 33 | if wr < 0.2: 34 | candidates = list(np.random.randint(0, high=len(child), size=2)) 35 | temp = child["st" + str(candidates[0])] 36 | child["st" + str(candidates[0])] = child["st" + str(candidates[1])] 37 | child["st" + str(candidates[1])] = temp 38 | elif wr >= 0.2 and wr < 0.5: 39 | num = np.random.randint(0, high=len(child)) 40 | value = np.random.choice(["state", "value"]) 41 | if value == "value": 42 | duration_list = [] 43 | if child["st" + str(num)]["state"] == "straight": 44 | duration_list = np.arange(cf.model["min_len"], cf.model["max_len"], 1) 45 | else: 46 | duration_list = np.arange(cf.model["min_angle"], cf.model["max_angle"], 5) 47 | 48 | child["st" + str(num)][value] = int(np.random.choice(duration_list)) 49 | 50 | elif value == "state": 51 | if child["st" + str(num)][value] == "straight": 52 | child["st" + str(num)][value] = np.random.choice(["left", "right"]) 53 | duration_list = np.arange(cf.model["min_angle"], cf.model["max_angle"], 5) 54 | child["st" + str(num)]["value"] = int(np.random.choice(duration_list)) 55 | else: 56 | child["st" + str(num)][value] = "straight" 57 | duration_list = np.arange(cf.model["min_len"], cf.model["max_len"], 1) 58 | child["st" + str(num)]["value"] = int(np.random.choice(duration_list)) 59 | 60 | else: 61 | cand = list(np.random.randint(0, high=len(child), size=int(len(child) / 2))) 62 | while cand: 63 | c1 = np.random.choice(cand) 64 | cand.remove(c1) 65 | if cand: 66 | c2 = np.random.choice(cand) 67 | cand.remove(c2) 68 | temp = child["st" + str(c1)] 69 | child["st" + str(c1)] = child["st" + str(c2)] 70 | child["st" + str(c2)] = temp 71 | else: 72 | if child["st" + str(c1)]["state"] == "straight": 73 | duration_list = np.arange(cf.model["min_len"], cf.model["max_len"], 1) 74 | else: 75 | duration_list = np.arange(cf.model["min_angle"], cf.model["max_angle"], 5) 76 | child["st" + str(c1)]["value"] = int(np.random.choice(duration_list)) 77 | 78 | sn.states = child 79 | sn.get_points() 80 | sn.remove_invalid_cases() 81 | sn.novelty = sn.calc_novelty(old_states, sn.states) 82 | 83 | X[i, 0] = sn 84 | 85 | return X 86 | -------------------------------------------------------------------------------- /docs/user_documentation/feature_extraction.md: -------------------------------------------------------------------------------- 1 | # Feature Extraction 2 | SDC-Scissor uses road features of the test specifications for its machine learning models. 3 | To obtain the road features, we need to extract them from the test specification with the `extract-features` command. 4 | 5 | ```text 6 | Usage: sdc-scissor extract-features [OPTIONS] 7 | 8 | Extract road features from given test scenarios. 9 | 10 | Options: 11 | -t, --tests PATH Path to directory containing the tests 12 | -s, --segmentation TEXT 13 | --help 14 | ``` 15 | 16 | The following figure illustrates a road specification with its segments, direct distances of the segments sd well the spanned ares for measuring the diversity. 17 | The exact features are described in the table below. 18 | 19 | ![](../images/diversity-feature.jpeg) 20 | 21 | The command will output a CSV file describing the following features per test case: 22 | 23 | | Feature | Description | 24 | |---------------------|-----------------------------------------------------------------------------------------------------------| 25 | | direct_distance | Direct distance from the start point to the end point | 26 | | road_distance | Distance of the road | 27 | | num_l_turns | Number of left turns | 28 | | num_r_turns | Number of right turns | 29 | | num_straights | Number of straight road segments without turns | 30 | | median_angle | Median turn angle of all turns of the road | 31 | | total_angle | Total cumulated angle of all turns | 32 | | mean_angle | Mean of all turn angles | 33 | | std_angle | Standard deviation of the turn angles of the road | 34 | | max_angle | Maximum turn angle of the road | 35 | | min_angle | Minimum turn angle of the road | 36 | | median_pivot_off | Median radius of all turns of the road | 37 | | mean_pivot_off | Mean arithmetic mean of all turn radius | 38 | | std_pivot_off | Standard deviation of turn radius of the road | 39 | | max_pivot_off | Maximum radius of all turns of the road | 40 | | min_pivot_off | Minimum radius of all turns of the road | 41 | | mean_road_diversity | Mean area of all spanned areas by segments between the actual road and the direct distance of the segment | 42 | | full_road_diversity | Cumulative area spanned by all roads and direct distances per segment | 43 | | test_duration | Duration of the test (not available before test execution) | 44 | | safety | Test outcome specifying if the car was in lane or not (not available before test execution) | 45 | 46 | ## Options 47 | The command `extract-features` comes with several options. 48 | Those options are mainly about to configure the feature extraction process. 49 | 50 | ```{eval-rst} 51 | .. autofunction:: sdc_scissor.cli.extract_features 52 | ``` 53 | 54 | ## Feature Statistics 55 | To get an overview of the descriptive statistics of the extracted road features of your tests you can run the `feature-statistics` command. 56 | 57 | ```{eval-rst} 58 | .. autofunction:: sdc_scissor.cli.feature_statistics 59 | ``` 60 | --------------------------------------------------------------------------------