├── README.md ├── grader.py └── grader_config.sample.json /README.md: -------------------------------------------------------------------------------- 1 | # Script to test multiple maven java projects and store the outputs in a json file 2 | 3 | This script was designed to help the grading of projects within the Gothenburg University Object Oriented Programming course DIT043. The idea is that groups create maven projects as part of the curriculum, and the teaching team wanted to automate a portion of the grading of these projects. this script allows for the teaching team to put all the student groups projects within a directory and run the teaching teams unit tests against all the student group projects. The maven test results or output to a json file a defined grades directory; if no grades directory is defined one will be made at the same location as the script. The methods of the script are commented to explain the purpose of the method. The script could be extended to work with other frameworks then just maven, however a defined project structure is necessary. 4 | 5 | ## Configuration 6 | 7 | to customize the behavior of the script you can create a grader_config.json file in the same directory as the grader.py script 8 | 9 | **for the purposes of this config ./ is equivalent to location of grader.py** 10 | ```javascript 11 | { 12 | "projects_dir": "", // This is the directory of the student projects that you wish to be graded 13 | "project_structure": { // This defines where the tests directory and pom.xml files are found relative to the root of the individual projects. 14 | "tests_dir": "", // test Directory - default: src/test 15 | "pom_file": "" // pom.xml - default: pom.xml 16 | }, 17 | "resources_path": { // Location of the Teaching teams resources to be used to test the student projects 18 | "tests": "", // test directoy - default: ./resources/test 19 | "pom": "" // pom.xml file - default: ./resources/pom.xml 20 | }, 21 | "test_command": "", // command to run on projects - default: mvn test 22 | "grade_output_dir": "", // the directory to output the grades as json and csv files - default: ./grades/{timestamp}grades.json and ./grades/{timestamp}grades.csv 23 | "output_timestamp_format": "" // the format for the prefix of the grades outputfiles - default: %Y-%m-%d-%H-%M-%S_ 24 | } 25 | ``` 26 | 27 | ## Contributors 28 | - [DrakeAxelrod](https://github.com/DrakeAxelrod) 29 | 30 | ## Required Python Library 31 | 32 | pandas - this is to convert from json to csv 33 | 34 | ## Extensions that could be made 35 | 36 | - Threading to increase the speed 37 | - the ability to work with other frameworks then just maven 38 | - additional output formats to be specified in config 39 | -------------------------------------------------------------------------------- /grader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright 2021 Drake Axelrod 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software 5 | and associated documentation files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom 8 | the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or 11 | substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 14 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 15 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 16 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 17 | CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 18 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | """ 20 | import os 21 | import json 22 | from pathlib import Path 23 | import shutil 24 | import sys 25 | import re 26 | import pandas as pd 27 | import datetime 28 | 29 | # config defaults 30 | CWD = str(Path.cwd()) 31 | CONFIG_NAME = "grader_config.json" 32 | CONFIG_PROJECTS_DIR = "/projects" 33 | CONFIG_PROJECT_STRUCTURE = { "tests_dir": "src/test", "pom_file": "pom.xml" } 34 | CONFIG_RESOURCE_PATH= { "tests": CWD + "/resources/test", "pom": CWD + "/resources/pom.xml" } 35 | CONFIG_TEST_COMMAND = "mvn test" 36 | CONFIG_GRADES_DIR = CWD + "/grades" 37 | CONFIG_OUTPUT_TIMESTAMP_FORMAT = "%Y-%m-%d-%H-%M-%S_" 38 | config: dict = json.load(open(CONFIG_NAME )) if os.path.isfile(CONFIG_NAME) else { } 39 | config.setdefault("projects_dir", CWD + "/projects" ), 40 | config.setdefault("project_structure", CONFIG_PROJECT_STRUCTURE) 41 | config.setdefault("resources_path", CONFIG_RESOURCE_PATH) 42 | config.setdefault("test_command", CONFIG_TEST_COMMAND) 43 | config.setdefault("grades_output_dir", CONFIG_GRADES_DIR) 44 | config.setdefault("output_timestamp_format", CONFIG_OUTPUT_TIMESTAMP_FORMAT) 45 | 46 | # assign config settings 47 | PROJECTS_DIR = Path(config["projects_dir"]) 48 | TEST_COMMAND = config["test_command"] 49 | GRADES_OUTPUT_DIR = Path(config["grades_output_dir"]) 50 | TEST_DIR = Path(config["resources_path"]["tests"]) 51 | POM_FILE = Path(config["resources_path"]["pom"]) 52 | PROJECT_TESTS_PATH = Path(config["project_structure"]["tests_dir"]) 53 | PROJECT_POM_FILE_PATH = Path(config["project_structure"]["pom_file"]) 54 | TIMESTAMP_FORMAT = config["output_timestamp_format"] 55 | 56 | # individual project symantics 57 | MODIFIED_PROJECT_TESTS_PATH = PROJECT_TESTS_PATH.with_name("original_test") 58 | MODIFIED_PROJECT_POM_FILE_PATH = PROJECT_POM_FILE_PATH.with_name("original_pom.xml") 59 | 60 | #make sure everything is where its supposed to be or inform the user that something is wrong 61 | if not PROJECTS_DIR.is_dir(): 62 | print(f"projects directory does not exist or has been configured incorrectly") 63 | exit(1) 64 | if not GRADES_OUTPUT_DIR.is_dir(): 65 | Path.mkdir(GRADES_OUTPUT_DIR) 66 | if not TEST_DIR.is_dir(): 67 | print(f"test suite directory does not exist or has been configured incorrectly") 68 | exit(1) 69 | if not POM_FILE.is_file(): 70 | print(f"pom file does not exist or has been configured incorrectly") 71 | exit(1) 72 | 73 | all_projects = list(PROJECTS_DIR.iterdir()) 74 | all_projects.sort() 75 | 76 | # progressbar - code was taken from https://stackoverflow.com/a/34482761 77 | def progressbar(it, prefix="", size=60, file=sys.stdout): 78 | count = len(it) 79 | def show(j): 80 | x = int(size*j/count) 81 | file.write("%s[%s%s] %i/%i\r" % (prefix, "#"*x, "."*(size-x), j, count)) 82 | file.flush() 83 | show(0) 84 | for i, item in enumerate(it): 85 | yield item 86 | show(i+1) 87 | file.write("\n") 88 | file.flush() 89 | 90 | # timestamp - code was taken from https://stackoverflow.com/a/5215012 91 | def timeStamped(fname): 92 | fmt = '%Y-%m-%d-%H-%M-%S_{fname}' 93 | fmt = f"{TIMESTAMP_FORMAT}{fname}" 94 | return datetime.datetime.now().strftime(fmt).format(fname=fname) 95 | 96 | # parse the output of mvn test specifically 97 | def sanitize_results(result: str) -> str: 98 | start_str = "T E S T S\n[INFO] -------------------------------------------------------\n" 99 | start = result.find(start_str) + len(start_str) 100 | end = result.find("BUILD SUCCESS") 101 | substring = result[start:end] 102 | line_arr = substring.splitlines() 103 | arr = [] 104 | for line in line_arr: 105 | s = re.sub("Running |\[INFO\] |\n|-*", "", line) 106 | if s != "": 107 | arr.append(s) 108 | project_dict: dict = {} 109 | while len(arr) > 0: 110 | val: str = arr.pop() 111 | key = arr.pop() 112 | project_dict[key] = str_to_dict(val) 113 | # add field for results for csv 114 | project_dict["Results:"]["Time elapsed"] = "empty" 115 | # remove time elapsed 116 | for project in project_dict: 117 | del project_dict[project]["Time elapsed"] 118 | return project_dict 119 | 120 | 121 | # parses a str such as "{'Tests run': 45, 'Failures': 0, 'Errors': 0, 'Skipped': 0}" to a dict 122 | def str_to_dict(string): 123 | my_dict = {} 124 | str_arr = string.split(",") 125 | for s in str_arr: 126 | i = s.split(': ') 127 | # if its an int cast otherwise dont 128 | my_dict[i[0].strip()] = int(i[1]) if (i[1]).isdigit() else i[1] 129 | return my_dict 130 | 131 | # renames a dir or file in a project 132 | def rename_resource(project: Path, original: Path, modifed: Path): 133 | try: 134 | Path(project / original).rename(Path(project / modifed)) 135 | except: 136 | print(f"could not rename the {original} in {project.stem}") 137 | 138 | # changes the projects test dir and pom.xml names so that they are not used in mvn test 139 | def invalidate_project_resources(project: Path): 140 | rename_resource(project, PROJECT_TESTS_PATH, MODIFIED_PROJECT_TESTS_PATH) 141 | rename_resource(project, PROJECT_POM_FILE_PATH, MODIFIED_PROJECT_POM_FILE_PATH) 142 | 143 | # repairs the changes from _invalidate_project_resources 144 | def fix_project_resources(project: Path): 145 | rename_resource(project, MODIFIED_PROJECT_TESTS_PATH, PROJECT_TESTS_PATH) 146 | rename_resource(project, MODIFIED_PROJECT_POM_FILE_PATH, PROJECT_POM_FILE_PATH) 147 | 148 | # copies the designated test dir and pom.xml into the project 149 | def cp_resources_to_project(project: Path): 150 | path = project/PROJECT_TESTS_PATH 151 | try: 152 | shutil.copytree(TEST_DIR, path) 153 | except: 154 | print(f"could not find {TEST_DIR} or {path}") 155 | try: 156 | shutil.copy(POM_FILE, project/PROJECT_POM_FILE_PATH) 157 | except: 158 | print(f"could not find {POM_FILE} or {path}") 159 | 160 | # removes the copied test dir and pom.xml from the project 161 | def rm_resources_from_project(project: Path): 162 | shutil.rmtree(str(project/PROJECT_TESTS_PATH)) 163 | os.remove(str(project/PROJECT_POM_FILE_PATH)) 164 | 165 | # grades a project via mvn test 166 | def grade_project(project: Path): 167 | invalidate_project_resources(project) 168 | cp_resources_to_project(project) 169 | result = _run_mvn_test(project) 170 | rm_resources_from_project(project) 171 | fix_project_resources(project) 172 | return result 173 | 174 | # runs mvn test from the current project root directory 175 | def _run_mvn_test(project: Path): 176 | os.chdir(project) 177 | return sanitize_results(os.popen(TEST_COMMAND).read()) 178 | 179 | # loops through the projects and grades them building a dict of results 180 | def grade_all_projects(): 181 | grades = {} 182 | for project in progressbar(list(all_projects), "Grading Projects: ", 60): 183 | try: 184 | grades[project.stem] = grade_project(project) 185 | except: 186 | print(f"could not grade {project} due to an error in the project structure") 187 | return grades 188 | 189 | # parses the grades dict into a json and csv 190 | def output_grades(grades: dict): 191 | json_path: str = str(GRADES_OUTPUT_DIR/timeStamped("grades.json")) 192 | csv_path: str = str(GRADES_OUTPUT_DIR/timeStamped("grades.csv")) 193 | # save as json (needed for csv) 194 | with open(json_path, 'w') as f: 195 | json.dump(grades, indent=4 ,fp=f) 196 | f.close() 197 | # save as csv 198 | df: pd.DataFrame = pd.read_json(json_path) 199 | df = df.to_csv() 200 | with open(csv_path, 'w') as f: 201 | f.write(df) 202 | f.close 203 | 204 | # makes the script go zoom! 205 | output_grades(grade_all_projects()) 206 | -------------------------------------------------------------------------------- /grader_config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "projects_dir": "", 3 | "project_structure": { 4 | "tests_dir": "", 5 | "pom_file": "" 6 | }, 7 | "resources_path": { 8 | "tests": "", 9 | "pom": "" 10 | }, 11 | "test_command": "", 12 | "grade_output_dir": "", 13 | "output_timestamp_format": "" 14 | } 15 | --------------------------------------------------------------------------------