├── .dockerignore ├── .env.development ├── .env.production ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── .prettierrc.js ├── Dockerfile ├── LICENSE ├── README.md ├── babel.config.js ├── build_zip.bash ├── contributing.md ├── jest.config.js ├── ota_interface.py ├── pack.py ├── package-lock.json ├── package.json ├── public ├── favicon.ico └── index.html ├── src ├── App.vue ├── assets │ └── logo.png ├── components │ ├── BaseCheckbox.vue │ ├── BaseFile.vue │ ├── BaseInput.vue │ ├── BaseRadio.vue │ ├── BaseRadioGroup.vue │ ├── BaseSelect.vue │ ├── BatchOTAOptions.vue │ ├── BuildLibrary.vue │ ├── BuildTable.vue │ ├── ChainOTAOptions.vue │ ├── FileList.vue │ ├── FileSelect.vue │ ├── JobConfiguration.vue │ ├── JobDisplay.vue │ ├── OTAJobTable.vue │ ├── OTAOptions.vue │ ├── PartialCheckbox.vue │ ├── SingleOTAOptions.vue │ └── UploadFile.vue ├── main.js ├── plugins │ └── vuetify.js ├── router │ └── index.js ├── services │ ├── ApiService.js │ ├── FormDate.js │ ├── JobSubmission.js │ └── TableService.js ├── store │ └── index.js └── views │ ├── About.vue │ ├── JobConfigure.vue │ ├── JobDetails.vue │ ├── JobList.vue │ └── NotFound.vue ├── target_lib.py ├── test ├── test_ab_partitions.txt └── test_build.prop ├── test_ota_interface.py ├── test_suite.py ├── test_target_lib.py ├── vue.config.js ├── web_server.py └── web_server_flask.py /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | Dockerfile 3 | .vscode 4 | dist 5 | output 6 | target 7 | *.db 8 | .git 9 | -------------------------------------------------------------------------------- /.env.development: -------------------------------------------------------------------------------- 1 | NODE_ENV=development 2 | -------------------------------------------------------------------------------- /.env.production: -------------------------------------------------------------------------------- 1 | NODE_ENV=production 2 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | src/services/update_metadata_pb.js -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es6": true, 5 | "node": true 6 | }, 7 | "parserOptions": { 8 | "parser": "babel-eslint" 9 | }, 10 | "extends": [ 11 | "plugin:vue/recommended" 12 | ], 13 | "rules": { 14 | "indent": [ 15 | "error", 16 | 2 17 | ], 18 | "vue/no-multiple-template-root": 0, 19 | "vue/attribute-hyphenation": 0 20 | } 21 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | /target 5 | /output 6 | stderr* 7 | stdout* 8 | yarn.lock 9 | otatools.zip 10 | 11 | 12 | # local env files 13 | .env.local 14 | .env.*.local 15 | 16 | # Log files 17 | npm-debug.log* 18 | yarn-debug.log* 19 | yarn-error.log* 20 | pnpm-debug.log* 21 | 22 | # Editor directories and files 23 | .idea 24 | .vscode 25 | *.suo 26 | *.ntvs* 27 | *.njsproj 28 | *.sln 29 | *.sw? 30 | *.db 31 | 32 | packaged 33 | **/.DS_Store 34 | *.pyc 35 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module.exports = { 18 | singleQuote: true, 19 | semi: false, 20 | useTabs: false 21 | } 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # build stage 16 | FROM node:lts-alpine as build-stage 17 | WORKDIR /app 18 | COPY package*.json ./ 19 | COPY yarn.lock ./ 20 | RUN yarn install 21 | COPY src ./src 22 | COPY public ./public 23 | COPY *.js . 24 | COPY .env* . 25 | COPY .eslint* . 26 | 27 | RUN npm run build 28 | 29 | # production stage 30 | FROM ubuntu:20.04 as production-stage 31 | RUN apt-get update && apt-get --no-install-recommends install -y python3.9 unzip xxd cgpt openjdk-16-jre-headless zip less 32 | 33 | WORKDIR /app 34 | VOLUME [ "/app/target", "/app/output"] 35 | 36 | COPY otatools.zip . 37 | RUN zip otatools.zip -d bin/sign_apex bin/aapt2 bin/merge_target_files bin/sign_target_files_apks bin/add_img_to_target_files bin/build_image bin/validate_target_files bin/img_from_target_files bin/check_target_files_vintf bin/build_super_image bin/mkuserimg_mke2fs bin/mk_combined_img bin/apexer bin/build_verity_metadata bin/fc_sort "*.pyc" || true 38 | COPY --from=build-stage /app/dist ./dist 39 | COPY *.py . 40 | 41 | EXPOSE 8000 42 | CMD ["python3.9", "web_server.py"] 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ota-generator 2 | 3 | ## Introduction 4 | ota-generator is a web interface for `ota_from_target_files`. Currently, it can only run locally. 5 | 6 | `ota_from_target_files` is Android's standard tool for building OTA packages. It's source 7 | is available in [aosp](https://cs.android.com/android/platform/superproject/+/master:build/make/tools/releasetools/ota_from_target_files.py). 8 | Binaries of `ota_from_target_files` is available in [ci.android.com](https://ci.android.com). 9 | For documentation about `ota_from_target_files` , click on the aosp link. 10 | 11 | OTAGUI use VUE.js as a frontend and python as a backend interface to ota_from_target_files. 12 | 13 | ## Usage 14 | 15 | ### Download otatools.zip 16 | 1. Goto https://ci.android.com/builds/branches/aosp-master/grid 17 | 2. Click on any of the green squares in aosp_cf_x86_64_phone column 18 | 3. Goto artifacts tab and download otatools.zip, put it in the same folder with this README.md 19 | 20 | Use `npm build` to install the dependencies. 21 | 22 | Create a `target` directory to store the target files and a `output` directory 23 | to store the output files: 24 | ``` 25 | mkdir target 26 | mkdir output 27 | ``` 28 | 29 | Finally, run the python http-server and vue.js server: 30 | ``` 31 | python3 web_server.py & 32 | npm run serve 33 | ``` 34 | ### Run with Docker 35 | 36 | 1. Build the image `docker build -t $USER/ota-generator .` 37 | 38 | 2. Run: `docker run -it -p 8000:8000 -v target:/app/target -v output:/app/output $USER/ota-generator` 39 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module.exports = { 18 | presets: ["@vue/cli-plugin-babel/preset"] 19 | }; 20 | -------------------------------------------------------------------------------- /build_zip.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | zip otatools.zip -d bin/sign_apex bin/aapt2 \ 3 | bin/merge_target_files bin/sign_target_files_apks \ 4 | bin/add_img_to_target_files bin/build_image \ 5 | bin/validate_target_files bin/img_from_target_files \ 6 | bin/check_target_files_vintf bin/build_super_image \ 7 | bin/mkuserimg_mke2fs bin/mk_combined_img bin/apexer \ 8 | bin/zucchini \ 9 | bin/build_verity_metadata bin/fc_sort "*.pyc" || true 10 | 11 | image_id=$(docker build -q .) 12 | container_id=$(docker run -d --entrypoint /usr/bin/sleep ${image_id} 120) 13 | docker container exec ${container_id} zip /app.zip -r /app 14 | docker container cp ${container_id}:/app.zip . 15 | docker container stop ${container_id} 16 | docker container rm ${container_id} 17 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code Reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module.exports = { 18 | preset: '@vue/cli-plugin-unit-jest', 19 | transform: { 20 | '^.+\\.vue$': 'vue-jest' 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ota_interface.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import subprocess 16 | import os 17 | import pipes 18 | import threading 19 | from dataclasses import dataclass, asdict, field 20 | import logging 21 | import sqlite3 22 | import time 23 | 24 | 25 | @dataclass 26 | class JobInfo: 27 | """ 28 | A class for ota job information 29 | """ 30 | id: str 31 | target: str 32 | incremental: str = '' 33 | verbose: bool = False 34 | partial: list[str] = field(default_factory=list) 35 | output: str = '' 36 | status: str = 'Running' 37 | downgrade: bool = False 38 | extra: str = '' 39 | stdout: str = '' 40 | stderr: str = '' 41 | start_time: int = 0 42 | finish_time: int = 0 43 | isPartial: bool = False 44 | isIncremental: bool = False 45 | 46 | @property 47 | def is_running(self): 48 | return self.status == 'Running' 49 | 50 | @property 51 | def is_killed(self): 52 | return self.status == 'Killed' 53 | 54 | def __post_init__(self): 55 | 56 | def enforce_bool(t): return t if isinstance(t, bool) else bool(t) 57 | self.verbose, self.downgrade = map( 58 | enforce_bool, 59 | [self.verbose, self.downgrade]) 60 | if self.incremental: 61 | self.isIncremental = True 62 | if self.partial: 63 | self.isPartial = True 64 | else: 65 | self.partial = [] 66 | if type(self.partial) == str: 67 | self.partial = self.partial.split(',') 68 | 69 | def to_sql_form_dict(self): 70 | """ 71 | Convert this instance to a dict, which can be later used to insert into 72 | the SQL database. 73 | Format: 74 | id: string, target: string, incremental: string, verbose: int, 75 | partial: string, output:string, status:string, 76 | downgrade: bool, extra: string, stdout: string, stderr:string, 77 | start_time:int, finish_time: int(not required) 78 | """ 79 | sql_form_dict = asdict(self) 80 | sql_form_dict['partial'] = ','.join(sql_form_dict['partial']) 81 | def bool_to_int(t): return 1 if t else 0 82 | sql_form_dict['verbose'], sql_form_dict['downgrade'] = map( 83 | bool_to_int, 84 | [sql_form_dict['verbose'], sql_form_dict['downgrade']]) 85 | return sql_form_dict 86 | 87 | def to_dict_basic(self): 88 | """ 89 | Convert the instance to a dict, which includes the file name of target. 90 | """ 91 | basic_info = asdict(self) 92 | basic_info['target_name'] = self.target.split('/')[-1] 93 | if self.isIncremental: 94 | basic_info['incremental_name'] = self.incremental.split('/')[-1] 95 | return basic_info 96 | 97 | def to_dict_detail(self, target_lib, offset=0): 98 | """ 99 | Convert this instance into a dict, which includes some detailed information 100 | of the target/source build, i.e. build version and file name. 101 | """ 102 | detail_info = asdict(self) 103 | try: 104 | with open(self.stdout, 'r') as fout: 105 | detail_info['stdout'] = fout.read() 106 | with open(self.stderr, 'r') as ferr: 107 | detail_info['stderr'] = ferr.read() 108 | except FileNotFoundError: 109 | detail_info['stdout'] = 'NO STD OUTPUT IS FOUND' 110 | detail_info['stderr'] = 'NO STD ERROR IS FOUND' 111 | target_info = target_lib.get_build_by_path(self.target) 112 | detail_info['target_name'] = target_info.file_name 113 | detail_info['target_build_version'] = target_info.build_version 114 | if self.incremental: 115 | incremental_info = target_lib.get_build_by_path( 116 | self.incremental) 117 | detail_info['incremental_name'] = incremental_info.file_name 118 | detail_info['incremental_build_version'] = incremental_info.build_version 119 | return detail_info 120 | 121 | 122 | class DependencyError(Exception): 123 | pass 124 | 125 | 126 | class ProcessesManagement: 127 | """ 128 | A class manage the ota generate process 129 | """ 130 | 131 | @staticmethod 132 | def check_external_dependencies(): 133 | try: 134 | java_version = subprocess.check_output(["java", "--version"]) 135 | print("Java version:", java_version.decode()) 136 | except Exception as e: 137 | raise DependencyError( 138 | "java not found in PATH. Attempt to generate OTA might fail. " + str(e)) 139 | try: 140 | zip_version = subprocess.check_output(["zip", "-v"]) 141 | print("Zip version:", zip_version.decode()) 142 | except Exception as e: 143 | raise DependencyError( 144 | "zip command not found in PATH. Attempt to generate OTA might fail. " + str(e)) 145 | 146 | def __init__(self, *, working_dir='output', db_path=None, otatools_dir=None): 147 | """ 148 | create a table if not exist 149 | """ 150 | ProcessesManagement.check_external_dependencies() 151 | self.working_dir = working_dir 152 | self.logs_dir = os.path.join(working_dir, 'logs') 153 | self.otatools_dir = otatools_dir 154 | os.makedirs(self.working_dir, exist_ok=True) 155 | os.makedirs(self.logs_dir, exist_ok=True) 156 | if not db_path: 157 | db_path = os.path.join(self.working_dir, "ota_database.db") 158 | self.path = db_path 159 | with sqlite3.connect(self.path) as connect: 160 | cursor = connect.cursor() 161 | cursor.execute(""" 162 | CREATE TABLE if not exists Jobs ( 163 | ID TEXT, 164 | TargetPath TEXT, 165 | IncrementalPath TEXT, 166 | Verbose INTEGER, 167 | Partial TEXT, 168 | OutputPath TEXT, 169 | Status TEXT, 170 | Downgrade INTEGER, 171 | OtherFlags TEXT, 172 | STDOUT TEXT, 173 | STDERR TEXT, 174 | StartTime INTEGER, 175 | FinishTime INTEGER 176 | ) 177 | """) 178 | for job in self.get_running_jobs(): 179 | end_time = min(os.stat(job.stdout).st_mtime, 180 | os.stat(job.stderr).st_mtime) 181 | logging.info( 182 | "Updating %s to status 'Killed', end time %d", job.id, end_time) 183 | self.update_status(job.id, 'Killed', end_time) 184 | 185 | def insert_database(self, job_info): 186 | """ 187 | Insert the job_info into the database 188 | Args: 189 | job_info: JobInfo 190 | """ 191 | with sqlite3.connect(self.path) as connect: 192 | cursor = connect.cursor() 193 | cursor.execute(""" 194 | INSERT INTO Jobs (ID, TargetPath, IncrementalPath, Verbose, Partial, OutputPath, Status, Downgrade, OtherFlags, STDOUT, STDERR, StartTime, Finishtime) 195 | VALUES (:id, :target, :incremental, :verbose, :partial, :output, :status, :downgrade, :extra, :stdout, :stderr, :start_time, :finish_time) 196 | """, job_info.to_sql_form_dict()) 197 | 198 | def get_status_by_ID(self, id): 199 | """ 200 | Return the status of job as a instance of JobInfo 201 | Args: 202 | id: string 203 | Return: 204 | JobInfo 205 | """ 206 | with sqlite3.connect(self.path) as connect: 207 | cursor = connect.cursor() 208 | logging.info(id) 209 | cursor.execute(""" 210 | SELECT * 211 | FROM Jobs WHERE ID=(?) 212 | """, (str(id),)) 213 | row = cursor.fetchone() 214 | status = JobInfo(*row) 215 | return status 216 | 217 | def get_running_jobs(self): 218 | with sqlite3.connect(self.path) as connect: 219 | cursor = connect.cursor() 220 | cursor.execute(""" 221 | SELECT * 222 | FROM Jobs 223 | WHERE Status == 'Running' 224 | """) 225 | rows = cursor.fetchall() 226 | statuses = [JobInfo(*row) for row in rows] 227 | return statuses 228 | 229 | def get_status(self): 230 | """ 231 | Return the status of all jobs as a list of JobInfo 232 | Return: 233 | List[JobInfo] 234 | """ 235 | with sqlite3.connect(self.path) as connect: 236 | cursor = connect.cursor() 237 | cursor.execute(""" 238 | SELECT * 239 | FROM Jobs 240 | """) 241 | rows = cursor.fetchall() 242 | statuses = [JobInfo(*row) for row in rows] 243 | return statuses 244 | 245 | def update_status(self, id, status, finish_time): 246 | """ 247 | Change the status and finish time of job in the database 248 | Args: 249 | id: string 250 | status: string 251 | finish_time: int 252 | """ 253 | with sqlite3.connect(self.path) as connect: 254 | cursor = connect.cursor() 255 | cursor.execute(""" 256 | UPDATE Jobs SET Status=(?), FinishTime=(?) 257 | WHERE ID=(?) 258 | """, 259 | (status, finish_time, id)) 260 | 261 | def ota_run(self, command, id, stdout_path, stderr_path): 262 | """ 263 | Initiate a subprocess to run the ota generation. Wait until it finished and update 264 | the record in the database. 265 | """ 266 | stderr_pipes = pipes.Template() 267 | stdout_pipes = pipes.Template() 268 | ferr = stderr_pipes.open(stdout_path, 'w') 269 | fout = stdout_pipes.open(stderr_path, 'w') 270 | env = {} 271 | if self.otatools_dir: 272 | env['PATH'] = os.path.join( 273 | self.otatools_dir, "bin") + ":" + os.environ["PATH"] 274 | # TODO(lishutong): Enable user to use self-defined stderr/stdout path 275 | try: 276 | proc = subprocess.Popen( 277 | command, stderr=ferr, stdout=fout, shell=False, env=env, cwd=self.otatools_dir) 278 | self.update_status(id, 'Running', 0) 279 | except FileNotFoundError as e: 280 | logging.error('ota_from_target_files is not set properly %s', e) 281 | self.update_status(id, 'Error', int(time.time())) 282 | raise 283 | except Exception as e: 284 | logging.error('Failed to execute ota_from_target_files %s', e) 285 | self.update_status(id, 'Error', int(time.time())) 286 | raise 287 | 288 | def wait_result(): 289 | try: 290 | exit_code = proc.wait() 291 | finally: 292 | if exit_code == 0: 293 | self.update_status(id, 'Finished', int(time.time())) 294 | else: 295 | self.update_status(id, 'Error', int(time.time())) 296 | threading.Thread(target=wait_result).start() 297 | 298 | def ota_generate(self, args, id): 299 | """ 300 | Read in the arguments from the frontend and start running the OTA 301 | generation process, then update the records in database. 302 | Format of args: 303 | output: string, extra_keys: List[string], extra: string, 304 | isIncremental: bool, isPartial: bool, partial: List[string], 305 | incremental: string, target: string, verbose: bool 306 | args: 307 | args: dict 308 | id: string 309 | """ 310 | command = ['ota_from_target_files'] 311 | # Check essential configuration is properly set 312 | if not os.path.isfile(args['target']): 313 | raise FileNotFoundError 314 | if not 'output' in args: 315 | args['output'] = os.path.join(self.working_dir, str(id) + '.zip') 316 | if args['verbose']: 317 | command.append('-v') 318 | if args['extra_keys']: 319 | args['extra'] = '--' + \ 320 | ' --'.join(args['extra_keys']) + ' ' + args['extra'] 321 | if args['extra']: 322 | command += args['extra'].strip().split(' ') 323 | if args['isIncremental']: 324 | if not os.path.isfile(args['incremental']): 325 | raise FileNotFoundError 326 | command.append('-i') 327 | command.append(os.path.realpath(args['incremental'])) 328 | if args['isPartial']: 329 | command.append('--partial') 330 | command.append(' '.join(args['partial'])) 331 | command.append(os.path.realpath(args['target'])) 332 | command.append(os.path.realpath(args['output'])) 333 | stdout = os.path.join(self.logs_dir, 'stdout.' + str(id)) 334 | stderr = os.path.join(self.logs_dir, 'stderr.' + str(id)) 335 | job_info = JobInfo(id, 336 | target=args['target'], 337 | incremental=args['incremental'] if args['isIncremental'] else '', 338 | verbose=args['verbose'], 339 | partial=args['partial'] if args['isPartial'] else [ 340 | ], 341 | output=args['output'], 342 | status='Pending', 343 | extra=args['extra'], 344 | start_time=int(time.time()), 345 | stdout=stdout, 346 | stderr=stderr 347 | ) 348 | self.insert_database(job_info) 349 | self.ota_run(command, id, job_info.stdout, job_info.stderr) 350 | logging.info( 351 | 'Starting generating OTA package with id {}: \n {}' 352 | .format(id, command)) 353 | -------------------------------------------------------------------------------- /pack.py: -------------------------------------------------------------------------------- 1 | #!/bin/python3 2 | 3 | """ 4 | Given a path to a python script, recursively resolve all imports and copy 5 | imported modules to a output directory. 6 | """ 7 | 8 | 9 | import ast 10 | import inspect 11 | import importlib 12 | import os 13 | import sys 14 | from types import ModuleType 15 | import shutil 16 | import re 17 | import stdlib_list 18 | 19 | from stdlib_list import stdlib_list 20 | 21 | stdlibs = stdlib_list() 22 | 23 | PYTHON_DIR = os.path.dirname(sys.executable) 24 | 25 | 26 | def get_module_relative_component(path: str): 27 | output = [] 28 | for p in sys.path: 29 | if path.startswith(p): 30 | output.append(path.removeprefix(p).removeprefix("/")) 31 | output.sort(key=len) 32 | if len(output) > 0: 33 | return output[0] 34 | 35 | 36 | _STDLIB_PATH_PATTERN = re.compile(r"lib/[Pp]ython[2-9]\.\d+$") 37 | 38 | 39 | def is_stdlib_module(module): 40 | return module.__name__ in stdlibs or \ 41 | PYTHON_DIR in module.__file__ or \ 42 | _STDLIB_PATH_PATTERN.search(os.path.dirname(module.__file__)) 43 | 44 | 45 | def enumerate_modules(module, visited=None): 46 | if visited is None: 47 | yield module.__name__, module.__file__, os.path.basename(module.__file__) 48 | visited = set() 49 | # Process direct imports like 50 | # import abc 51 | # or 52 | # from module1 import module2 53 | # note if module2 is a function, the loop below won't process it. 54 | if is_stdlib_module(module): 55 | return 56 | for (k, v) in list(module.__dict__.items()): 57 | if isinstance(v, ModuleType) and hasattr(v, '__file__'): 58 | if is_stdlib_module(v): 59 | continue 60 | if v not in visited: 61 | visited.add(v) 62 | yield v.__name__, v.__file__, get_module_relative_component(v.__file__) 63 | yield from enumerate_modules(v, visited) 64 | # Process imports like 65 | # from module1 import module2 66 | # in this case we want to list module1 as a dependency as well 67 | try: 68 | source = inspect.getsource(module) 69 | except OSError: 70 | return 71 | module_ast = ast.parse(source) 72 | from_imports: list[ast.ImportFrom] = [ 73 | node for node in ast.walk(module_ast) 74 | if isinstance(node, ast.ImportFrom) 75 | ] 76 | relative_root = module.__name__ 77 | if hasattr(module, '__package__'): 78 | relative_root = module.__package__ 79 | for node in from_imports: 80 | if node.module is None: 81 | continue 82 | try: 83 | v = importlib.import_module( 84 | "."*node.level + node.module, relative_root) 85 | except ModuleNotFoundError: 86 | print("Module", node.module, "in", 87 | relative_root, "not found, ignored") 88 | continue 89 | except Exception as e: 90 | print("Module", node.module, "import failed due to", e, "ignored") 91 | continue 92 | if not hasattr(v, '__file__'): 93 | continue 94 | if is_stdlib_module(v): 95 | continue 96 | if v not in visited: 97 | visited.add(v) 98 | yield v.__name__, v.__file__, get_module_relative_component(v.__file__) 99 | yield from enumerate_modules(v, visited) 100 | 101 | 102 | def print_usage(argv): 103 | print("Usage:", argv[0], "some_python_script.py ") 104 | 105 | 106 | def main(argv): 107 | if len(argv) != 3: 108 | print_usage(argv) 109 | return 1 110 | path = argv[1] 111 | output_dir = argv[2] 112 | os.makedirs(output_dir, exist_ok=True) 113 | # spec = importlib.util.spec_from_file_location(path, path) 114 | # module = importlib.util.module_from_spec(spec) 115 | # spec.loader.exec_module(module) 116 | import web_server_flask as module 117 | for (name, path, relative_path) in enumerate_modules(module): 118 | target_path = os.path.join(output_dir, relative_path) 119 | os.makedirs(os.path.dirname(target_path), exist_ok=True) 120 | shutil.copy(path, target_path) 121 | print(name, path, relative_path) 122 | 123 | 124 | if __name__ == '__main__': 125 | sys.exit(main(sys.argv)) 126 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ota-generator", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "serve": "vue-cli-service serve", 7 | "build": "vue-cli-service build", 8 | "test:unit": "vue-cli-service test:unit", 9 | "lint": "vue-cli-service lint", 10 | "build_zip": "./build_zip.bash" 11 | }, 12 | "dependencies": { 13 | "@mdi/font": "5.9.55", 14 | "axios": "^0.25.0", 15 | "core-js": "^3.6.5", 16 | "eslint-config-airbnb-base": "^14.2.1", 17 | "jest": "^27.4.7", 18 | "material-design-icons": "^3.0.1", 19 | "roboto-fontface": "*", 20 | "vue": "^3.0.0-0", 21 | "vue-router": "^4.0.0-0", 22 | "vue-uuid": "^2.0.2", 23 | "vue3-table-lite": "^1.0.5", 24 | "vuetify": "^3.0.0-alpha.0", 25 | "vuex": "^4.0.0-0" 26 | }, 27 | "devDependencies": { 28 | "@vue/cli-plugin-babel": "^3.3.0", 29 | "@vue/cli-plugin-eslint": "^3.1.1", 30 | "@vue/cli-plugin-router": "3.12.1", 31 | "@vue/cli-plugin-unit-jest": "^3.12.1", 32 | "@vue/cli-plugin-vuex": "~4.5.0", 33 | "@vue/cli-service": "^3.12.1", 34 | "@vue/compiler-sfc": "^3.0.0-0", 35 | "@vue/eslint-config-prettier": "^6.0.0", 36 | "@vue/test-utils": "^2.0.0-0", 37 | "babel-eslint": "^10.1.0", 38 | "eslint": "^8.8.0", 39 | "eslint-plugin-prettier": "^3.1.3", 40 | "eslint-plugin-vue": "^7.0.0-0", 41 | "prettier": "^1.19.1", 42 | "sass": "~1.32.0", 43 | "sass-loader": "^10.0.0", 44 | "vue-cli-plugin-vuetify": "~2.4.1", 45 | "vue-jest": "^5.0.0-0" 46 | }, 47 | "eslintConfig": { 48 | "root": true, 49 | "env": { 50 | "node": true 51 | }, 52 | "extends": [ 53 | "plugin:vue/vue3-essential", 54 | "eslint:recommended", 55 | "@vue/prettier" 56 | ], 57 | "parserOptions": { 58 | "parser": "babel-eslint" 59 | }, 60 | "rules": {}, 61 | "overrides": [ 62 | { 63 | "files": [ 64 | "**/__tests__/*.{j,t}s?(x)", 65 | "**/tests/unit/**/*.spec.{j,t}s?(x)" 66 | ], 67 | "env": { 68 | "jest": true 69 | } 70 | } 71 | ] 72 | }, 73 | "browserslist": [ 74 | "> 1%", 75 | "last 2 versions", 76 | "not dead" 77 | ] 78 | } 79 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/ota-generator/afbb9d0d10e5cf59a5be7efa4177e6ec209b3b74/public/favicon.ico -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | <%= htmlWebpackPlugin.options.title %> 25 | 26 | 27 | 30 |
31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/App.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 42 | 43 | 52 | 53 | -------------------------------------------------------------------------------- /src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/ota-generator/afbb9d0d10e5cf59a5be7efa4177e6ec209b3b74/src/assets/logo.png -------------------------------------------------------------------------------- /src/components/BaseCheckbox.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 32 | 33 | 47 | 48 | -------------------------------------------------------------------------------- /src/components/BaseFile.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 37 | 38 | 74 | 75 | -------------------------------------------------------------------------------- /src/components/BaseInput.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 28 | 29 | 43 | 44 | -------------------------------------------------------------------------------- /src/components/BaseRadio.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 26 | 27 | -------------------------------------------------------------------------------- /src/components/BaseRadioGroup.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 28 | 29 | -------------------------------------------------------------------------------- /src/components/BaseSelect.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/components/BatchOTAOptions.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 43 | 44 | -------------------------------------------------------------------------------- /src/components/BuildLibrary.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 57 | 58 | 129 | 130 | -------------------------------------------------------------------------------- /src/components/BuildTable.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 28 | 29 | -------------------------------------------------------------------------------- /src/components/ChainOTAOptions.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 38 | 39 | -------------------------------------------------------------------------------- /src/components/FileList.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 76 | 77 | 135 | 136 | -------------------------------------------------------------------------------- /src/components/FileSelect.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 39 | 40 | 41 | 42 | 60 | 61 | -------------------------------------------------------------------------------- /src/components/JobConfiguration.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 38 | 39 | 61 | 62 | -------------------------------------------------------------------------------- /src/components/JobDisplay.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 32 | 33 | 52 | 53 | 67 | -------------------------------------------------------------------------------- /src/components/OTAJobTable.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 28 | 29 | -------------------------------------------------------------------------------- /src/components/OTAOptions.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 67 | 68 | -------------------------------------------------------------------------------- /src/components/PartialCheckbox.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 48 | 49 | 111 | 112 | -------------------------------------------------------------------------------- /src/components/SingleOTAOptions.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 43 | 44 | -------------------------------------------------------------------------------- /src/components/UploadFile.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 31 | 32 | 63 | 64 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createApp } from 'vue' 18 | import vuetify from './plugins/vuetify' 19 | import App from './App.vue' 20 | import router from './router' 21 | import store from './store' 22 | 23 | const app = createApp(App) 24 | app.use(router) 25 | app.use(store) 26 | app.use(vuetify) 27 | 28 | app.mount('#app') -------------------------------------------------------------------------------- /src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import '@mdi/font/css/materialdesignicons.css' 18 | import 'vuetify/lib/styles/main.sass' 19 | import { createVuetify } from 'vuetify' 20 | import * as components from 'vuetify/lib/components' 21 | import * as directives from 'vuetify/lib/directives' 22 | 23 | export default createVuetify({ 24 | components, 25 | directives, 26 | }) 27 | -------------------------------------------------------------------------------- /src/router/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createRouter, createWebHistory } from 'vue-router' 18 | import JobList from '@/views/JobList.vue' 19 | import JobDetails from '@/views/JobDetails.vue' 20 | import About from '@/views/About.vue' 21 | import JobConfigure from '@/views/JobConfigure.vue' 22 | import NotFound from '@/views/NotFound.vue' 23 | 24 | const routes = [ 25 | { 26 | path: '/', 27 | name: 'JobList', 28 | component: JobList 29 | }, 30 | { 31 | path: '/check-job/:id', 32 | name: 'JobDetails', 33 | props: true, 34 | component: JobDetails 35 | }, 36 | { 37 | path: '/about', 38 | name: 'About', 39 | component: About 40 | }, 41 | { 42 | path: '/create', 43 | name: 'Create', 44 | component: JobConfigure 45 | }, 46 | { 47 | path: '/:catchAll(.*)', 48 | name: 'Not Found', 49 | component: NotFound 50 | } 51 | ] 52 | 53 | const router = createRouter({ 54 | history: createWebHistory(process.env.BASE_URL), 55 | routes 56 | }) 57 | 58 | export default router 59 | -------------------------------------------------------------------------------- /src/services/ApiService.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import axios from 'axios' 18 | 19 | const baseURL = process.env.NODE_ENV === 'production' ? '' : 'http://localhost:5000'; 20 | 21 | console.log(`Build mode: ${process.env.NODE_ENV}, API base url ${baseURL}`); 22 | 23 | const apiClient = axios.create({ 24 | baseURL, 25 | withCredentials: false, 26 | headers: { 27 | Accept: 'application/json', 28 | 'Content-Type': 'application/json' 29 | } 30 | }); 31 | 32 | export default { 33 | getDownloadURLForJob(job) { 34 | return `${baseURL}/download/${job.output}`; 35 | }, 36 | getJobs() { 37 | return apiClient.get("/check") 38 | }, 39 | getJobById(id) { 40 | return apiClient.get("/check/" + id) 41 | }, 42 | async getBuildList() { 43 | let resp = await apiClient.get("/file"); 44 | return resp.data || []; 45 | }, 46 | async reconstructBuildList() { 47 | let resp = await apiClient.get("/reconstruct_build_list"); 48 | return resp.data; 49 | }, 50 | uploadTarget(file, onUploadProgress) { 51 | let formData = new FormData() 52 | formData.append('file', file) 53 | return apiClient.post("/file/" + file.name, 54 | formData, 55 | { 56 | onUploadProgress 57 | }) 58 | }, 59 | async postInput(input, id) { 60 | try { 61 | let resp = await apiClient.post( 62 | '/run/' + id, JSON.stringify(input)); 63 | return resp.data; 64 | } catch (error) { 65 | if (error.response.data) { 66 | return error.response.data; 67 | } else { 68 | throw error; 69 | } 70 | } 71 | } 72 | } -------------------------------------------------------------------------------- /src/services/FormDate.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export default{ 18 | formDate(unixTime) { 19 | let formTime = new Date(unixTime * 1000) 20 | let date = 21 | formTime.getFullYear() + 22 | '-' + 23 | (formTime.getMonth() + 1) + 24 | '-' + 25 | formTime.getDate() 26 | let time = 27 | formTime.getHours() + 28 | ':' + 29 | formTime.getMinutes() + 30 | ':' + 31 | formTime.getSeconds() 32 | return date + ' ' + time 33 | } 34 | } -------------------------------------------------------------------------------- /src/services/JobSubmission.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /** 18 | * @fileoverview Class OTAInput is used to configure and create a process in 19 | * the backend to to start OTA package generation. 20 | * @package vue-uuid 21 | * @package ApiServices 22 | */ 23 | import { uuid } from 'vue-uuid' 24 | import ApiServices from './ApiService.js' 25 | 26 | export class OTAConfiguration { 27 | /** 28 | * Initialize the input for the api /run/ 29 | */ 30 | constructor() { 31 | /** 32 | * Please refer to: 33 | * https://cs.android.com/android/platform/superproject/+/master:build/make/tools/releasetools/ota_from_target_files.py 34 | * for the complete and up-to-date configurations that can be set for 35 | * the OTA package generation. 36 | * TODO (lishutong): there are dependencies on this flags, 37 | * disable checkboxes of which dependencies are not fulfilled. 38 | */ 39 | this.verbose = false, 40 | this.isIncremental = false, 41 | this.partial = [], 42 | this.isPartial = false, 43 | this.extra_keys = [], 44 | this.extra = '' 45 | } 46 | 47 | /** 48 | * Take in multiple paths of target and incremental builds and generate 49 | * OTA packages between them. If there are n incremental sources and m target 50 | * builds, there will be n x m OTA packages in total. If there is 0 51 | * incremental package, full OTA will be generated. 52 | * @param {Array} targetBuilds 53 | * @param {Array} incrementalSources 54 | * @return Array 55 | */ 56 | async sendForms(targetBuilds, incrementalSources = []) { 57 | const responses = [] 58 | if (!this.isIncremental) { 59 | responses.push( 60 | ... await Promise.all( 61 | targetBuilds.map(async (target) => await this.sendForm(target)) 62 | ) 63 | ) 64 | } else { 65 | for (const incremental of incrementalSources) { 66 | responses.push( 67 | ... await Promise.all( 68 | targetBuilds.map( 69 | async (target) => await this.sendForm(target, incremental) 70 | ) 71 | ) 72 | ) 73 | } 74 | } 75 | return responses 76 | } 77 | 78 | /** 79 | * Take in an ordered list of target builds and generate OTA packages between 80 | * them in order. For example, if there are n target builds, there will be 81 | * n-1 OTA packages. 82 | * @param {Array} targetBuilds 83 | * @return Array 84 | */ 85 | async sendChainForms(targetBuilds) { 86 | const responses = [] 87 | this.isIncremental = true 88 | for (let i = 0; i < targetBuilds.length - 1; i++) { 89 | let response = 90 | await this.sendForm(targetBuilds[i + 1], targetBuilds[i]) 91 | responses.push(response) 92 | } 93 | return responses 94 | } 95 | 96 | /** 97 | * Start an OTA package generation from target build to incremental source. 98 | * Throw an error if not succeed, otherwise will return the message from 99 | * the backend. 100 | * @param {String} targetBuild 101 | * @param {String} incrementalSource 102 | * @return String 103 | */ 104 | async sendForm(targetBuild, incrementalSource = '') { 105 | let jsonOptions = Object.assign({}, this) 106 | jsonOptions.target = targetBuild 107 | jsonOptions.incremental = incrementalSource 108 | jsonOptions.isIncremental = !!incrementalSource; 109 | jsonOptions.id = uuid.v1() 110 | for (let flag of OTAExtraFlags) { 111 | if (jsonOptions[flag.key]) { 112 | if (jsonOptions.extra_keys.indexOf(flag.key) === -1) { 113 | jsonOptions.extra_keys.push(flag.key) 114 | } 115 | } 116 | } 117 | let data = await ApiServices.postInput(jsonOptions, jsonOptions.id) 118 | return data; 119 | } 120 | 121 | /** 122 | * Reset all the flags being set in this object. 123 | */ 124 | reset() { 125 | for (let flag of OTAExtraFlags) { 126 | if (this[flag.key]) { 127 | delete this[flag.key] 128 | } 129 | } 130 | this.constructor() 131 | } 132 | } 133 | 134 | export const OTABasicFlags = [ 135 | { 136 | key: 'isIncremental', 137 | label: 'Incremental OTA', 138 | requireArg: 'incremental' 139 | }, 140 | { 141 | key: 'isPartial', 142 | label: 'Partial OTA', 143 | requireArg: 'partial' 144 | }, 145 | { 146 | key: 'verbose', 147 | label: 'Verbose' 148 | },] 149 | 150 | export const OTAExtraFlags = [ 151 | { 152 | key: 'downgrade', 153 | label: 'Downgrade', 154 | depend: ['isIncremental'] 155 | }, 156 | { 157 | key: 'override_timestamp', 158 | label: 'Override time stamp' 159 | }, 160 | { 161 | key: 'wipe_user_data', 162 | label: 'Wipe User data' 163 | }, 164 | //{ 165 | // key: 'retrofit_dynamic_partitions', 166 | // label: 'Support dynamic partition' 167 | //}, 168 | { 169 | key: 'skip_compatibility_check', 170 | label: 'Skip compatibility check' 171 | }, 172 | //{ 173 | // key: 'output_metadata_path', 174 | // label: 'Output metadata path' 175 | //}, 176 | { 177 | key: 'force_non_ab', 178 | label: 'Generate non-A/B package' 179 | }, 180 | /** TODO(lishutong): the following comments are flags 181 | * that requires file operation, will add these functions later. 182 | */ 183 | //{key: 'oem_settings', label: 'Specify the OEM properties', 184 | // requireArg: 'oem_settings_files'}, 185 | //{key: 'binary', label: 'Use given binary', requireArg: 'binary_file', 186 | // depend: ['force_non_ab']}, 187 | { 188 | key: 'block', 189 | label: 'Block-based OTA', 190 | depend: ['force_non_ab'] 191 | }, 192 | //{key: 'extra_script', label: 'Extra script', requireArg: 'script_file', 193 | // depend: ['force_non_ab']}, 194 | { 195 | key: 'full_bootloader', 196 | label: 'Full bootloader', 197 | depend: ['force_non_ab', 'isIncremental'] 198 | }, 199 | { 200 | key: 'full_radio', 201 | label: 'Full radio', 202 | depend: ['force_non_ab', 'isIncremental'] 203 | }, 204 | //{key: 'log_diff', label: 'Log difference', 205 | // requireArg: 'log_diff_path',depend: ['force_non_ab', 'isIncremental']}, 206 | //{key: 'oem_no_mount', label: 'Do not mount OEM partition', 207 | // depend: ['force_non_ab', 'oem_settings']}, 208 | //{ 209 | // key: 'stash_threshold', 210 | // label: 'Threshold for maximum stash size', 211 | // requireArg: 'stash_threshold_float', 212 | // depend: ['force_non_ab'] 213 | //}, 214 | //{ 215 | // key: 'worker_threads', 216 | // label: 'Number of worker threads', 217 | // requireArg: 'worker_threads_int', 218 | // depend: ['force_non_ab', 'isIncremental'] 219 | //}, 220 | //{ 221 | // key: 'verify', 222 | // label: 'Verify the checksum', 223 | // depend: ['force_non_ab', 'isIncremental'] 224 | //}, 225 | { 226 | key: 'two_step', 227 | label: 'Generate two-step OTA', 228 | depend: ['force_non_ab'] 229 | }, 230 | { 231 | key: 'disable_fec_computation', 232 | label: 'Disable the on device FEC computation', 233 | depend: ['isIncremental'], 234 | exclude: ['force_non_ab'] 235 | }, 236 | { 237 | key: 'include_secondary', 238 | label: 'Include secondary slot images', 239 | exclude: ['force_non_ab', 'isIncremental'] 240 | }, 241 | //{key: 'payload_signer', label: 'Specify the signer', 242 | // requireArg: ['payload_signer_singer'], exclude: ['force_non_ab']}, 243 | //{key: 'payload_singer_args', label: 'Specify the args for signer', 244 | // requireArg: ['payload_signer_args_args], exclude: ['force_non_ab']}, 245 | //{ 246 | // key: 'payload_signer_maximum_signature_size', 247 | // label: 'The maximum signature size (in bytes)', 248 | // requireArg: ['payload_signer_maximum_signature_size_int'], 249 | // exclude: ['force_non_ab'] 250 | //}, 251 | //{key: 'boot_variable_file', label: 'Specify values of ro.boot.*', 252 | // requireArg: ['boot_variable_file_file'], exclude: ['force_non_ab']}, 253 | { 254 | key: 'skip_postinstall', 255 | label: 'Skip the postinstall', 256 | exclude: ['force_non_ab'] 257 | }, 258 | //{key: 'custom_image', label: 'Use custom image', 259 | // requireArg: ['custom_image_files'], exclude: ['force_non_ab]}, 260 | { 261 | key: 'disable_vabc', 262 | label: 'Disable Virtual A/B compression', 263 | exclude: ['force_non_ab'] 264 | }, 265 | { 266 | key: 'vabc_downgrade', 267 | label: "Don't disable VABC for downgrading", 268 | depend: ['isIncremental', 'downgrade'], 269 | exclude: ['force_non_ab'] 270 | }, 271 | ] 272 | 273 | /** export const requireArgs = new Map([ 274 | [ 275 | "stash_threshold_float", 276 | { 277 | type: "BaseInput", 278 | label: "Threshold for maximum stash size" 279 | } 280 | ], 281 | [ 282 | "worker_threads", 283 | { 284 | type: "BaseInput", 285 | label: "Number of worker threads" 286 | } 287 | ], 288 | [ 289 | "payload_signer_maximum_signature_size", 290 | { 291 | type: "BaseInput", 292 | label: "The maximum signature size (in bytes)" 293 | } 294 | ], 295 | ]) */ -------------------------------------------------------------------------------- /src/services/TableService.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export function TableSort(arr, key, sortOrder, offset, limit) { 18 | let orderNumber = 1 19 | if (sortOrder==="desc") { 20 | orderNumber = -1 21 | } 22 | return arr.sort(function(a, b) { 23 | var keyA = a[key], 24 | keyB = b[key]; 25 | if (keyA < keyB) return -orderNumber; 26 | if (keyA > keyB) return orderNumber; 27 | return 0; 28 | }).slice(offset, offset + limit); 29 | } -------------------------------------------------------------------------------- /src/store/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { createStore } from 'vuex' 18 | import { OTAConfiguration, OTAExtraFlags } from '@/services/JobSubmission.js' 19 | 20 | export default createStore({ 21 | state: { 22 | otaConfig: new OTAConfiguration(), 23 | targetBuilds: [], 24 | sourceBuilds: [] 25 | }, 26 | mutations: { 27 | REUSE_CONFIG(state, config) { 28 | state.otaConfig.verbose = config.verbose 29 | state.otaConfig.isIncremental = config.isIncremental 30 | state.otaConfig.partial = config.partial 31 | state.otaConfig.isPartial = config.isPartial 32 | state.targetBuilds = [config.target] 33 | if (config.isIncremental) 34 | state.sourceBuilds = [config.incremental] 35 | const extra = 36 | config.extra.split('--') 37 | .filter((s) => s!=='') 38 | .map((s) => s.trimRight()) 39 | extra.forEach( (key) => { 40 | if (OTAExtraFlags.filter((flags) => flags.key == key)) { 41 | state.otaConfig[key] = true 42 | } else { 43 | state.extra += key 44 | } 45 | }) 46 | }, 47 | SET_CONFIG(state, config) { 48 | state.otaConfig = config 49 | }, 50 | RESET_CONFIG(state) { 51 | state.otaConfig = new OTAConfiguration() 52 | }, 53 | SET_TARGET(state, target) { 54 | state.targetBuilds = [target] 55 | }, 56 | SET_SOURCE(state, target) { 57 | state.sourceBuilds = [target] 58 | }, 59 | SET_TARGETS(state, targets) { 60 | state.targetBuilds = targets 61 | }, 62 | SET_SOURCES(state, targets) { 63 | state.sourceBuilds = targets 64 | }, 65 | SET_ISINCREMENTAL(state, isIncremental) { 66 | state.otaConfig.isIncremental = isIncremental 67 | } 68 | }, 69 | actions: {}, 70 | modules: {} 71 | }) 72 | -------------------------------------------------------------------------------- /src/views/About.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 22 | -------------------------------------------------------------------------------- /src/views/JobConfigure.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 56 | 57 | 99 | 100 | > -------------------------------------------------------------------------------- /src/views/JobDetails.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 60 | 61 | 136 | 137 | -------------------------------------------------------------------------------- /src/views/JobList.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 29 | 30 | -------------------------------------------------------------------------------- /src/views/NotFound.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 25 | 26 | -------------------------------------------------------------------------------- /target_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from dataclasses import dataclass, asdict, field 16 | import sqlite3 17 | import time 18 | import logging 19 | import os 20 | import zipfile 21 | import re 22 | import json 23 | 24 | 25 | class BuildFileInvalidError(Exception): 26 | pass 27 | 28 | 29 | @dataclass 30 | class BuildInfo: 31 | """ 32 | A class for Android build information. 33 | """ 34 | file_name: str 35 | path: str 36 | time: int 37 | build_id: str = '' 38 | build_version: str = '' 39 | build_flavor: str = '' 40 | partitions: list[str] = field(default_factory=list) 41 | 42 | def analyse_buildprop(self): 43 | """ 44 | Analyse the build's version info and partitions included 45 | Then write them into the build_info 46 | """ 47 | def extract_info(pattern, lines): 48 | # Try to match a regex in a list of string 49 | line = list(filter(pattern.search, lines))[0] 50 | if line: 51 | return pattern.search(line).group(0) 52 | else: 53 | return '' 54 | 55 | with zipfile.ZipFile(self.path) as build: 56 | try: 57 | with build.open('SYSTEM/build.prop', 'r') as build_prop: 58 | raw_info = build_prop.readlines() 59 | pattern_id = re.compile(b'(?<=ro\.build\.id\=).+') 60 | pattern_version = re.compile( 61 | b'(?<=ro\.build\.version\.incremental\=).+') 62 | pattern_flavor = re.compile(b'(?<=ro\.build\.flavor\=).+') 63 | self.build_id = extract_info( 64 | pattern_id, raw_info).decode('utf-8') 65 | self.build_version = extract_info( 66 | pattern_version, raw_info).decode('utf-8') 67 | self.build_flavor = extract_info( 68 | pattern_flavor, raw_info).decode('utf-8') 69 | with build.open('META/ab_partitions.txt', 'r') as partition_info: 70 | raw_info = partition_info.readlines() 71 | for line in raw_info: 72 | self.partitions.append(line.decode('utf-8').rstrip()) 73 | except KeyError as e: 74 | raise BuildFileInvalidError("Invalid build due to " + str(e)) 75 | 76 | def to_sql_form_dict(self): 77 | """ 78 | Because sqlite can only store text but self.partitions is a list 79 | Turn the list into a string joined by ',', for example: 80 | ['system', 'vendor'] => 'system,vendor' 81 | """ 82 | sql_form_dict = asdict(self) 83 | sql_form_dict['partitions'] = ','.join(sql_form_dict['partitions']) 84 | return sql_form_dict 85 | 86 | def to_dict(self): 87 | """ 88 | Return as a normal dict. 89 | """ 90 | return asdict(self) 91 | 92 | 93 | class TargetLib: 94 | """ 95 | A class that manages the builds in database. 96 | """ 97 | 98 | def __init__(self, working_dir="target", db_path=None): 99 | """ 100 | Create a build table if not existing 101 | """ 102 | self.working_dir = working_dir 103 | if db_path is None: 104 | db_path = os.path.join(working_dir, "ota_database.db") 105 | self.db_path = db_path 106 | with sqlite3.connect(self.db_path) as connect: 107 | cursor = connect.cursor() 108 | cursor.execute(""" 109 | CREATE TABLE if not exists Builds ( 110 | FileName TEXT, 111 | UploadTime INTEGER, 112 | Path TEXT, 113 | BuildID TEXT, 114 | BuildVersion TEXT, 115 | BuildFlavor TEXT, 116 | Partitions TEXT 117 | ) 118 | """) 119 | 120 | def new_build(self, filename, path): 121 | """ 122 | Insert a new build into the database 123 | Args: 124 | filename: the name of the file 125 | path: the relative path of the file 126 | """ 127 | build_info = BuildInfo(filename, path, int(time.time())) 128 | build_info.analyse_buildprop() 129 | # Ignore name specified by user, instead use a standard format 130 | build_info.path = os.path.join(self.working_dir, "{}-{}-{}.zip".format( 131 | build_info.build_flavor, build_info.build_id, build_info.build_version)) 132 | if path != build_info.path: 133 | os.rename(path, build_info.path) 134 | with sqlite3.connect(self.db_path) as connect: 135 | cursor = connect.cursor() 136 | cursor.execute(""" 137 | SELECT * FROM Builds WHERE FileName=:file_name and Path=:path 138 | """, build_info.to_sql_form_dict()) 139 | if cursor.fetchall(): 140 | cursor.execute(""" 141 | DELETE FROM Builds WHERE FileName=:file_name and Path=:path 142 | """, build_info.to_sql_form_dict()) 143 | cursor.execute(""" 144 | INSERT INTO Builds (FileName, UploadTime, Path, BuildID, BuildVersion, BuildFlavor, Partitions) 145 | VALUES (:file_name, :time, :path, :build_id, :build_version, :build_flavor, :partitions) 146 | """, build_info.to_sql_form_dict()) 147 | 148 | def new_build_from_dir(self): 149 | """ 150 | Update the database using files under a directory 151 | Args: 152 | path: a directory 153 | """ 154 | build_dir = self.working_dir 155 | if os.path.isdir(build_dir): 156 | builds_name = os.listdir(build_dir) 157 | for build_name in builds_name: 158 | path = os.path.join(build_dir, build_name) 159 | if build_name.endswith(".zip") and zipfile.is_zipfile(path): 160 | self.new_build(build_name, path) 161 | elif os.path.isfile(build_dir) and build_dir.endswith(".zip"): 162 | self.new_build(os.path.split(build_dir)[-1], build_dir) 163 | return self.get_builds() 164 | 165 | def sql_to_buildinfo(self, row): 166 | build_info = BuildInfo(*row[:6], row[6].split(',')) 167 | return build_info 168 | 169 | def get_builds(self): 170 | """ 171 | Get a list of builds in the database 172 | Return: 173 | A list of build_info, each of which is an object: 174 | (FileName, UploadTime, Path, Build ID, Build Version, Build Flavor, Partitions) 175 | """ 176 | with sqlite3.connect(self.db_path) as connect: 177 | cursor = connect.cursor() 178 | cursor.execute(""" 179 | SELECT FileName, Path, UploadTime, BuildID, BuildVersion, BuildFlavor, Partitions 180 | FROM Builds""") 181 | return list(map(self.sql_to_buildinfo, cursor.fetchall())) 182 | 183 | def get_build_by_path(self, path): 184 | """ 185 | Get a build in the database by its path 186 | Return: 187 | A build_info, which is an object: 188 | (FileName, UploadTime, Path, Build ID, Build Version, Build Flavor, Partitions) 189 | """ 190 | with sqlite3.connect(self.db_path) as connect: 191 | cursor = connect.cursor() 192 | cursor.execute(""" 193 | SELECT FileName, Path, UploadTime, BuildID, BuildVersion, BuildFlavor, Partitions 194 | FROM Builds WHERE Path==(?) 195 | """, (path, )) 196 | return self.sql_to_buildinfo(cursor.fetchone()) 197 | -------------------------------------------------------------------------------- /test/test_ab_partitions.txt: -------------------------------------------------------------------------------- 1 | system 2 | vendor -------------------------------------------------------------------------------- /test/test_build.prop: -------------------------------------------------------------------------------- 1 | ro.build.id=AOSP.MASTER 2 | ro.build.display.id=aosp_cf_x86_64_phone-userdebug S AOSP.MASTER 7392671 test-keys 3 | ro.build.version.incremental=7392671 4 | ro.build.version.sdk=30 5 | ro.build.version.preview_sdk=1 6 | ro.build.version.preview_sdk_fingerprint=5f1ee022916302ff92d66186575d0b95 7 | ro.build.version.codename=S 8 | ro.build.version.all_codenames=S 9 | ro.build.version.release=11 10 | ro.build.version.release_or_codename=S 11 | ro.build.version.security_patch=2021-05-05 12 | ro.build.version.base_os= 13 | ro.build.version.min_supported_target_sdk=23 14 | ro.build.date=Mon May 24 08:56:05 UTC 2021 15 | ro.build.date.utc=1621846565 16 | ro.build.type=userdebug 17 | ro.build.user=android-build 18 | ro.build.host=abfarm069 19 | ro.build.tags=test-keys 20 | ro.build.flavor=aosp_cf_x86_64_phone-userdebug 21 | ro.build.system_root_image=false -------------------------------------------------------------------------------- /test_ota_interface.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | from ota_interface import JobInfo, ProcessesManagement 17 | from unittest.mock import patch, mock_open, Mock, MagicMock 18 | import os 19 | import sqlite3 20 | import copy 21 | 22 | class TestJobInfo(unittest.TestCase): 23 | def setUp(self): 24 | self.test_id = '286feab0-f16b-11eb-a72a-f7b1de0921ef' 25 | self.test_target = 'target/build.zip' 26 | self.test_verbose = True 27 | self.test_status = 'Running' 28 | self.test_extra = '--downgrade' 29 | self.test_start_time = 1628698830 30 | self.test_finish_time = 1628698831 31 | 32 | def setup_job(self, incremental = '', partial = [], 33 | output = '', stdout = '', stderr = ''): 34 | job_info = JobInfo(id=self.test_id, 35 | target=self.test_target, 36 | incremental=incremental, 37 | verbose=self.test_verbose, 38 | partial=partial, 39 | output=output, 40 | status=self.test_status, 41 | extra=self.test_extra, 42 | start_time=self.test_start_time, 43 | finish_time=self.test_finish_time, 44 | stderr=stderr, 45 | stdout=stdout 46 | ) 47 | return job_info 48 | 49 | def test_init(self): 50 | # No incremental, no output, no stdout/stderr set. 51 | job_info1 = self.setup_job() 52 | for key, value in self.__dict__.items(): 53 | if key.startswith('test_'): 54 | self.assertEqual(job_info1.__dict__[key[5:]], value, 55 | 'The value of ' + key + 'is not initialized correctly' 56 | ) 57 | self.assertEqual(job_info1.output, 'output/'+self.test_id+'.zip', 58 | 'Default output cannot be setup correctly' 59 | ) 60 | self.assertEqual(job_info1.stderr, 'output/stderr.'+self.test_id, 61 | 'Default stderr cannot be setup correctly' 62 | ) 63 | self.assertEqual(job_info1.stdout, 'output/stdout.'+self.test_id, 64 | 'Default stdout cannot be setup correctly' 65 | ) 66 | # Test the incremental setup 67 | job_info2 = self.setup_job(incremental='target/source.zip') 68 | self.assertEqual(job_info2.incremental, 'target/source.zip', 69 | 'incremental source cannot be initialized correctly' 70 | ) 71 | self.assertTrue(job_info2.isIncremental, 72 | 'incremental status cannot be initialized correctly' 73 | ) 74 | # Test the stdout/stderr setup 75 | job_info3 = self.setup_job(stderr='output/stderr', 76 | stdout='output/stdout' 77 | ) 78 | self.assertEqual(job_info3.stderr, 'output/stderr', 79 | 'the stderr cannot be setup manually' 80 | ) 81 | self.assertEqual(job_info3.stdout, 'output/stdout', 82 | 'the stdout cannot be setup manually' 83 | ) 84 | # Test the output setup 85 | job_info4 = self.setup_job(output='output/output.zip') 86 | self.assertEqual(job_info4.output, 'output/output.zip', 87 | 'output cannot be setup manually' 88 | ) 89 | # Test the partial setup 90 | job_info5 = self.setup_job(partial=['system', 'vendor']) 91 | self.assertEqual(job_info5.partial, ['system', 'vendor'], 92 | 'partial list cannot be setup correctly' 93 | ) 94 | 95 | def test_to_sql_form_dict(self): 96 | partial_list = ['system', 'vendor'] 97 | job_info = self.setup_job(partial=partial_list) 98 | sql_dict = job_info.to_sql_form_dict() 99 | test_dict = {'id': self.test_id, 100 | 'target': self.test_target, 101 | 'incremental': '', 102 | 'verbose': int(self.test_verbose), 103 | 'partial': ','.join(partial_list), 104 | 'output': 'output/' + self.test_id + '.zip', 105 | 'status': self.test_status, 106 | 'extra': self.test_extra, 107 | 'stdout': 'output/stdout.' + self.test_id, 108 | 'stderr': 'output/stderr.' + self.test_id, 109 | 'start_time': self.test_start_time, 110 | 'finish_time': self.test_finish_time, 111 | 'isPartial': True, 112 | 'isIncremental': False 113 | } 114 | for key, value in test_dict.items(): 115 | self.assertEqual(value, sql_dict[key], 116 | 'the ' + key + ' is not converted to sql form dict correctly' 117 | ) 118 | 119 | def test_to_dict_basic(self): 120 | partial_list = ['system', 'vendor'] 121 | job_info = self.setup_job(partial=partial_list) 122 | basic_dict = job_info.to_dict_basic() 123 | test_dict = {'id': self.test_id, 124 | 'target': self.test_target, 125 | 'target_name': self.test_target.split('/')[-1], 126 | 'incremental': '', 127 | 'verbose': int(self.test_verbose), 128 | 'partial': partial_list, 129 | 'output': 'output/' + self.test_id + '.zip', 130 | 'status': self.test_status, 131 | 'extra': self.test_extra, 132 | 'stdout': 'output/stdout.' + self.test_id, 133 | 'stderr': 'output/stderr.' + self.test_id, 134 | 'start_time': self.test_start_time, 135 | 'finish_time': self.test_finish_time, 136 | 'isPartial': True, 137 | 'isIncremental': False 138 | } 139 | for key, value in test_dict.items(): 140 | self.assertEqual(value, basic_dict[key], 141 | 'the ' + key + ' is not converted to basic form dict correctly' 142 | ) 143 | 144 | def test_to_dict_detail(self): 145 | partial_list = ['system', 'vendor'] 146 | test_incremental = 'target/source.zip' 147 | job_info = self.setup_job(partial=partial_list, incremental=test_incremental) 148 | mock_target_lib = Mock() 149 | mock_target_lib.get_build_by_path = Mock( 150 | side_effect = [ 151 | Mock(file_name='build.zip', build_version=''), 152 | Mock(file_name='source.zip', build_version='') 153 | ] 154 | ) 155 | test_dict = {'id': self.test_id, 156 | 'target': self.test_target, 157 | 'incremental': 'target/source.zip', 158 | 'verbose': int(self.test_verbose), 159 | 'partial': partial_list, 160 | 'output': 'output/' + self.test_id + '.zip', 161 | 'status': self.test_status, 162 | 'extra': self.test_extra, 163 | 'stdout': 'NO STD OUTPUT IS FOUND', 164 | 'stderr': 'NO STD ERROR IS FOUND', 165 | 'start_time': self.test_start_time, 166 | 'finish_time': self.test_finish_time, 167 | 'isPartial': True, 168 | 'isIncremental': True 169 | } 170 | # Test with no stdout and stderr 171 | dict_detail = job_info.to_dict_detail(mock_target_lib) 172 | mock_target_lib.get_build_by_path.assert_any_call(self.test_target) 173 | mock_target_lib.get_build_by_path.assert_any_call(test_incremental) 174 | for key, value in test_dict.items(): 175 | self.assertEqual(value, dict_detail[key], 176 | 'the ' + key + ' is not converted to detailed dict correctly' 177 | ) 178 | # Test with mocked stdout and stderr 179 | mock_target_lib.get_build_by_path = Mock( 180 | side_effect = [ 181 | Mock(file_name='build.zip', build_version=''), 182 | Mock(file_name='source.zip', build_version='') 183 | ] 184 | ) 185 | mock_file = mock_open(read_data="mock output") 186 | with patch("builtins.open", mock_file): 187 | dict_detail = job_info.to_dict_detail(mock_target_lib) 188 | test_dict['stderr'] = 'mock output' 189 | test_dict['stdout'] = 'mock output' 190 | for key, value in test_dict.items(): 191 | self.assertEqual(value, dict_detail[key], 192 | 'the ' + key + ' is not converted to detailed dict correctly' 193 | ) 194 | 195 | class TestProcessesManagement(unittest.TestCase): 196 | def setUp(self): 197 | if os.path.isfile('test_process.db'): 198 | self.tearDown() 199 | self.processes = ProcessesManagement(db_path='test_process.db') 200 | testcase_job_info = TestJobInfo() 201 | testcase_job_info.setUp() 202 | self.test_job_info = testcase_job_info.setup_job(incremental='target/source.zip') 203 | self.processes.insert_database(self.test_job_info) 204 | 205 | def tearDown(self): 206 | os.remove('test_process.db') 207 | try: 208 | os.remove('output/stderr.'+self.test_job_info.id) 209 | os.remove('output/stdout.'+self.test_job_info.id) 210 | except FileNotFoundError: 211 | pass 212 | 213 | def test_init(self): 214 | # Test the database is created successfully 215 | self.assertTrue(os.path.isfile('test_process.db')) 216 | test_columns = [ 217 | {'name': 'ID','type':'TEXT'}, 218 | {'name': 'TargetPath','type':'TEXT'}, 219 | {'name': 'IncrementalPath','type':'TEXT'}, 220 | {'name': 'Verbose','type':'INTEGER'}, 221 | {'name': 'Partial','type':'TEXT'}, 222 | {'name': 'OutputPath','type':'TEXT'}, 223 | {'name': 'Status','type':'TEXT'}, 224 | {'name': 'Downgrade','type':'INTEGER'}, 225 | {'name': 'OtherFlags','type':'TEXT'}, 226 | {'name': 'STDOUT','type':'TEXT'}, 227 | {'name': 'STDERR','type':'TEXT'}, 228 | {'name': 'StartTime','type':'INTEGER'}, 229 | {'name': 'FinishTime','type':'INTEGER'}, 230 | ] 231 | connect = sqlite3.connect('test_process.db') 232 | cursor = connect.cursor() 233 | cursor.execute("PRAGMA table_info(jobs)") 234 | columns = cursor.fetchall() 235 | for column in test_columns: 236 | column_found = list(filter(lambda x: x[1]==column['name'], columns)) 237 | self.assertEqual(len(column_found), 1, 238 | 'The column ' + column['name'] + ' is not found in database' 239 | ) 240 | self.assertEqual(column_found[0][2], column['type'], 241 | 'The column' + column['name'] + ' has a wrong type' 242 | ) 243 | 244 | def test_get_status_by_ID(self): 245 | job_info = self.processes.get_status_by_ID(self.test_job_info.id) 246 | self.assertEqual(job_info, self.test_job_info, 247 | 'The data read from database is not the same one as inserted' 248 | ) 249 | 250 | def test_get_status(self): 251 | # Insert the same info again, but change the last digit of id to 0 252 | test_job_info2 = copy.copy(self.test_job_info) 253 | test_job_info2.id = test_job_info2.id[:-1] + '0' 254 | self.processes.insert_database(test_job_info2) 255 | job_infos = self.processes.get_status() 256 | self.assertEqual(len(job_infos), 2, 257 | 'The number of data entries is not the same as created' 258 | ) 259 | self.assertEqual(job_infos[0], self.test_job_info, 260 | 'The data list read from database is not the same one as inserted' 261 | ) 262 | self.assertEqual(job_infos[1], test_job_info2, 263 | 'The data list read from database is not the same one as inserted' 264 | ) 265 | 266 | def test_ota_run(self): 267 | # Test when the job exit normally 268 | mock_proc = Mock() 269 | mock_proc.wait = Mock(return_value=0) 270 | mock_Popen = Mock(return_value=mock_proc) 271 | test_command = [ 272 | "ota_from_target_files", "-v","build/target.zip", "output/ota.zip", 273 | ] 274 | mock_pipes_template = Mock() 275 | mock_pipes_template.open = Mock() 276 | mock_Template = Mock(return_value=mock_pipes_template) 277 | # Mock the subprocess.Popen, subprocess.Popen().wait and pipes.Template 278 | with patch("subprocess.Popen", mock_Popen), \ 279 | patch("pipes.Template", mock_Template): 280 | self.processes.ota_run(test_command, self.test_job_info.id) 281 | mock_Popen.assert_called_once() 282 | mock_proc.wait.assert_called_once() 283 | job_info = self.processes.get_status_by_ID(self.test_job_info.id) 284 | self.assertEqual(job_info.status, 'Finished') 285 | mock_Popen.reset_mock() 286 | mock_proc.wait.reset_mock() 287 | # Test when the job exit with prbolems 288 | mock_proc.wait = Mock(return_value=1) 289 | with patch("subprocess.Popen", mock_Popen), \ 290 | patch("pipes.Template", mock_Template): 291 | self.processes.ota_run(test_command, self.test_job_info.id) 292 | mock_Popen.assert_called_once() 293 | mock_proc.wait.assert_called_once() 294 | job_info = self.processes.get_status_by_ID(self.test_job_info.id) 295 | self.assertEqual(job_info.status, 'Error') 296 | 297 | def test_ota_generate(self): 298 | test_args = dict({ 299 | 'output': 'ota.zip', 300 | 'extra_keys': ['downgrade', 'wipe_user_data'], 301 | 'extra': '--disable_vabc', 302 | 'isIncremental': True, 303 | 'isPartial': True, 304 | 'partial': ['system', 'vendor'], 305 | 'incremental': 'target/source.zip', 306 | 'target': 'target/build.zip', 307 | 'verbose': True 308 | }) 309 | # Usually the order of commands make no difference, but the following 310 | # order has been validated, so it is best to follow this manner: 311 | # ota_from_target_files [flags like -v, --downgrade] 312 | # [-i incremental_source] [-p partial_list] target output 313 | test_command = [ 314 | 'ota_from_target_files', '-v', '--downgrade', 315 | '--wipe_user_data', '--disable_vabc', '-k', 316 | 'build/make/target/product/security/testkey', 317 | '-i', 'target/source.zip', 318 | '--partial', 'system vendor', 'target/build.zip', 'ota.zip' 319 | ] 320 | mock_os_path_isfile = Mock(return_value=True) 321 | mock_threading = Mock() 322 | mock_thread = Mock(return_value=mock_threading) 323 | with patch("os.path.isfile", mock_os_path_isfile), \ 324 | patch("threading.Thread", mock_thread): 325 | self.processes.ota_generate(test_args, id='test') 326 | job_info = self.processes.get_status_by_ID('test') 327 | self.assertEqual(job_info.status, 'Running', 328 | 'The job cannot be stored into database properly' 329 | ) 330 | # Test if the job stored into database properly 331 | for key, value in test_args.items(): 332 | # extra_keys is merged to extra when stored into database 333 | if key=='extra_keys': 334 | continue 335 | self.assertEqual(job_info.__dict__[key], value, 336 | 'The column ' + key + ' is not stored into database properly' 337 | ) 338 | # Test if the command is in its order 339 | self.assertEqual(mock_thread.call_args[1]['args'][0], test_command, 340 | 'The subprocess command is not in its good shape' 341 | ) 342 | 343 | if __name__ == '__main__': 344 | unittest.main() -------------------------------------------------------------------------------- /test_suite.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os, unittest 16 | 17 | class Tests(): 18 | def suite(self): 19 | modules_to_test = [] 20 | test_dir = os.listdir('.') 21 | for test in test_dir: 22 | if test.startswith('test') and test.endswith('.py'): 23 | modules_to_test.append(test.rstrip('.py')) 24 | 25 | alltests = unittest.TestSuite() 26 | for module in map(__import__, modules_to_test): 27 | alltests.addTest(unittest.findTestCases(module)) 28 | return alltests 29 | 30 | if __name__ == '__main__': 31 | MyTests = Tests() 32 | unittest.main(defaultTest='MyTests.suite', verbosity=2) -------------------------------------------------------------------------------- /test_target_lib.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import unittest 16 | from unittest.mock import patch, mock_open, Mock, MagicMock 17 | from target_lib import BuildInfo, TargetLib 18 | import zipfile 19 | import os 20 | import sqlite3 21 | from tempfile import NamedTemporaryFile 22 | 23 | class CreateTestBuild(): 24 | def __init__(self, include_build_prop=True, include_ab_partitions=True): 25 | self.file = NamedTemporaryFile(dir='test/') 26 | self.test_path = self.file.name 27 | with zipfile.ZipFile(self.file, mode='w') as package: 28 | if include_build_prop: 29 | package.write('test/test_build.prop', 'SYSTEM/build.prop') 30 | if include_ab_partitions: 31 | package.write('test/test_ab_partitions.txt', 32 | 'META/ab_partitions.txt') 33 | self.test_info = { 34 | 'file_name': self.test_path.split('/')[-1], 35 | 'path': self.test_path, 36 | 'time': 1628698830, 37 | 'build_id': 'AOSP.MASTER', 38 | 'build_version': '7392671', 39 | 'build_flavor': 'aosp_cf_x86_64_phone-userdebug', 40 | 'partitions': ['system', 'vendor'], 41 | } 42 | if not include_build_prop: 43 | self.test_info['build_id'] = '' 44 | self.test_info['build_version'] = '' 45 | self.test_info['build_flavor'] = '' 46 | if not include_ab_partitions: 47 | self.test_info['partitions'] = [] 48 | 49 | def clean(self): 50 | self.file.close() 51 | 52 | class TestBuildInfo(unittest.TestCase): 53 | def setUp(self): 54 | """ 55 | Create a virtual Android build, which only have build.prop 56 | and ab_partitions.txt 57 | """ 58 | self.test_build = CreateTestBuild() 59 | self.build_info = BuildInfo( 60 | self.test_build.test_info['file_name'], 61 | self.test_build.test_info['path'], 62 | self.test_build.test_info['time'] 63 | ) 64 | self.build_info.analyse_buildprop() 65 | 66 | def tearDown(self): 67 | self.test_build.clean() 68 | 69 | def test_analyse_buildprop(self): 70 | # Test if the build.prop and ab_partitions are not empty 71 | for key, value in self.test_build.test_info.items(): 72 | self.assertEqual(value, self.build_info.__dict__[key], 73 | 'The ' + key + ' is not parsed correctly.' 74 | ) 75 | # Test if the ab_partitions is empty 76 | test_build_no_partitions = CreateTestBuild(include_ab_partitions=False) 77 | build_info = BuildInfo( 78 | test_build_no_partitions.test_info['file_name'], 79 | test_build_no_partitions.test_info['path'], 80 | test_build_no_partitions.test_info['time'] 81 | ) 82 | build_info.analyse_buildprop() 83 | self.assertEqual(build_info.partitions, 84 | test_build_no_partitions.test_info['partitions'], 85 | 'The partition list is not empty if ab_partitions is not provided.' 86 | ) 87 | test_build_no_partitions.clean() 88 | 89 | def test_to_sql_form_dict(self): 90 | sql_dict = self.build_info.to_sql_form_dict() 91 | for key, value in self.test_build.test_info.items(): 92 | if key != 'partitions': 93 | self.assertEqual(value, sql_dict[key], 94 | 'The ' + key + ' is not parsed correctly.' 95 | ) 96 | else: 97 | self.assertEqual(','.join(value), sql_dict[key], 98 | 'The partition list is not coverted to sql form properly.' 99 | ) 100 | 101 | def test_to_dict(self): 102 | ordinary_dict = self.build_info.to_dict() 103 | for key, value in self.test_build.test_info.items(): 104 | self.assertEqual(value, ordinary_dict[key], 105 | 'The ' + key + ' is not parsed correctly.' 106 | ) 107 | 108 | 109 | class TestTargetLib(unittest.TestCase): 110 | def setUp(self): 111 | self.test_path = 'test/test_target_lib.db' 112 | self.tearDown() 113 | self.target_build = TargetLib(path=self.test_path) 114 | 115 | def tearDown(self): 116 | if os.path.isfile(self.test_path): 117 | os.remove(self.test_path) 118 | 119 | def test_init(self): 120 | # Test the database is created successfully 121 | self.assertTrue(os.path.isfile(self.test_path)) 122 | test_columns = [ 123 | {'name': 'FileName','type':'TEXT'}, 124 | {'name': 'Path','type':'TEXT'}, 125 | {'name': 'BuildID','type':'TEXT'}, 126 | {'name': 'BuildVersion','type':'TEXT'}, 127 | {'name': 'BuildFlavor','type':'TEXT'}, 128 | {'name': 'Partitions','type':'TEXT'}, 129 | {'name': 'UploadTime','type':'INTEGER'}, 130 | ] 131 | connect = sqlite3.connect(self.test_path) 132 | cursor = connect.cursor() 133 | cursor.execute("PRAGMA table_info(Builds)") 134 | columns = cursor.fetchall() 135 | for column in test_columns: 136 | column_found = list(filter(lambda x: x[1]==column['name'], columns)) 137 | self.assertEqual(len(column_found), 1, 138 | 'The column ' + column['name'] + ' is not found in database' 139 | ) 140 | self.assertEqual(column_found[0][2], column['type'], 141 | 'The column ' + column['name'] + ' has a wrong type' 142 | ) 143 | 144 | def test_new_build(self): 145 | test_build = CreateTestBuild() 146 | self.target_build.new_build( 147 | filename=test_build.test_info['file_name'], 148 | path=test_build.test_path 149 | ) 150 | connect = sqlite3.connect(self.test_path) 151 | cursor = connect.cursor() 152 | cursor.execute("SELECT * FROM BUILDS") 153 | entries = cursor.fetchall() 154 | self.assertEqual(len(entries), 1, 155 | 'The test build cannot be added into the database.' 156 | ) 157 | test_build.clean() 158 | 159 | def test_get_builds(self): 160 | test_build = CreateTestBuild() 161 | # time.time() has to be mocked, otherwise it will be the current time 162 | mock_time = Mock(return_value=test_build.test_info['time']) 163 | with patch('time.time', mock_time): 164 | self.target_build.new_build( 165 | filename=test_build.test_info['file_name'], 166 | path=test_build.test_path 167 | ) 168 | # build_list is read and parsed from the database 169 | # build_info is directly parsed from the package 170 | # Test if the read/write database process changes the data entry 171 | build_list = self.target_build.get_builds() 172 | build_info = BuildInfo( 173 | test_build.test_info['file_name'], 174 | test_build.test_info['path'], 175 | test_build.test_info['time'] 176 | ) 177 | build_info.analyse_buildprop() 178 | self.assertEqual(build_list[0], build_info, 179 | 'The list of build info cannot be extracted from database.' 180 | ) 181 | test_build.clean() 182 | 183 | def test_get_build_by_path(self): 184 | test_build = CreateTestBuild() 185 | # time.time() has to be mocked, otherwise it will be the current time 186 | mock_time = Mock(return_value=test_build.test_info['time']) 187 | with patch('time.time', mock_time): 188 | self.target_build.new_build( 189 | filename=test_build.test_info['file_name'], 190 | path=test_build.test_path 191 | ) 192 | build = self.target_build.get_build_by_path(test_build.test_info['path']) 193 | build_info = BuildInfo( 194 | test_build.test_info['file_name'], 195 | test_build.test_info['path'], 196 | test_build.test_info['time'] 197 | ) 198 | build_info.analyse_buildprop() 199 | self.assertEqual(build, build_info, 200 | 'Build info cannot be extracted by path.' 201 | ) 202 | test_build.clean() 203 | 204 | 205 | if __name__ == '__main__': 206 | unittest.main() 207 | -------------------------------------------------------------------------------- /vue.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const path = require('path') 18 | 19 | module.exports = { 20 | configureWebpack: { 21 | resolve: { 22 | symlinks: false, 23 | alias: { 24 | vue: path.resolve('./node_modules/vue') 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /web_server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """ 16 | A local HTTP server for Android OTA package generation. 17 | Based on OTA_from_target_files.py 18 | 19 | Usage:: 20 | python ./web_server.py [] 21 | 22 | API:: 23 | GET /check : check the status of all jobs 24 | GET /check/ : check the status of the job with 25 | GET /file : fetch the target file list 26 | GET /file/ : Add build file(s) in , and return the target file list 27 | GET /download/ : download the ota package with 28 | POST /run/ : submit a job with , 29 | arguments set in a json uploaded together 30 | POST /file/ : upload a target file 31 | [TODO] POST /cancel/ : cancel a job with 32 | 33 | TODO: 34 | - Avoid unintentionally path leakage 35 | - Avoid overwriting build when uploading build with same file name 36 | 37 | Other GET request will be redirected to the static request under 'dist' directory 38 | """ 39 | 40 | from http.server import BaseHTTPRequestHandler, SimpleHTTPRequestHandler, HTTPServer 41 | from socketserver import ThreadingMixIn 42 | from threading import Lock 43 | from ota_interface import ProcessesManagement 44 | from target_lib import TargetLib 45 | import logging 46 | import json 47 | import cgi 48 | import os 49 | import stat 50 | import zipfile 51 | 52 | LOCAL_ADDRESS = '0.0.0.0' 53 | 54 | 55 | class CORSSimpleHTTPHandler(SimpleHTTPRequestHandler): 56 | def end_headers(self): 57 | try: 58 | origin_address, _ = cgi.parse_header(self.headers['Origin']) 59 | self.send_header('Access-Control-Allow-Credentials', 'true') 60 | self.send_header('Access-Control-Allow-Origin', origin_address) 61 | except TypeError: 62 | pass 63 | super().end_headers() 64 | 65 | 66 | class RequestHandler(CORSSimpleHTTPHandler): 67 | def _set_response(self, code=200, type='text/html'): 68 | self.send_response(code) 69 | self.send_header('Content-type', type) 70 | self.end_headers() 71 | 72 | def do_OPTIONS(self): 73 | self.send_response(200) 74 | self.send_header('Access-Control-Allow-Methods', 'GET, OPTIONS') 75 | self.send_header("Access-Control-Allow-Headers", "X-Requested-With") 76 | self.send_header("Access-Control-Allow-Headers", "Content-Type") 77 | self.end_headers() 78 | 79 | def do_GET(self): 80 | if self.path == '/check' or self.path == '/check/': 81 | statuses = jobs.get_status() 82 | self._set_response(type='application/json') 83 | self.wfile.write( 84 | json.dumps([status.to_dict_basic() 85 | for status in statuses]).encode() 86 | ) 87 | elif self.path.startswith('/check/'): 88 | id = self.path[7:] 89 | status = jobs.get_status_by_ID(id=id) 90 | self._set_response(type='application/json') 91 | self.wfile.write( 92 | json.dumps(status.to_dict_detail(target_lib)).encode() 93 | ) 94 | elif self.path.startswith('/file') or self.path.startswith("/reconstruct_build_list"): 95 | if self.path == '/file' or self.path == '/file/': 96 | file_list = target_lib.get_builds() 97 | else: 98 | file_list = target_lib.new_build_from_dir() 99 | builds_info = [build.to_dict() for build in file_list] 100 | self._set_response(type='application/json') 101 | self.wfile.write( 102 | json.dumps(builds_info).encode() 103 | ) 104 | logging.debug( 105 | "GET request:\nPath:%s\nHeaders:\n%s\nBody:\n%s\n", 106 | str(self.path), str(self.headers), file_list 107 | ) 108 | return 109 | elif self.path.startswith('/download'): 110 | self.path = self.path[10:] 111 | return CORSSimpleHTTPHandler.do_GET(self) 112 | else: 113 | if not os.path.exists('dist' + self.path): 114 | logging.info('redirect to dist') 115 | self.path = '/dist/' 116 | else: 117 | self.path = '/dist' + self.path 118 | return CORSSimpleHTTPHandler.do_GET(self) 119 | 120 | def do_POST(self): 121 | if self.path.startswith('/run'): 122 | content_type, _ = cgi.parse_header(self.headers['content-type']) 123 | if content_type != 'application/json': 124 | self.send_response(400) 125 | self.end_headers() 126 | return 127 | content_length = int(self.headers['Content-Length']) 128 | post_data = json.loads(self.rfile.read(content_length)) 129 | try: 130 | jobs.ota_generate(post_data, id=str(self.path[5:])) 131 | self._set_response(code=200) 132 | self.send_header("Content-Type", 'application/json') 133 | self.wfile.write(json.dumps( 134 | {"success": True, "msg": "OTA Generator started running"}).encode()) 135 | except Exception as e: 136 | logging.warning( 137 | "Failed to run ota_from_target_files %s", e.__traceback__) 138 | self.send_error( 139 | 400, "Failed to run ota_from_target_files", str(e)) 140 | logging.debug( 141 | "POST request:\nPath:%s\nHeaders:\n%s\nBody:\n%s\n", 142 | str(self.path), str(self.headers), 143 | json.dumps(post_data) 144 | ) 145 | elif self.path.startswith('/file'): 146 | file_name = os.path.join('target', self.path[6:]) 147 | file_length = int(self.headers['Content-Length']) 148 | with open(file_name, 'wb') as output_file: 149 | # Unwrap the uploaded file first (due to the usage of FormData) 150 | # The wrapper has a boundary line at the top and bottom 151 | # and some file information in the beginning 152 | # There are a file content line, a file name line, and an empty line 153 | # The boundary line in the bottom is 4 bytes longer than the top one 154 | # Please refer to the following links for more details: 155 | # https://stackoverflow.com/questions/8659808/how-does-http-file-upload-work 156 | # https://datatracker.ietf.org/doc/html/rfc1867 157 | upper_boundary = self.rfile.readline() 158 | file_length -= len(upper_boundary) * 2 + 4 159 | file_length -= len(self.rfile.readline()) 160 | file_length -= len(self.rfile.readline()) 161 | file_length -= len(self.rfile.readline()) 162 | BUFFER_SIZE = 1024*1024 163 | for offset in range(0, file_length, BUFFER_SIZE): 164 | chunk = self.rfile.read( 165 | min(file_length-offset, BUFFER_SIZE)) 166 | output_file.write(chunk) 167 | target_lib.new_build(self.path[6:], file_name) 168 | self._set_response(code=201) 169 | self.wfile.write( 170 | "File received, saved into {}".format( 171 | file_name).encode('utf-8') 172 | ) 173 | else: 174 | self.send_error(400) 175 | 176 | 177 | class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): 178 | pass 179 | 180 | 181 | def run_server(SeverClass=ThreadedHTTPServer, HandlerClass=RequestHandler, port=8000): 182 | server_address = (LOCAL_ADDRESS, port) 183 | server_instance = SeverClass(server_address, HandlerClass) 184 | try: 185 | logging.info( 186 | 'Server is on, address:\n %s', 187 | 'http://' + str(server_address[0]) + ':' + str(port)) 188 | server_instance.serve_forever() 189 | except KeyboardInterrupt: 190 | pass 191 | server_instance.server_close() 192 | logging.info('Server has been turned off.') 193 | 194 | 195 | if __name__ == '__main__': 196 | from sys import argv 197 | print(argv) 198 | logging.basicConfig(level=logging.INFO) 199 | EXTRACT_DIR = None 200 | if os.path.exists("otatools.zip"): 201 | logging.info("Found otatools.zip, extracting...") 202 | EXTRACT_DIR = "/tmp/otatools-" + str(os.getpid()) 203 | os.makedirs(EXTRACT_DIR, exist_ok=True) 204 | with zipfile.ZipFile("otatools.zip", "r") as zfp: 205 | zfp.extractall(EXTRACT_DIR) 206 | # mark all binaries executable by owner 207 | bin_dir = os.path.join(EXTRACT_DIR, "bin") 208 | for filename in os.listdir(bin_dir): 209 | os.chmod(os.path.join(bin_dir, filename), stat.S_IRWXU) 210 | logging.info("Extracted otatools to {}".format(EXTRACT_DIR)) 211 | if not os.path.isdir('target'): 212 | os.mkdir('target', 755) 213 | if not os.path.isdir('output'): 214 | os.mkdir('output', 755) 215 | target_lib = TargetLib() 216 | jobs = ProcessesManagement(otatools_dir=EXTRACT_DIR) 217 | if len(argv) == 2: 218 | run_server(port=int(argv[1])) 219 | else: 220 | run_server() 221 | -------------------------------------------------------------------------------- /web_server_flask.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | from flask import request 4 | from target_lib import TargetLib 5 | from ota_interface import ProcessesManagement 6 | import flask 7 | import tempfile 8 | import os 9 | from flask import Flask 10 | import logging 11 | import json 12 | import traceback 13 | from flask_cors import CORS 14 | 15 | 16 | app = Flask(__name__, static_url_path='', static_folder='dist') 17 | CORS(app) 18 | 19 | jobs = ProcessesManagement() 20 | target_lib = TargetLib() 21 | 22 | 23 | @app.route("/check", methods=["GET"]) 24 | def check(): 25 | statuses = jobs.get_status() 26 | return flask.jsonify(statuses) 27 | 28 | 29 | @app.route("/check/", methods=["GET"]) 30 | def check_with_id(id): 31 | status = jobs.get_status_by_ID(id=id) 32 | return flask.jsonify(status) 33 | 34 | 35 | @app.route("/file", methods=["GET"]) 36 | def list_builds(): 37 | file_list = target_lib.get_builds() 38 | return flask.jsonify(file_list) 39 | 40 | 41 | @app.route("/run/", methods=["POST"]) 42 | def generate_ota(id): 43 | try: 44 | config = request.json 45 | jobs.ota_generate(config, id=str(id)) 46 | return flask.jsonify({"success": True, "msg": "OTA Generator started running"}) 47 | except Exception as e: 48 | traceback.print_exc() 49 | return flask.Response(json.dumps({"success": False, "msg": str(e)}), status=501, mimetype=app.config["JSONIFY_MIMETYPE"]) 50 | 51 | 52 | @app.route("/file/", methods=["POST"]) 53 | def upload_build(filename): 54 | file_length = request.content_length 55 | # Unwrap the uploaded file first (due to the usage of FormData) 56 | # The wrapper has a boundary line at the top and bottom 57 | # and some file information in the beginning 58 | # There are a file content line, a file name line, and an empty line 59 | # The boundary line in the bottom is 4 bytes longer than the top one 60 | # Please refer to the following links for more details: 61 | # https://stackoverflow.com/questions/8659808/how-does-http-file-upload-work 62 | # https://datatracker.ietf.org/doc/html/rfc1867 63 | upper_boundary = request.stream.readline() 64 | file_length -= len(upper_boundary) * 2 + 4 65 | file_length -= len(request.stream.readline()) 66 | file_length -= len(request.stream.readline()) 67 | file_length -= len(request.stream.readline()) 68 | BUFFER_SIZE = 1024*1024 69 | with tempfile.NamedTemporaryFile(filename, "w+") as fp: 70 | for offset in range(0, file_length, BUFFER_SIZE): 71 | chunk = request.stream.read( 72 | min(file_length-offset, BUFFER_SIZE)) 73 | fp.write(chunk) 74 | target_lib.new_build(filename, fp.name) 75 | if not os.path.exists(fp.name): 76 | # if tempfile gets deleted/moved, exit block will yield an error 77 | # therefore touch it if does not exist 78 | with open(fp.name, 'wb'): 79 | pass 80 | return flask.Response(status=200) 81 | 82 | 83 | @app.route("/download/") 84 | def download_output(path): 85 | if not path.startswith(jobs.working_dir): 86 | return flask.Response("{} is forbidden, only files in {} can be downloaded".format(path, jobs.working_dir), status=403) 87 | return flask.send_from_directory(".", path) 88 | 89 | # Force re-write all paths to index.html, 90 | # as we are serving a Single Page Application 91 | 92 | 93 | @app.route('/', defaults={'path': ''}, methods=["GET"]) 94 | @app.route('/', methods=["GET"]) 95 | def index(path): 96 | return flask.send_from_directory(app.static_folder, "index.html") 97 | 98 | 99 | if __name__ == "__main__": 100 | app.run() 101 | --------------------------------------------------------------------------------