├── .github
└── workflows
│ └── assemble.yml
├── .gitignore
├── CONTRIBUTING.md
├── CaptureSync.iml
├── LICENSE
├── LICENSE_GOOGLE_RESEARCH
├── README.md
├── app
├── app.iml
├── build.gradle
├── proguard-rules.pro
└── src
│ └── main
│ ├── AndroidManifest.xml
│ ├── java
│ └── com
│ │ └── googleresearch
│ │ └── capturesync
│ │ ├── AutoFitSurfaceView.java
│ │ ├── CameraController.java
│ │ ├── CaptureRequestFactory.java
│ │ ├── Constants.java
│ │ ├── Frame.java
│ │ ├── ImageMetadataSynchronizer.java
│ │ ├── MainActivity.java
│ │ ├── PhaseAlignController.java
│ │ ├── ResultProcessor.java
│ │ ├── SoftwareSyncController.java
│ │ └── softwaresync
│ │ ├── CSVLogger.java
│ │ ├── ClientInfo.java
│ │ ├── NetworkHelpers.java
│ │ ├── RpcCallback.java
│ │ ├── SimpleNetworkTimeProtocol.java
│ │ ├── SntpListener.java
│ │ ├── SntpOffsetResponse.java
│ │ ├── SoftwareSyncBase.java
│ │ ├── SoftwareSyncClient.java
│ │ ├── SoftwareSyncLeader.java
│ │ ├── SyncConstants.java
│ │ ├── SystemTicker.java
│ │ ├── Ticker.java
│ │ ├── TimeDomainConverter.java
│ │ ├── TimeUtils.java
│ │ └── phasealign
│ │ ├── PeriodCalculator.java
│ │ ├── PhaseAligner.java
│ │ ├── PhaseConfig.java
│ │ └── PhaseResponse.java
│ └── res
│ ├── layout
│ └── activity_main.xml
│ ├── mipmap-xxxhdpi
│ └── ic_launcher.png
│ ├── raw
│ ├── default_phaseconfig.json
│ ├── pixel1_phaseconfig.json
│ ├── pixel2_phaseconfig.json
│ ├── pixel3_240fps_phaseconfig.json
│ └── pixel3_60fps_phaseconfig.json
│ └── values
│ ├── colors.xml
│ ├── dimens.xml
│ ├── strings.xml
│ └── styles.xml
├── build.gradle
├── gradle
└── wrapper
│ ├── gradle-wrapper.jar
│ └── gradle-wrapper.properties
├── gradlew
├── gradlew.bat
├── scripts
└── yuv2rgb.py
├── settings.gradle
└── utils
├── .gitignore
├── README.md
├── extract.py
├── extract.sh
├── get_match.py
├── make_demo.sh
├── match.sh
├── requirements.txt
├── split.py
├── src
├── __init__.py
└── extraction_utils.py
├── stitch.py
└── stitching_demo
└── stitch_two.sh
/.github/workflows/assemble.yml:
--------------------------------------------------------------------------------
1 | name: Assemble
2 |
3 | on: [push, workflow_dispatch]
4 |
5 | jobs:
6 | assemble:
7 | runs-on: ubuntu-latest
8 |
9 | steps:
10 | - uses: actions/checkout@v2
11 |
12 | - uses: actions/setup-java@v2
13 | with:
14 | distribution: 'adopt'
15 | java-version: '8'
16 |
17 | - name: Cache Gradle packages
18 | uses: actions/cache@v2
19 | with:
20 | path: |
21 | ~/.gradle/caches
22 | ~/.gradle/wrapper
23 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
24 | restore-keys: |
25 | ${{ runner.os }}-gradle-
26 |
27 | - name: Assemble release
28 | run: ./gradlew assembleRelease
29 |
30 | - name: Run lint for release
31 | run: ./gradlew lintRelease
32 |
33 | - name: Assemble debug
34 | run: ./gradlew assembleDebug
35 |
36 | - name: Run lint for debug
37 | run: ./gradlew lintDebug
38 |
39 | - name: Upload lint results
40 | uses: actions/upload-artifact@v2
41 | with:
42 | name: lint-results
43 | path: app/build/reports/lint-results-*.*
44 |
45 | - name: Cleanup Gradle Cache
46 | # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions.
47 | # Restoring these files from a GitHub Actions cache might cause problems for future builds.
48 | run: |
49 | rm -f ~/.gradle/caches/modules-2/modules-2.lock
50 | rm -f ~/.gradle/caches/modules-2/gc.properties
51 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | _archive/
2 | _devdocs/
3 | _docs/*.jpg
4 | _docs/ads.txt
5 | _docs/app-ads.txt
6 | _docs/gplay.html
7 | _other/
8 | _saved/
9 |
10 | .gradle/
11 |
12 | .idea/*
13 | !.idea/inspectionProfiles
14 |
15 | build/
16 | app/release/
17 | gfx/
18 | testdata/
19 |
20 | *.db
21 | *.iml
22 | *.apk
23 | *.ap_
24 |
25 | local.properties
26 | uninstall.bat
27 |
28 | *.keystore
29 |
30 | opencamera-extended-firebase-adminsdk-yv5yz-e33a8ce5c1.json
31 |
32 | *.csv
33 |
34 | **/*.png
35 | **/*.zip
36 | **/*.mp4
37 | **/*.tif
38 |
39 | *.mp4
40 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google/conduct/).
29 |
--------------------------------------------------------------------------------
/CaptureSync.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright 2021 Mobile Robotics Lab. at Skoltech
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/LICENSE_GOOGLE_RESEARCH:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright 2019 The Google Research Authors
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | If you use this application, please cite [Sub-millisecond Video Synchronization of Multiple Android Smartphones](https://arxiv.org/abs/2107.00987):
4 | ```
5 | @misc{akhmetyanov2021submillisecond,
6 | title={Sub-millisecond Video Synchronization of Multiple Android Smartphones},
7 | author={Azat Akhmetyanov and Anastasiia Kornilova and Marsel Faizullin and David Pozo and Gonzalo Ferrer},
8 | year={2021},
9 | eprint={2107.00987},
10 | archivePrefix={arXiv},
11 | primaryClass={cs.CV}
12 | }
13 | ```
14 | ### Usage:
15 |
16 |
17 | #### Leader smartphone setup
18 |
19 | 1. Start a Wi-Fi hotspot.
20 | 2. The app should display connected clients and buttons for recording control
21 |
22 | #### Client smartphones setup
23 |
24 | 1. Enable WiFi and connect to the Wi-Fi hotspot.
25 |
26 | #### Recording video
27 |
28 | 1. [Optional step] Press the ```calculate period``` button. The app will analyze frame stream and use the calculated frame period in further synchronization steps.
29 | 2. Adjust exposure and ISO to your needs.
30 | 3. Press the ```phase align``` button.
31 | 4. Press the ```record video``` button to start synchronized video recording.
32 | 5. Get videos from RecSync folder in smartphone root directory.
33 |
34 | #### Extraction and matching of the frames
35 |
36 | ```
37 | Requirements:
38 |
39 | - Python
40 | - ffmpeg
41 | ```
42 |
43 | 1. Navigate to ```utils``` directory in the repository.
44 | 2. Run ```./match.sh ```.
45 | 3. Frames will be extracted to directories ```output/1``` and ```output/2``` with timestamps in filenames, output directory will also contain ```match.csv``` file in the following format:
46 | ```
47 | timestamp_1(ns) timestamp_2(ns)
48 | ```
49 |
50 | ### Our contribution:
51 |
52 | - Integrated **synchronized video recording**
53 | - Scripts for extraction, alignment and processing of video frames
54 | - Experiment with flash blinking to evaluate video frames synchronization accuracy
55 | - Panoramic video demo with automated Hugin stitching
56 |
57 | ### Panoramic video stitching demo
58 |
59 | ### [Link to youtube demo video](https://youtu.be/W6iANtCuQ-o)
60 |
61 | - We provide scripts to **stitch 2 syncronized smatphone videos** with Hujin panorama CLI tools
62 | - Usage:
63 | - Run ```./make_demo.sh {VIDEO_LEFT} {VIDEO_RIGHT}```
64 |
65 | ### This work is based on "Wireless Software Synchronization of Multiple Distributed Cameras"
66 |
67 | Reference code for the paper
68 | [Wireless Software Synchronization of Multiple Distributed Cameras](https://arxiv.org/abs/1812.09366).
69 | _Sameer Ansari, Neal Wadhwa, Rahul Garg, Jiawen Chen_, ICCP 2019.
70 |
--------------------------------------------------------------------------------
/app/app.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | generateDebugSources
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
--------------------------------------------------------------------------------
/app/build.gradle:
--------------------------------------------------------------------------------
1 | apply plugin: 'com.android.application'
2 |
3 | android {
4 | compileSdkVersion 29
5 | buildToolsVersion "29.0.2"
6 | defaultConfig {
7 | applicationId "com.googleresearch.capturesync"
8 | minSdkVersion 28
9 | targetSdkVersion 29
10 | versionCode 1
11 | versionName "1.0"
12 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
13 | }
14 | buildTypes {
15 | release {
16 | minifyEnabled false
17 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
18 | }
19 | }
20 | compileOptions {
21 | sourceCompatibility = 1.8
22 | targetCompatibility = 1.8
23 | }
24 | }
25 |
26 | dependencies {
27 | implementation fileTree(dir: 'libs', include: ['*.jar'])
28 | implementation 'androidx.appcompat:appcompat:1.0.2'
29 | testImplementation 'junit:junit:4.12'
30 | androidTestImplementation 'androidx.test.ext:junit:1.1.0'
31 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1'
32 | }
33 |
--------------------------------------------------------------------------------
/app/proguard-rules.pro:
--------------------------------------------------------------------------------
1 | # Add project specific ProGuard rules here.
2 | # You can control the set of applied configuration files using the
3 | # proguardFiles setting in build.gradle.
4 | #
5 | # For more details, see
6 | # http://developer.android.com/guide/developing/tools/proguard.html
7 |
8 | # If your project uses WebView with JS, uncomment the following
9 | # and specify the fully qualified class name to the JavaScript interface
10 | # class:
11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview {
12 | # public *;
13 | #}
14 |
15 | # Uncomment this to preserve the line number information for
16 | # debugging stack traces.
17 | #-keepattributes SourceFile,LineNumberTable
18 |
19 | # If you keep the line number information, uncomment this to
20 | # hide the original source file name.
21 | #-renamesourcefileattribute SourceFile
22 |
--------------------------------------------------------------------------------
/app/src/main/AndroidManifest.xml:
--------------------------------------------------------------------------------
1 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
17 |
18 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/AutoFitSurfaceView.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.content.Context;
20 | import android.util.AttributeSet;
21 | import android.util.Log;
22 | import android.view.SurfaceView;
23 |
24 | /** A {@link SurfaceView} that can be adjusted to a specified aspect ratio. */
25 | public class AutoFitSurfaceView extends SurfaceView {
26 |
27 | private static final String TAG = "AutoFitSurfaceView";
28 |
29 | private int ratioWidth = 0;
30 | private int ratioHeight = 0;
31 |
32 | public AutoFitSurfaceView(Context context) {
33 | this(context, null);
34 | }
35 |
36 | public AutoFitSurfaceView(Context context, AttributeSet attrs) {
37 | this(context, attrs, 0);
38 | }
39 |
40 | public AutoFitSurfaceView(Context context, AttributeSet attrs, int defStyle) {
41 | super(context, attrs, defStyle);
42 | }
43 |
44 | /**
45 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio
46 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that is,
47 | * calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result.
48 | *
49 | * @param width Relative horizontal size
50 | * @param height Relative vertical size
51 | */
52 | public void setAspectRatio(int width, int height) {
53 | if (width < 0 || height < 0) {
54 | throw new IllegalArgumentException("Size cannot be negative.");
55 | }
56 | ratioWidth = width;
57 | ratioHeight = height;
58 | requestLayout();
59 | }
60 |
61 | @Override
62 | protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) {
63 | super.onMeasure(widthMeasureSpec, heightMeasureSpec);
64 | int width = MeasureSpec.getSize(widthMeasureSpec);
65 | int height = MeasureSpec.getSize(heightMeasureSpec);
66 | if (0 == ratioWidth || 0 == ratioHeight) {
67 | Log.d(
68 | TAG,
69 | String.format(
70 | "aspect ratio is 0 x 0 (uninitialized), setting measured" + " dimension to: %d x %d",
71 | width, height));
72 | setMeasuredDimension(width, height);
73 | } else {
74 | if (width < height * ratioWidth / ratioHeight) {
75 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height));
76 | setMeasuredDimension(width, width * ratioHeight / ratioWidth);
77 | } else {
78 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height));
79 | setMeasuredDimension(height * ratioWidth / ratioHeight, height);
80 | }
81 | }
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/CameraController.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Mobile Robotics Lab. at Skoltech.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.annotation.SuppressLint;
20 | import android.graphics.ImageFormat;
21 | import android.hardware.camera2.CameraCaptureSession.CaptureCallback;
22 | import android.hardware.camera2.CameraCharacteristics;
23 | import android.hardware.camera2.CameraDevice;
24 | import android.hardware.camera2.CaptureResult;
25 | import android.media.ImageReader;
26 | import android.os.Handler;
27 | import android.os.HandlerThread;
28 | import android.util.Log;
29 | import android.util.Size;
30 | import android.view.Surface;
31 | import com.googleresearch.capturesync.ImageMetadataSynchronizer.CaptureRequestTag;
32 | import com.googleresearch.capturesync.softwaresync.TimeDomainConverter;
33 | import com.googleresearch.capturesync.softwaresync.TimeUtils;
34 | import com.googleresearch.capturesync.softwaresync.phasealign.PeriodCalculator;
35 |
36 | import java.io.IOException;
37 | import java.text.SimpleDateFormat;
38 | import java.util.ArrayList;
39 | import java.util.List;
40 | import java.util.TimeZone;
41 |
42 | /** High level camera controls. */
43 | public class CameraController {
44 | private static final String TAG = "CameraController";
45 |
46 | // Thread on which to receive ImageReader callbacks.
47 | private HandlerThread imageThread;
48 | private Handler imageHandler;
49 |
50 | // Thread on which to receive Synchronized results.
51 | private HandlerThread syncThread;
52 | private Handler syncHandler;
53 |
54 | // ImageReaders, which puts the images into
55 | // their respective queues.
56 | private final List imageReaders;
57 |
58 | private final ImageMetadataSynchronizer imageMetadataSynchronizer;
59 |
60 | private final ResultProcessor resultProcessor;
61 |
62 | public CaptureCallback getSynchronizerCaptureCallback() {
63 | return imageMetadataSynchronizer.getCaptureCallback();
64 | }
65 |
66 | /**
67 | * Camera frames come in continuously and are thrown away. When a desired timestamp {@code
68 | * goalSynchronizedTimestampNs} is set, the first frame with synchronized timestamp at or after
69 | * the desired timestamp is saved to disk.
70 | */
71 | private long goalSynchronizedTimestampNs;
72 |
73 | private String goalOutputDirName;
74 |
75 | private PeriodCalculator periodCalculator;
76 |
77 | private CaptureRequestFactory requestFactory;
78 |
79 | /**
80 | * Constructs the high level CameraController object.
81 | *
82 | *
If {@code rawImageResolution} is not null, it will create an ImageReader for the raw stream
83 | * and stream frames to it. If {@code yuvImageResolution} is not null, it will create an
84 | * ImageReader for the yuv stream and stream frames to it. If {@code viewfinderSurface} is not
85 | * null, it will stream frames to it.
86 | */
87 | @SuppressLint("DefaultLocale")
88 | public CameraController(
89 | CameraCharacteristics cameraCharacteristics,
90 | Size rawImageResolution,
91 | Size yuvImageResolution,
92 | PhaseAlignController phaseAlignController,
93 | MainActivity context,
94 | TimeDomainConverter timeDomainConverter
95 | ) {
96 | imageThread = new HandlerThread("ImageThread");
97 | imageThread.start();
98 | imageHandler = new Handler(imageThread.getLooper());
99 |
100 | syncThread = new HandlerThread("SyncThread");
101 | syncThread.start();
102 | syncHandler = new Handler(syncThread.getLooper());
103 |
104 | resultProcessor =
105 | new ResultProcessor(
106 | timeDomainConverter, context, Constants.SAVE_JPG_FROM_YUV, Constants.JPG_QUALITY);
107 |
108 | imageReaders = new ArrayList<>();
109 | final int imageBuffer = 1;
110 | if (rawImageResolution != null) {
111 | imageReaders.add(
112 | ImageReader.newInstance(
113 | rawImageResolution.getWidth(),
114 | rawImageResolution.getHeight(),
115 | ImageFormat.RAW10,
116 | imageBuffer));
117 | }
118 |
119 | if (yuvImageResolution != null) {
120 | imageReaders.add(
121 | ImageReader.newInstance(
122 | yuvImageResolution.getWidth(),
123 | yuvImageResolution.getHeight(),
124 | ImageFormat.YUV_420_888,
125 | imageBuffer));
126 | }
127 |
128 | imageMetadataSynchronizer = new ImageMetadataSynchronizer(imageReaders, imageHandler, context);
129 | imageMetadataSynchronizer.registerCallback(
130 | output -> {
131 | CaptureResult result = output.result;
132 | Object userTag = CaptureRequestTag.getUserTag(result);
133 | // We don't skip them because every frame counts when restoring video sequence
134 | // if (userTag != null && userTag.equals(PhaseAlignController.INJECT_FRAME)) {
135 | // Log.v(TAG, "Skipping phase align injection frame.");
136 | // output.close();
137 | // return;
138 | // }
139 |
140 | int sequenceId = result.getSequenceId();
141 | long unSyncTimestampNs = result.get(CaptureResult.SENSOR_TIMESTAMP);
142 | context.onTimestampNs(unSyncTimestampNs);
143 | long synchronizedTimestampNs =
144 | timeDomainConverter.leaderTimeForLocalTimeNs(
145 | unSyncTimestampNs);
146 |
147 | double timestampMs = TimeUtils.nanosToMillis((double) synchronizedTimestampNs);
148 | double frameDurationMs =
149 | TimeUtils.nanosToMillis((double) result.get(CaptureResult.SENSOR_FRAME_DURATION));
150 |
151 | long phaseNs = phaseAlignController.updateCaptureTimestamp(synchronizedTimestampNs);
152 | double phaseMs = TimeUtils.nanosToMillis((double) phaseNs);
153 | // Log.v(
154 | // TAG,
155 | // String.format(
156 | // "onCaptureCompleted: timestampMs = %,.3f, frameDurationMs = %,.6f, phase ="
157 | // + " %,.3f, sequence id = %d",
158 | // timestampMs, frameDurationMs, phaseMs, sequenceId));
159 | // TODO: log this to csv
160 | try {
161 | synchronized(this) {
162 | if (context.getLogger() != null && !context.getLogger().isClosed() && context.getLastVideoSeqId() != null && context.getLastVideoSeqId() == sequenceId) {
163 | context.getLogger().logLine(String.format("%d",
164 | synchronizedTimestampNs));
165 | }
166 | }
167 | } catch (IOException e) {
168 | e.printStackTrace();
169 | }
170 | if (shouldSaveFrame(synchronizedTimestampNs)) {
171 | Log.d(TAG, "Sync frame found! Committing and processing");
172 | Frame frame = new Frame(result, output);
173 | resultProcessor.submitProcessRequest(frame, goalOutputDirName);
174 | resetGoal();
175 | } else {
176 | output.close();
177 | }
178 | },
179 | syncHandler);
180 | }
181 |
182 | /* Check if given timestamp is or passed goal timestamp in the synchronized leader time domain. */
183 | private boolean shouldSaveFrame(long synchronizedTimestampNs) {
184 | return goalSynchronizedTimestampNs != 0
185 | && synchronizedTimestampNs >= goalSynchronizedTimestampNs;
186 | }
187 |
188 | private void resetGoal() {
189 | goalSynchronizedTimestampNs = 0;
190 | }
191 |
192 | public List getOutputSurfaces() {
193 | List surfaces = new ArrayList<>();
194 | for (ImageReader reader : imageReaders) {
195 | surfaces.add(reader.getSurface());
196 | }
197 | return surfaces;
198 | }
199 |
200 | public void configure(CameraDevice device) {
201 | requestFactory = new CaptureRequestFactory(device);
202 | }
203 |
204 | public void close() {
205 | imageMetadataSynchronizer.close();
206 |
207 | imageThread.quitSafely();
208 | try {
209 | imageThread.join();
210 | imageThread = null;
211 | imageHandler = null;
212 | } catch (InterruptedException e) {
213 | Log.e(TAG, "Failed to join imageThread");
214 | }
215 |
216 | syncThread.quitSafely();
217 | try {
218 | syncThread.join();
219 | syncThread = null;
220 | syncHandler = null;
221 | } catch (InterruptedException e) {
222 | Log.e(TAG, "Failed to join syncThread");
223 | }
224 | }
225 |
226 | public CaptureRequestFactory getRequestFactory() {
227 | return requestFactory;
228 | }
229 |
230 | // Input desired capture time in leader time domain (first frame that >= that timestamp).
231 | public void setUpcomingCaptureStill(long desiredSynchronizedCaptureTimeNs) {
232 | goalOutputDirName = getTimeStr(desiredSynchronizedCaptureTimeNs);
233 | goalSynchronizedTimestampNs = desiredSynchronizedCaptureTimeNs;
234 | Log.i(
235 | TAG,
236 | String.format(
237 | "Request sync still at %d to %s", goalSynchronizedTimestampNs, goalOutputDirName));
238 | }
239 |
240 | private String getTimeStr(long timestampNs) {
241 | SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd_HHmmss_SSS");
242 | simpleDateFormat.setTimeZone(TimeZone.getDefault());
243 | return simpleDateFormat.format(timestampNs / 1_000_000L);
244 | }
245 | }
246 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/CaptureRequestFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Mobile Robotics Lab. at Skoltech.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import static android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW;
20 | import static android.hardware.camera2.CameraMetadata.CONTROL_AE_MODE_OFF;
21 | import static android.hardware.camera2.CameraMetadata.CONTROL_AWB_MODE_AUTO;
22 | import static android.hardware.camera2.CameraMetadata.CONTROL_MODE_AUTO;
23 | import static android.hardware.camera2.CaptureRequest.CONTROL_AE_MODE;
24 | import static android.hardware.camera2.CaptureRequest.CONTROL_AF_MODE;
25 | import static android.hardware.camera2.CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE;
26 | import static android.hardware.camera2.CaptureRequest.CONTROL_AWB_MODE;
27 | import static android.hardware.camera2.CaptureRequest.CONTROL_MODE;
28 | import static android.hardware.camera2.CaptureRequest.SENSOR_EXPOSURE_TIME;
29 | import static android.hardware.camera2.CaptureRequest.SENSOR_SENSITIVITY;
30 |
31 | import android.hardware.camera2.CameraAccessException;
32 | import android.hardware.camera2.CameraDevice;
33 | import android.hardware.camera2.CaptureRequest;
34 | import android.view.Surface;
35 | import com.googleresearch.capturesync.ImageMetadataSynchronizer.CaptureRequestTag;
36 | import java.util.ArrayList;
37 | import java.util.List;
38 |
39 | /** Helper class for creating common {@link CaptureRequest.Builder} instances. */
40 | public class CaptureRequestFactory {
41 |
42 | private final CameraDevice device;
43 |
44 | public CaptureRequestFactory(CameraDevice camera) {
45 | device = camera;
46 | }
47 |
48 | /**
49 | * Makes a {@link CaptureRequest.Builder} for the viewfinder preview. This always adds the
50 | * viewfinder.
51 | */
52 | public CaptureRequest.Builder makePreview(
53 | Surface viewfinderSurface,
54 | List imageSurfaces,
55 | long sensorExposureTimeNs,
56 | int sensorSensitivity,
57 | boolean wantAutoExp)
58 | throws CameraAccessException {
59 |
60 | CaptureRequest.Builder builder = device.createCaptureRequest(TEMPLATE_PREVIEW);
61 | if (wantAutoExp) {
62 | builder.set(CONTROL_AE_MODE, CONTROL_AWB_MODE_AUTO);
63 |
64 | } else {
65 | // Manually set exposure and sensitivity using UI sliders on the leader.
66 | builder.set(CONTROL_AE_MODE, CONTROL_AE_MODE_OFF);
67 | builder.set(SENSOR_EXPOSURE_TIME, sensorExposureTimeNs);
68 | builder.set(SENSOR_SENSITIVITY, sensorSensitivity);
69 | }
70 |
71 | // Auto white balance used, these could be locked and sent from the leader instead.
72 | builder.set(CONTROL_AWB_MODE, CONTROL_AWB_MODE_AUTO);
73 |
74 | // Auto focus is used since different devices may have different manual focus values.
75 | builder.set(CONTROL_AF_MODE, CONTROL_AF_MODE_CONTINUOUS_PICTURE);
76 |
77 | if (viewfinderSurface != null) {
78 | builder.addTarget(viewfinderSurface);
79 | }
80 | List targetIndices = new ArrayList<>();
81 | for (int i = 0; i < imageSurfaces.size(); i++) {
82 | builder.addTarget(imageSurfaces.get(i));
83 | targetIndices.add(i);
84 | }
85 | builder.setTag(new CaptureRequestTag(targetIndices, null));
86 | return builder;
87 | }
88 |
89 | /**
90 | * An alternative capture request for video,
91 | * includes everything from preview + mediaRecorder
92 | */
93 | public CaptureRequest.Builder makeVideo(
94 | Surface recorderSurface,
95 | Surface viewfinderSurface,
96 | List imageSurfaces,
97 | long sensorExposureTimeNs,
98 | int sensorSensitivity,
99 | boolean wantAutoExp)
100 | throws CameraAccessException {
101 | CaptureRequest.Builder builder = makePreview(viewfinderSurface, imageSurfaces, sensorExposureTimeNs, sensorSensitivity, wantAutoExp);
102 | // Add recorder surface
103 | if (recorderSurface != null) {
104 | builder.addTarget(recorderSurface);
105 | }
106 | return builder;
107 | }
108 |
109 |
110 | public CaptureRequest.Builder makeFrameInjectionRequest(
111 | long desiredExposureTimeNs, List imageSurfaces) throws CameraAccessException {
112 | CaptureRequest.Builder builder = device.createCaptureRequest(TEMPLATE_PREVIEW);
113 | builder.set(CONTROL_MODE, CONTROL_MODE_AUTO);
114 | builder.set(CONTROL_AE_MODE, CONTROL_AE_MODE_OFF);
115 | builder.set(SENSOR_EXPOSURE_TIME, desiredExposureTimeNs);
116 | // TODO: Inserting frame duration directly would be more accurate than inserting exposure since
117 | // {@code frame duration ~ exposure + variable overhead}. However setting frame duration may not
118 | // be supported on many android devices, so we use exposure time here.
119 |
120 | List targetIndices = new ArrayList<>();
121 | for (int i = 0; i < imageSurfaces.size(); i++) {
122 | builder.addTarget(imageSurfaces.get(i));
123 | targetIndices.add(i);
124 | }
125 | builder.setTag(new CaptureRequestTag(targetIndices, PhaseAlignController.INJECT_FRAME));
126 |
127 | return builder;
128 | }
129 | }
130 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/Constants.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.hardware.camera2.CameraCharacteristics;
20 | import com.googleresearch.capturesync.softwaresync.TimeUtils;
21 |
22 | /** Constants including what type of images to save and the save directory. */
23 | public final class Constants {
24 | public static final int DEFAULT_CAMERA_FACING = CameraCharacteristics.LENS_FACING_BACK;
25 |
26 | /**
27 | * Delay from capture button press to capture, giving network time to send messages to clients.
28 | */
29 | public static final long FUTURE_TRIGGER_DELAY_NS = TimeUtils.millisToNanos(500);
30 |
31 | /* Set at least one of {SAVE_YUV, SAVE_RAW} to true to save any data. */
32 | public static final boolean SAVE_YUV = true;
33 |
34 | // TODO: Implement saving ImageFormat.RAW10 to DNG.
35 | // DngCreator works with ImageFormat.RAW_SENSOR but it is slow and power-hungry.
36 | public static final boolean SAVE_RAW = false;
37 |
38 | // TODO(samansari): Turn SAVE_JPG_FROM_YUV into a checkbox instead.
39 | /* Set true to save a JPG to the gallery for preview. This is slow but gives you a "postview". */
40 | public static final boolean SAVE_JPG_FROM_YUV = true;
41 | public static final int JPG_QUALITY = 95;
42 |
43 | public static final boolean USE_FULL_SCREEN_IMMERSIVE = false;
44 |
45 | private Constants() {}
46 | }
47 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/Frame.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.hardware.camera2.CaptureResult;
20 | import android.media.Image;
21 | import com.googleresearch.capturesync.ImageMetadataSynchronizer.Output;
22 | import java.io.Closeable;
23 |
24 | /** A holder for a CaptureResult with multiple outputs. */
25 | public class Frame implements Closeable {
26 | public final CaptureResult result;
27 | public final Output output;
28 | private boolean closed = false;
29 |
30 | public Frame(CaptureResult result, Output output) {
31 | this.result = result;
32 | this.output = output;
33 | }
34 |
35 | @Override
36 | public void close() {
37 | if (closed) {
38 | throw new IllegalStateException("This Frame is already closed");
39 | }
40 | for (Image image : output.images) {
41 | image.close();
42 | }
43 | output.close();
44 | closed = true;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/PhaseAlignController.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.os.Handler;
20 | import android.util.Log;
21 |
22 | import com.googleresearch.capturesync.softwaresync.phasealign.PhaseAligner;
23 | import com.googleresearch.capturesync.softwaresync.phasealign.PhaseConfig;
24 | import com.googleresearch.capturesync.softwaresync.phasealign.PhaseResponse;
25 |
26 | /**
27 | * Calculates and adjusts camera phase by inserting frames of varying exposure lengths.
28 | *
29 | *
Phase alignment is an iterative process. Running for more iterations results in higher
30 | * accuracy up to the stability of the camera and the accuracy of the phase alignment configuration
31 | * values.
32 | */
33 | public class PhaseAlignController {
34 | public static final String INJECT_FRAME = "injection_frame";
35 | private static final String TAG = "PhaseAlignController";
36 |
37 | // Maximum number of phase alignment iteration steps in the alignment process.
38 | // TODO(samansari): Make this a parameter that you pass in to this class. Then make the class that
39 | // constructs this pass the constant in.
40 | private static final int MAX_ITERATIONS = 50;
41 | // Delay after an alignment step to wait for phase to settle before starting the next iteration.
42 | private static final int PHASE_SETTLE_DELAY_MS = 400;
43 | private final MainActivity context;
44 |
45 | private final Handler handler;
46 | private final Object lock = new Object();
47 | private boolean inAlignState = false;
48 |
49 | private PhaseAligner phaseAligner;
50 | private final PhaseConfig phaseConfig;
51 | private PhaseResponse latestResponse;
52 |
53 | public PhaseAlignController(PhaseConfig config, MainActivity context) {
54 | handler = new Handler();
55 | phaseConfig = config;
56 | phaseAligner = new PhaseAligner(config);
57 | Log.v(TAG, "Loaded phase align config.");
58 | this.context = context;
59 | }
60 |
61 | protected void setPeriodNs(long periodNs) {
62 | phaseConfig.setPeriodNs(periodNs);
63 | this.phaseAligner = new PhaseAligner(phaseConfig);
64 | }
65 |
66 | /**
67 | * Update the latest phase response from the latest frame timestamp to keep track of phase.
68 | *
69 | *
The timestamp is nanoseconds in the synchronized leader clock domain.
70 | *
71 | * @return phase of timestamp in nanoseconds in the same domain as given.
72 | */
73 | public long updateCaptureTimestamp(long timestampNs) {
74 | // TODO(samansaari) : Rename passTimestamp -> updateCaptureTimestamp or similar in softwaresync.
75 | latestResponse = phaseAligner.passTimestamp(timestampNs);
76 | // TODO (samansari) : Pull this into an interface/callback.
77 | context.runOnUiThread(() -> context.updatePhaseTextView(latestResponse.diffFromGoalNs()));
78 | return latestResponse.phaseNs();
79 | }
80 |
81 | /** Submit an frame with a specific exposure to offset future frames and align phase. */
82 | private void doPhaseAlignStep() {
83 | Log.i(
84 | TAG,
85 | String.format(
86 | "Current Phase: %.3f ms, Diff: %.3f ms, inserting frame exposure %.6f ms, lower bound"
87 | + " %.6f ms.",
88 | latestResponse.phaseNs() * 1e-6f,
89 | latestResponse.diffFromGoalNs() * 1e-6f,
90 | latestResponse.exposureTimeToShiftNs() * 1e-6f,
91 | phaseAligner.getConfig().minExposureNs() * 1e-6f));
92 |
93 | // TODO(samansari): Make this an interface.
94 | context.injectFrame(latestResponse.exposureTimeToShiftNs());
95 | }
96 |
97 | public void startAlign() {
98 | synchronized (lock) {
99 | if (inAlignState) {
100 | Log.i(TAG, "startAlign() called while already aligning.");
101 | return;
102 | }
103 | inAlignState = true;
104 | // Start inserting frames every {@code PHASE_SETTLE_DELAY_MS} ms to try and push the phase to
105 | // the goal phase. Stop after aligned to threshold or after {@code MAX_ITERATIONS}.
106 | handler.post(() -> work(MAX_ITERATIONS));
107 | }
108 | }
109 |
110 | private void work(int iterationsLeft) {
111 | // Check if Aligned / Not Aligned but able to iterate / Ran out of iterations.
112 | if (latestResponse.isAligned()) { // Aligned.
113 | Log.i(
114 | TAG,
115 | String.format(
116 | "Reached: Current Phase: %.3f ms, Diff: %.3f ms",
117 | latestResponse.phaseNs() * 1e-6f, latestResponse.diffFromGoalNs() * 1e-6f));
118 | synchronized (lock) {
119 | inAlignState = false;
120 | }
121 |
122 | Log.d(TAG, "Aligned.");
123 | } else if (!latestResponse.isAligned() && iterationsLeft > 0) {
124 | // Not aligned but able to run another alignment iteration.
125 | doPhaseAlignStep();
126 | Log.v(TAG, "Queued another phase align step.");
127 | // TODO (samansari) : Replace this brittle delay-based solution to a response-based one.
128 | handler.postDelayed(
129 | () -> work(iterationsLeft - 1), PHASE_SETTLE_DELAY_MS); // Try again after it settles.
130 | } else { // Reached max iterations before aligned.
131 | Log.i(
132 | TAG,
133 | String.format(
134 | "Failed to Align, Stopping at: Current Phase: %.3f ms, Diff: %.3f ms",
135 | latestResponse.phaseNs() * 1e-6f, latestResponse.diffFromGoalNs() * 1e-6f));
136 | synchronized (lock) {
137 | inAlignState = false;
138 | }
139 | Log.d(TAG, "Finishing alignment, reached max iterations.");
140 | }
141 | }
142 | }
143 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/SoftwareSyncController.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync;
18 |
19 | import android.content.Context;
20 | import android.graphics.Color;
21 | import android.net.wifi.WifiManager;
22 | import android.provider.Settings.Secure;
23 | import android.util.Log;
24 | import android.widget.TextView;
25 |
26 | import com.googleresearch.capturesync.softwaresync.ClientInfo;
27 | import com.googleresearch.capturesync.softwaresync.NetworkHelpers;
28 | import com.googleresearch.capturesync.softwaresync.RpcCallback;
29 | import com.googleresearch.capturesync.softwaresync.SoftwareSyncBase;
30 | import com.googleresearch.capturesync.softwaresync.SoftwareSyncClient;
31 | import com.googleresearch.capturesync.softwaresync.SoftwareSyncLeader;
32 | import com.googleresearch.capturesync.softwaresync.SyncConstants;
33 | import com.googleresearch.capturesync.softwaresync.TimeUtils;
34 |
35 | import java.io.Closeable;
36 | import java.io.IOException;
37 | import java.net.InetAddress;
38 | import java.net.SocketException;
39 | import java.net.UnknownHostException;
40 | import java.util.HashMap;
41 | import java.util.Map;
42 | import java.util.Map.Entry;
43 |
44 | // Note : Needs Network permissions.
45 |
46 | /** Controller managing setup and tear down the SoftwareSync object. */
47 | public class SoftwareSyncController implements Closeable {
48 |
49 | private static final String TAG = "SoftwareSyncController";
50 | private final MainActivity context;
51 | private final TextView statusView;
52 | private final PhaseAlignController phaseAlignController;
53 | private boolean isLeader;
54 | SoftwareSyncBase softwareSync;
55 |
56 | /* Tell devices to save the frame at the requested trigger time. */
57 | public static final int METHOD_SET_TRIGGER_TIME = 200_000;
58 |
59 | /* Tell devices to phase align. */
60 | public static final int METHOD_DO_PHASE_ALIGN = 200_001;
61 | /* Tell devices to set manual exposure and white balance to the requested values. */
62 | public static final int METHOD_SET_2A = 200_002;
63 | public static final int METHOD_START_RECORDING = 200_003;
64 | public static final int METHOD_STOP_RECORDING = 200_004;
65 |
66 | private long upcomingTriggerTimeNs;
67 |
68 | /**
69 | * Constructor passed in with: - context - For setting UI elements and triggering captures. -
70 | * captureButton - The button used to send at trigger request by the leader. - statusView - The
71 | * TextView used to show currently connected clients on the leader device.
72 | */
73 | public SoftwareSyncController(
74 | MainActivity context, PhaseAlignController phaseAlignController, TextView statusView) {
75 | this.context = context;
76 | this.phaseAlignController = phaseAlignController;
77 | this.statusView = statusView;
78 |
79 | setupSoftwareSync();
80 | }
81 |
82 | @SuppressWarnings("StringSplitter")
83 | private void setupSoftwareSync() {
84 | Log.w(TAG, "setup SoftwareSync");
85 | if (softwareSync != null) {
86 | return;
87 | }
88 |
89 | // Get Wifi Manager and use NetworkHelpers to determine local and leader IP addresses.
90 | WifiManager wifiManager = (WifiManager) context.getSystemService(Context.WIFI_SERVICE);
91 | InetAddress leaderAddress;
92 | InetAddress localAddress;
93 |
94 | // Use last 4 digits of the serial as the name of the client.
95 | String name = lastFourSerial();
96 | Log.w(TAG, "Name/Serial# (Last 4 digits): " + name);
97 |
98 | try {
99 | NetworkHelpers networkHelper = new NetworkHelpers(wifiManager);
100 | localAddress = NetworkHelpers.getIPAddress();
101 | // TODO: hotspot patch
102 | leaderAddress = networkHelper.getHotspotServerAddress();
103 |
104 | // Note: This is a brittle way of checking leadership that may not work on all devices.
105 | // Leader only if it is the one with same IP address as the server, or a zero IP address.
106 | if (localAddress.equals(leaderAddress)) {
107 | Log.d(TAG, "Leader == Local Address");
108 | isLeader = true;
109 | } else if (localAddress.equals(InetAddress.getByName("0.0.0.0"))) {
110 | // Log.d(TAG, "Leader == 0.0.0.0");
111 | // TODO: hotspot patch
112 | isLeader = true;
113 | }
114 | // isLeader = true;
115 |
116 | Log.w(
117 | TAG,
118 | String.format(
119 | "Current IP: %s , Leader IP: %s | Leader? %s",
120 | localAddress, leaderAddress, isLeader ? "Y" : "N"));
121 | } catch (UnknownHostException | SocketException e) {
122 | if (isLeader) {
123 | Log.e(TAG, "Error: " + e);
124 | throw new IllegalStateException(
125 | "Unable to get IP addresses, check if WiFi hotspot is enabled.", e);
126 | } else {
127 | throw new IllegalStateException(
128 | "Unable to get IP addresses, check Network permissions.", e);
129 | }
130 | }
131 |
132 | // Set up shared rpcs.
133 | Map sharedRpcs = new HashMap<>();
134 | sharedRpcs.put(
135 | METHOD_SET_TRIGGER_TIME,
136 | payload -> {
137 | Log.v(TAG, "Setting next trigger to" + payload);
138 | upcomingTriggerTimeNs = Long.valueOf(payload);
139 | // TODO: (MROB) change to video
140 | context.setUpcomingCaptureStill(upcomingTriggerTimeNs);
141 | });
142 |
143 |
144 | sharedRpcs.put(
145 | METHOD_DO_PHASE_ALIGN,
146 | payload -> {
147 | // Note: One could pass the current phase of the leader and have all clients sync to
148 | // that, reducing potential error, though special attention should be placed to phases
149 | // close to the zero or period boundary.
150 | Log.v(TAG, "Starting phase alignment.");
151 | phaseAlignController.startAlign();
152 | });
153 |
154 | sharedRpcs.put(
155 | METHOD_SET_2A,
156 | payload -> {
157 | Log.v(TAG, "Received payload: " + payload);
158 |
159 | String[] segments = payload.split(",");
160 | if (segments.length != 2) {
161 | throw new IllegalArgumentException("Wrong number of segments in payload: " + payload);
162 | }
163 | long sensorExposureNs = Long.parseLong(segments[0]);
164 | int sensorSensitivity = Integer.parseInt(segments[1]);
165 | context.set2aAndUpdatePreview(sensorExposureNs, sensorSensitivity);
166 | });
167 |
168 | if (isLeader) {
169 | // Leader.
170 | long initTimeNs = TimeUtils.millisToNanos(System.currentTimeMillis());
171 | // Create rpc mapping specific to leader.
172 | Map leaderRpcs = new HashMap<>(sharedRpcs);
173 | leaderRpcs.put(SyncConstants.METHOD_MSG_ADDED_CLIENT, payload -> updateClientsUI());
174 | leaderRpcs.put(SyncConstants.METHOD_MSG_REMOVED_CLIENT, payload -> updateClientsUI());
175 | leaderRpcs.put(SyncConstants.METHOD_MSG_SYNCING, payload -> updateClientsUI());
176 | leaderRpcs.put(SyncConstants.METHOD_MSG_OFFSET_UPDATED, payload -> updateClientsUI());
177 | softwareSync = new SoftwareSyncLeader(name, initTimeNs, localAddress, leaderRpcs);
178 | } else {
179 | // Client.
180 | Map clientRpcs = new HashMap<>(sharedRpcs);
181 | clientRpcs.put(
182 | SyncConstants.METHOD_MSG_WAITING_FOR_LEADER,
183 | payload ->
184 | context.runOnUiThread(
185 | () -> statusView.setText(softwareSync.getName() + ": Waiting for Leader")));
186 | clientRpcs.put(
187 | SyncConstants.METHOD_MSG_SYNCING,
188 | payload ->
189 | context.runOnUiThread(
190 | () -> statusView.setText(softwareSync.getName() + ": Waiting for Sync")));
191 |
192 | clientRpcs.put(
193 | METHOD_START_RECORDING,
194 | payload -> {
195 | Log.v(TAG, "Starting video");
196 | context.runOnUiThread(
197 | () -> context.startVideo(false)
198 | );
199 | });
200 |
201 | clientRpcs.put(
202 | METHOD_STOP_RECORDING,
203 | payload -> {
204 | Log.v(TAG, "Stopping video");
205 | context.runOnUiThread(
206 | context::stopVideo
207 | );
208 | });
209 |
210 | clientRpcs.put(
211 | SyncConstants.METHOD_MSG_OFFSET_UPDATED,
212 | payload ->
213 | context.runOnUiThread(
214 | () ->
215 | statusView.setText(
216 | String.format(
217 | "Client %s\n-Synced to Leader %s",
218 | softwareSync.getName(), softwareSync.getLeaderAddress()))));
219 | softwareSync = new SoftwareSyncClient(name, localAddress, leaderAddress, clientRpcs);
220 | }
221 |
222 | if (isLeader) {
223 | context.runOnUiThread(
224 | () -> {
225 | statusView.setText("Leader : " + softwareSync.getName());
226 | statusView.setTextColor(Color.rgb(0, 139, 0)); // Dark green.
227 | });
228 | } else {
229 | context.runOnUiThread(
230 | () -> {
231 | statusView.setText("Client : " + softwareSync.getName());
232 | statusView.setTextColor(Color.rgb(0, 0, 139)); // Dark blue.
233 | });
234 | }
235 | }
236 |
237 | /**
238 | * Show the number of connected clients on the leader status UI.
239 | *
240 | *
If the number of clients doesn't equal TOTAL_NUM_CLIENTS, show as bright red.
241 | */
242 | private void updateClientsUI() {
243 | SoftwareSyncLeader leader = ((SoftwareSyncLeader) softwareSync);
244 | final int clientCount = leader.getClients().size();
245 | context.runOnUiThread(
246 | () -> {
247 | StringBuilder msg = new StringBuilder();
248 | msg.append(
249 | String.format("Leader %s: %d clients.\n", softwareSync.getName(), clientCount));
250 | for (Entry entry : leader.getClients().entrySet()) {
251 | ClientInfo client = entry.getValue();
252 | if (client.syncAccuracy() == 0) {
253 | msg.append(String.format("-Client %s: syncing...\n", client.name()));
254 | } else {
255 | msg.append(
256 | String.format(
257 | "-Client %s: %.2f ms sync\n", client.name(), client.syncAccuracy() / 1e6));
258 | }
259 | }
260 | statusView.setText(msg.toString());
261 | });
262 | }
263 |
264 | @Override
265 | public void close() {
266 | Log.w(TAG, "close SoftwareSyncController");
267 | if (softwareSync != null) {
268 | try {
269 | softwareSync.close();
270 | } catch (IOException e) {
271 | throw new IllegalStateException("Error closing SoftwareSync", e);
272 | }
273 | softwareSync = null;
274 | }
275 | }
276 |
277 | private String lastFourSerial() {
278 | String serial = Secure.getString(context.getContentResolver(), Secure.ANDROID_ID);
279 | if (serial.length() <= 4) {
280 | return serial;
281 | } else {
282 | return serial.substring(serial.length() - 4);
283 | }
284 | }
285 |
286 | public boolean isLeader() {
287 | return isLeader;
288 | }
289 | }
290 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/CSVLogger.java:
--------------------------------------------------------------------------------
1 | package com.googleresearch.capturesync.softwaresync;
2 |
3 | import android.content.Context;
4 | import android.hardware.camera2.CameraAccessException;
5 | import android.hardware.camera2.CameraCharacteristics;
6 | import android.hardware.camera2.CameraManager;
7 | import android.hardware.camera2.CameraMetadata;
8 | import android.os.Build;
9 | import android.os.Environment;
10 |
11 | import com.googleresearch.capturesync.MainActivity;
12 |
13 | import java.io.BufferedWriter;
14 | import java.io.File;
15 | import java.io.FileWriter;
16 | import java.io.IOException;
17 | import java.nio.file.Files;
18 | import java.nio.file.Path;
19 | import java.nio.file.Paths;
20 | import java.util.HashMap;
21 | import java.util.Map;
22 |
23 | public class CSVLogger {
24 | private final BufferedWriter writer;
25 |
26 | public boolean isClosed() {
27 | return isClosed;
28 | }
29 |
30 | private volatile boolean isClosed;
31 |
32 | public CSVLogger(String dirName, String filename, MainActivity context) throws IOException {
33 | isClosed = true;
34 | File sdcard = Environment.getExternalStorageDirectory();
35 | Path dir = Files.createDirectories(Paths.get(sdcard.getAbsolutePath(), dirName));
36 | File file = new File(dir.toFile(), filename);
37 | writer = new BufferedWriter(new FileWriter(file, true));
38 |
39 | // Important: adding comment with metadata before isClosed is changed
40 | // writer.write("# " + Build.MODEL);
41 | // writer.write("\n");
42 | // writer.write("# " + Build.VERSION.SDK_INT);
43 | // writer.write("\n");
44 | //
45 | //
46 | // CameraManager manager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE);
47 | // try {
48 | // String[] idList = manager.getCameraIdList();
49 | //
50 | // Map levels = new HashMap() {{
51 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY, "LEGACY");
52 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED, "LIMITED");
53 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL, "FULL");
54 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL, "EXTERNAL");
55 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3, "LEVEL_3");
56 | // }};
57 | //
58 | //
59 | // int maxCameraCnt = idList.length;
60 | // writer.write("# " + maxCameraCnt);
61 | // writer.write("\n");
62 | // for (int index = 0; index < maxCameraCnt; index++) {
63 | // String cameraId = manager.getCameraIdList()[index];
64 | // CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
65 | // String deviceLevel = levels.get(characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL));
66 | // String source = characteristics.get(
67 | // CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE
68 | // ) == CameraMetadata.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME ? "REALTIME" : "UNKNOWN";
69 | // writer.write("# " + source + " " + deviceLevel);
70 | // writer.write("\n");
71 | // }
72 | // } catch (CameraAccessException e) {
73 | // e.printStackTrace();
74 | // }
75 |
76 |
77 | isClosed = false;
78 | }
79 |
80 | public void logLine(String line) throws IOException {
81 | writer.write(line);
82 | writer.write("\n");
83 | };
84 |
85 | public void close() {
86 | try {
87 | isClosed = true;
88 | writer.close();
89 | } catch (IOException e) {
90 | e.printStackTrace();
91 | }
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/ClientInfo.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import java.net.InetAddress;
20 |
21 | /**
22 | * Utility immutable class for providing accessors for client name, address, ip local address
23 | * ending, current best accuracy, and last known heartbeat.
24 | */
25 | public final class ClientInfo {
26 | private final String name;
27 | private final InetAddress address;
28 | private final long offsetNs;
29 | private final long syncAccuracyNs;
30 | private final long lastHeartbeatNs;
31 |
32 | static ClientInfo create(
33 | String name, InetAddress address, long offset, long syncAccuracy, long lastHeartbeat) {
34 | return new ClientInfo(name, address, offset, syncAccuracy, lastHeartbeat);
35 | }
36 |
37 | static ClientInfo create(String name, InetAddress address) {
38 | return new ClientInfo(
39 | name, address, /*offsetNs=*/ 0, /*syncAccuracyNs=*/ 0, /*lastHeartbeatNs=*/ 0);
40 | }
41 |
42 | private ClientInfo(
43 | String name, InetAddress address, long offsetNs, long syncAccuracyNs, long lastHeartbeatNs) {
44 | this.name = name;
45 | this.address = address;
46 | this.offsetNs = offsetNs;
47 | this.syncAccuracyNs = syncAccuracyNs;
48 | this.lastHeartbeatNs = lastHeartbeatNs;
49 | }
50 |
51 | public String name() {
52 | return name;
53 | }
54 |
55 | public InetAddress address() {
56 | return address;
57 | }
58 |
59 | /**
60 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain. The client can
61 | * take their local_time to get leader_time via: local_time (leader - client) = leader_time.
62 | */
63 | public long offset() {
64 | return offsetNs;
65 | }
66 |
67 | /**
68 | * The worst case error in the clock domains between leader and client for this response, in
69 | * nanoseconds of the AP SystemClock domain.
70 | */
71 | public long syncAccuracy() {
72 | return syncAccuracyNs;
73 | }
74 |
75 | /* The last time a client heartbeat was detected. */
76 | public long lastHeartbeat() {
77 | return lastHeartbeatNs;
78 | }
79 |
80 | @Override
81 | public String toString() {
82 | return String.format("%s[%.2f ms]", name(), TimeUtils.nanosToMillis((double) syncAccuracy()));
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/NetworkHelpers.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.net.wifi.WifiManager;
20 | import java.net.Inet4Address;
21 | import java.net.InetAddress;
22 | import java.net.NetworkInterface;
23 | import java.net.SocketException;
24 | import java.net.UnknownHostException;
25 | import java.util.Collections;
26 | import java.util.List;
27 |
28 | /** Helper functions for determining local IP address and host IP address on the network. */
29 | public final class NetworkHelpers {
30 | private final WifiManager wifiManager;
31 |
32 | /**
33 | * Constructor providing the wifi manager for determining hotspot leader address.
34 | *
35 | * @param wifiManager via '(WifiManager) context.getSystemService(Context.WIFI_SERVICE);''
36 | */
37 | public NetworkHelpers(WifiManager wifiManager) {
38 | this.wifiManager = wifiManager;
39 | }
40 |
41 | /**
42 | * Returns the IP address of the hotspot host. Requires ACCESS_WIFI_STATE permission. Note: This
43 | * may not work on several devices.
44 | *
45 | * @return IP address of the hotspot host.
46 | */
47 | public InetAddress getHotspotServerAddress() throws SocketException, UnknownHostException {
48 | if (wifiManager.isWifiEnabled()) {
49 | // Return the DHCP server address, which is the hotspot ip address.
50 | int serverAddress = wifiManager.getDhcpInfo().serverAddress;
51 | // DhcpInfo integer addresses are Little Endian and InetAddresses.fromInteger() are Big Endian
52 | // so reverse the bytes before converting from Integer.
53 | byte[] addressBytes = {
54 | (byte) (0xff & serverAddress),
55 | (byte) (0xff & (serverAddress >> 8)),
56 | (byte) (0xff & (serverAddress >> 16)),
57 | (byte) (0xff & (serverAddress >> 24))
58 | };
59 | return InetAddress.getByAddress(addressBytes);
60 | }
61 | // If wifi is disabled, then this is the hotspot host, return the local ip address.
62 | return getIPAddress();
63 | }
64 |
65 | /**
66 | * Finds this devices's IPv4 address that is not localhost and not on a dummy interface.
67 | *
68 | * @return the String IP address on success.
69 | * @throws SocketException on failure to find a suitable IP address.
70 | */
71 | public static InetAddress getIPAddress() throws SocketException {
72 | List interfaces = Collections.list(NetworkInterface.getNetworkInterfaces());
73 | for (NetworkInterface intf : interfaces) {
74 | for (InetAddress addr : Collections.list(intf.getInetAddresses())) {
75 | if (!addr.isLoopbackAddress()
76 | && !intf.getName().equals("dummy0")
77 | && addr instanceof Inet4Address) {
78 | return addr;
79 | }
80 | }
81 | }
82 | throw new SocketException("No viable IP Network addresses found.");
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/RpcCallback.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /** Interface for RPC callbacks, the base methods used for communicating between devices. */
20 | public interface RpcCallback {
21 |
22 | /**
23 | * The callback method called when an RPC is received.
24 | *
25 | * @param payload Contains the payload sent by the RPC.
26 | */
27 | void call(String payload);
28 | }
29 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SimpleNetworkTimeProtocol.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.util.Log;
20 | import java.io.IOException;
21 | import java.net.DatagramPacket;
22 | import java.net.DatagramSocket;
23 | import java.net.InetAddress;
24 | import java.net.SocketTimeoutException;
25 | import java.nio.ByteBuffer;
26 | import java.nio.LongBuffer;
27 | import java.util.HashSet;
28 | import java.util.Set;
29 | import java.util.concurrent.ExecutorService;
30 | import java.util.concurrent.Executors;
31 | import java.util.concurrent.TimeUnit;
32 |
33 | /**
34 | * Simple Network Time Protocol (SNTP) for clock synchronization logic between leader and clients.
35 | * This implements the leader half of the protocol, with SntpListener implementing the client side.
36 | *
37 | *
Provides a doSNTP function allowing the leader to initiate synchronization with a client
38 | * address. The SntpListener class is used by the clients to handle responding to these messages.
39 | */
40 | public class SimpleNetworkTimeProtocol implements AutoCloseable {
41 | private static final String TAG = "SimpleNetworkTimeProtocol";
42 |
43 | private final DatagramSocket nptpSocket;
44 | private final int nptpPort;
45 |
46 | /** Sequentially manages SNTP synchronization of clients. */
47 | private final ExecutorService nptpExecutor = Executors.newSingleThreadExecutor();
48 |
49 | /** Keeps track of SNTP client sync tasks already in the pipeline to avoid duplicate requests. */
50 | private final Set clientSyncTasks = new HashSet<>();
51 |
52 | private final Object clientSyncTasksLock = new Object();
53 | private final SoftwareSyncLeader leader;
54 | private final Ticker localClock;
55 |
56 | public SimpleNetworkTimeProtocol(
57 | Ticker localClock, DatagramSocket nptpSocket, int nptpPort, SoftwareSyncLeader leader) {
58 | this.localClock = localClock;
59 | this.nptpSocket = nptpSocket;
60 | this.nptpPort = nptpPort;
61 | this.leader = leader;
62 | }
63 |
64 | /**
65 | * Check if requesting client is already in the queue. If not, then submit a new task to do n-PTP
66 | * synchronization with that client. Synchronization involves sending and receiving messages on
67 | * the nptp socket, calculating the clock offsetNs, and finally sending an rpc to update the
68 | * offsetNs on the client.
69 | */
70 | @SuppressWarnings("FutureReturnValueIgnored")
71 | void submitNewSyncRequest(final InetAddress clientAddress) {
72 | // Skip if we have already enqueued a sync task with this client.
73 | synchronized (clientSyncTasksLock) {
74 | if (clientSyncTasks.contains(clientAddress)) {
75 | Log.w(TAG, "Already queued sync with " + clientAddress + ", skipping.");
76 | return;
77 | } else {
78 | clientSyncTasks.add(clientAddress);
79 | }
80 | }
81 |
82 | // Add SNTP request to executor queue.
83 | nptpExecutor.submit(
84 | () -> {
85 | // If the client no longer exists, no need to synchronize.
86 | if (!leader.getClients().containsKey(clientAddress)) {
87 | Log.w(TAG, "Client was removed, exiting SNTP routine.");
88 | return true;
89 | }
90 |
91 | Log.d(TAG, "Starting sync with client" + clientAddress);
92 | // Calculate clock offsetNs between client and leader using a naive
93 | // version of the precision time protocol (SNTP).
94 | SntpOffsetResponse response = doSNTP(clientAddress);
95 |
96 | if (response.status()) {
97 | // Apply local offsetNs to bestOffset so everyone has the same offsetNs.
98 | final long alignedOffset = response.offsetNs() + leader.getLeaderFromLocalNs();
99 |
100 | // Update client sync accuracy locally.
101 | leader.updateClientWithOffsetResponse(clientAddress, response);
102 |
103 | // Send an RPC to update the offsetNs on the client.
104 | Log.d(TAG, "Sending offsetNs update to " + clientAddress + ": " + alignedOffset);
105 | leader.sendRpc(
106 | SyncConstants.METHOD_OFFSET_UPDATE, String.valueOf(alignedOffset), clientAddress);
107 | }
108 |
109 | // Pop client from the queue regardless of success state. Clients will be added back in
110 | // the queue as needed based on their state at the next heartbeat.
111 | synchronized (clientSyncTasksLock) {
112 | clientSyncTasks.remove(clientAddress);
113 | }
114 |
115 | if (response.status()) {
116 | leader.onRpc(SyncConstants.METHOD_MSG_OFFSET_UPDATED, clientAddress.toString());
117 | }
118 |
119 | return response.status();
120 | });
121 | }
122 |
123 | /**
124 | * Performs Min filter SNTP synchronization with the client over the socket using UDP.
125 | *
126 | *
Naive PTP protocol is as follows:
127 | *
128 | *
[1]At time t0 in the leader clock domain, Leader sends the message (t0).
129 | *
130 | *
[2]At time t1 in the client clock domain, Client receives the message (t0).
131 | *
132 | *
[3]At time t2 in the client clock domain, Client sends the message (t0,t1,t2).
133 | *
134 | *
[4]At time t3 in the leader clock domain, Leader receives the message (t0,t1,t2).
135 | *
136 | *
Final Clock offsetNs is calculated using the message with the smallest round-trip latency.
142 | *
143 | * @param clientAddress The client InetAddress to perform synchronization with.
144 | * @return SntpOffsetResponse containing the offsetNs and sync accuracy with the client.
145 | */
146 | private SntpOffsetResponse doSNTP(InetAddress clientAddress) throws IOException {
147 | final int longSize = Long.SIZE / Byte.SIZE;
148 | byte[] buf = new byte[longSize * 3];
149 | long bestLatency = Long.MAX_VALUE; // Start with initial high round trip
150 | long bestOffset = 0;
151 | // If there are several failed SNTP round trip sync messages, fail out.
152 | int missingMessageCountdown = 10;
153 | SntpOffsetResponse failureResponse =
154 | SntpOffsetResponse.create(/*offset=*/ 0, /*syncAccuracy=*/ 0, false);
155 |
156 | for (int i = 0; i < SyncConstants.NUM_SNTP_CYCLES; i++) {
157 | // 1 - Send UDP SNTP message to the client with t0 at time t0.
158 | long t0 = localClock.read();
159 | ByteBuffer t0bytebuffer = ByteBuffer.allocate(longSize);
160 | t0bytebuffer.putLong(t0);
161 | nptpSocket.send(new DatagramPacket(t0bytebuffer.array(), longSize, clientAddress, nptpPort));
162 |
163 | // Steps 2 and 3 happen on client side B.
164 | // 4 - Recv UDP message with t0,t0',t1 at time t1'.
165 | DatagramPacket packet = new DatagramPacket(buf, buf.length);
166 | try {
167 | nptpSocket.receive(packet);
168 | } catch (SocketTimeoutException e) {
169 | // If we didn't receive a message in time, then skip this PTP pair and continue.
170 | Log.w(TAG, "UDP PTP message missing, skipping");
171 | missingMessageCountdown--;
172 | if (missingMessageCountdown <= 0) {
173 | Log.w(
174 | TAG, String.format("Missed too many messages, leaving doSNTP for %s", clientAddress));
175 | return failureResponse;
176 | }
177 | continue;
178 | }
179 | final long t3 = localClock.read();
180 |
181 | if (packet.getLength() != 3 * longSize) {
182 | Log.w(TAG, "Corrupted UDP message, skipping");
183 | continue;
184 | }
185 | ByteBuffer t3buffer = ByteBuffer.allocate(longSize * 3);
186 | t3buffer.put(packet.getData(), 0, packet.getLength());
187 | t3buffer.flip();
188 | LongBuffer longbuffer = t3buffer.asLongBuffer();
189 | final long t0Msg = longbuffer.get();
190 | final long t1Msg = longbuffer.get();
191 | final long t2Msg = longbuffer.get();
192 |
193 | // Confirm that the received message contains the same t0 as the t0 from this cycle,
194 | // otherwise skip.
195 | if (t0Msg != t0) {
196 | Log.w(
197 | TAG,
198 | String.format(
199 | "Out of order PTP message received, skipping: Expected %d vs %d", t0, t0Msg));
200 |
201 | // Note: Wait or catch and throw away the next message to get back in sync.
202 | try {
203 | nptpSocket.receive(packet);
204 | } catch (SocketTimeoutException e) {
205 | // If still waiting, continue.
206 | }
207 | // Since this was an incorrect cycle, move on to a new cycle.
208 | continue;
209 | }
210 |
211 | final long timeOffset = ((t1Msg - t0) + (t2Msg - t3)) / 2;
212 | final long roundTripLatency = (t3 - t0) - (t2Msg - t1Msg);
213 |
214 | Log.v(
215 | TAG,
216 | String.format(
217 | "% 3d | PTP: %d,%d,%d,%d | Latency: %,.3f ms",
218 | i, t0, t1Msg, t2Msg, t3, TimeUtils.nanosToMillis((double) roundTripLatency)));
219 |
220 | if (roundTripLatency < bestLatency) {
221 | bestOffset = timeOffset;
222 | bestLatency = roundTripLatency;
223 | // If round trip latency is under minimum round trip latency desired, stop here.
224 | if (roundTripLatency < SyncConstants.MIN_ROUND_TRIP_LATENCY_NS) {
225 | break;
226 | }
227 | }
228 | }
229 |
230 | Log.v(
231 | TAG,
232 | String.format(
233 | "Client %s : SNTP best latency %,d ns, offsetNs %,d ns",
234 | clientAddress, bestLatency, bestOffset));
235 |
236 | return SntpOffsetResponse.create(bestOffset, bestLatency, true);
237 | }
238 |
239 | @Override
240 | public void close() {
241 | nptpExecutor.shutdown();
242 | // Wait up to 0.5 seconds for the executor service to finish.
243 | try {
244 | nptpExecutor.awaitTermination(500, TimeUnit.MILLISECONDS);
245 | } catch (InterruptedException e) {
246 | throw new IllegalStateException("SNTP Executor didn't close gracefully: " + e);
247 | }
248 | }
249 | }
250 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpListener.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.util.Log;
20 | import java.io.IOException;
21 | import java.net.DatagramPacket;
22 | import java.net.DatagramSocket;
23 | import java.net.SocketTimeoutException;
24 | import java.nio.ByteBuffer;
25 |
26 | /**
27 | * SNTP listener thread, which is expected to only run while the client is waiting for the leader to
28 | * synchronize with it (WAITING_FOR_LEADER state). The {@link SimpleNetworkTimeProtocol} class is
29 | * used by the leader to send the SNTP messages that this thread listens for.
30 | */
31 | public class SntpListener extends Thread {
32 |
33 | private static final String TAG = "SntpListener";
34 | private boolean running;
35 | private final DatagramSocket nptpSocket;
36 | private final int nptpPort;
37 | private final Ticker localClock;
38 |
39 | public SntpListener(Ticker localClock, DatagramSocket nptpSocket, int nptpPort) {
40 | this.localClock = localClock;
41 | this.nptpSocket = nptpSocket;
42 | this.nptpPort = nptpPort;
43 | }
44 |
45 | public void stopRunning() {
46 | running = false;
47 | }
48 |
49 | @Override
50 | public void run() {
51 | running = true;
52 |
53 | Log.w(TAG, "Starting SNTP Listener thread.");
54 |
55 | byte[] buf = new byte[SyncConstants.SNTP_BUFFER_SIZE];
56 | while (running && !nptpSocket.isClosed()) {
57 | DatagramPacket packet = new DatagramPacket(buf, buf.length);
58 | try {
59 | // Listen for PTP messages.
60 | nptpSocket.receive(packet);
61 |
62 | // 2 (B) - Recv UDP message with t0 at time t0'.
63 | long t0r = localClock.read();
64 |
65 | final int longSize = Long.SIZE / Byte.SIZE;
66 |
67 | if (packet.getLength() != longSize) {
68 | Log.e(
69 | TAG,
70 | "Received UDP message with incorrect packet length "
71 | + packet.getLength()
72 | + ", skipping.");
73 | continue;
74 | }
75 |
76 | // 3 (B) - Send UDP message with t0,t0',t1 at time t1.
77 | long t1 = localClock.read();
78 | ByteBuffer buffer = ByteBuffer.allocate(3 * longSize);
79 | buffer.put(packet.getData(), 0, longSize);
80 | buffer.putLong(longSize, t0r);
81 | buffer.putLong(2 * longSize, t1);
82 | byte[] bufferArray = buffer.array();
83 |
84 | // Send SNTP response back.
85 | DatagramPacket response =
86 | new DatagramPacket(bufferArray, bufferArray.length, packet.getAddress(), nptpPort);
87 | nptpSocket.send(response);
88 | } catch (SocketTimeoutException e) {
89 | // It is normal to time out most of the time, continue.
90 | } catch (IOException e) {
91 | if (nptpSocket.isClosed()) {
92 | // Stop here if socket is closed.
93 | return;
94 | }
95 | throw new IllegalStateException("SNTP Thread didn't close gracefully: " + e);
96 | }
97 | }
98 | Log.w(TAG, "SNTP Listener thread finished.");
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpOffsetResponse.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /** AutoValue class for SNTP offsetNs, synchronization accuracy and status. */
20 | public final class SntpOffsetResponse {
21 | private final long offsetNs;
22 | private final long syncAccuracyNs;
23 | private final boolean status;
24 |
25 | static SntpOffsetResponse create(long offset, long syncAccuracy, boolean status) {
26 | return new SntpOffsetResponse(offset, syncAccuracy, status);
27 | }
28 |
29 | private SntpOffsetResponse(long offsetNs, long syncAccuracyNs, boolean status) {
30 | this.offsetNs = offsetNs;
31 | this.syncAccuracyNs = syncAccuracyNs;
32 | this.status = status;
33 | }
34 |
35 | /**
36 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain.
37 | *
38 | *
The client can take their local_time to get leader_time via: local_time (leader - client) =
39 | * leader_time.
40 | */
41 | public long offsetNs() {
42 | return offsetNs;
43 | }
44 |
45 | /**
46 | * The worst case error in the clock domains between leader and client for this response, in
47 | * nanoseconds of the AP SystemClock domain.
48 | */
49 | public long syncAccuracyNs() {
50 | return syncAccuracyNs;
51 | }
52 |
53 | /** The success status of this response. */
54 | public boolean status() {
55 | return status;
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SoftwareSyncBase.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.os.HandlerThread;
20 | import android.util.Log;
21 | import java.io.Closeable;
22 | import java.io.IOException;
23 | import java.net.BindException;
24 | import java.net.DatagramPacket;
25 | import java.net.DatagramSocket;
26 | import java.net.InetAddress;
27 | import java.net.InetSocketAddress;
28 | import java.net.SocketException;
29 | import java.net.SocketTimeoutException;
30 | import java.nio.ByteBuffer;
31 | import java.util.HashMap;
32 | import java.util.Map;
33 | import java.util.concurrent.ExecutorService;
34 | import java.util.concurrent.Executors;
35 |
36 | /**
37 | * SoftwareSyncBase is the abstract base class to SoftwareSyncLeader and SoftwareSyncClient, holding
38 | * shared objects such as UDP ports and sockets, local client information and methods for starting
39 | * and stopping shared threads such as the rpc socket thread.
40 | *
41 | *
When the user is finished they should call the idempotent method close().
42 | */
43 | public abstract class SoftwareSyncBase implements Closeable, TimeDomainConverter {
44 | static final String TAG = "SoftwareSyncBase";
45 | private final ClientInfo localClientInfo; // Client info for this device.
46 | private final InetAddress leaderAddress;
47 | final Ticker localClock;
48 |
49 | /**
50 | * Offset to convert local time to leader time. leader_time = local_elapsed_time -
51 | * leader_from_local.
52 | */
53 | private long leaderFromLocalNs = 0;
54 |
55 | /* SNTP Setup */
56 | final int sntpPort;
57 | final DatagramSocket sntpSocket;
58 |
59 | /* RPC Setup. */
60 | private final int rpcPort;
61 | private final DatagramSocket rpcSocket;
62 | private final RpcThread rpcListenerThread;
63 | final Map rpcMap = new HashMap<>();
64 | /** Handle onRPC events on a separate thread. */
65 | private final ExecutorService rpcExecutor = Executors.newSingleThreadExecutor();
66 |
67 | SoftwareSyncBase(String name, Ticker localClock, InetAddress address, InetAddress leaderAddress) {
68 | this.rpcPort = SyncConstants.RPC_PORT;
69 | this.sntpPort = SyncConstants.SNTP_PORT;
70 | this.localClock = localClock;
71 |
72 | // Set up local ClientInfo from the provided address.
73 | localClientInfo = ClientInfo.create(name, address);
74 |
75 | // Leader device ip address is provided by the user.
76 | this.leaderAddress = leaderAddress;
77 |
78 | // Open sockets and start communication threads between leader and client devices.
79 | try {
80 | rpcSocket = new DatagramSocket(null);
81 | rpcSocket.setReuseAddress(true);
82 | rpcSocket.setSoTimeout(SyncConstants.SOCKET_WAIT_TIME_MS);
83 | rpcSocket.bind(new InetSocketAddress(SyncConstants.RPC_PORT));
84 |
85 | sntpSocket = new DatagramSocket(null);
86 | sntpSocket.setReuseAddress(true);
87 | sntpSocket.setSoTimeout(SyncConstants.SOCKET_WAIT_TIME_MS);
88 | sntpSocket.bind(new InetSocketAddress(SyncConstants.SNTP_PORT));
89 |
90 | } catch (BindException e) {
91 | throw new IllegalArgumentException("Socket already in use, close app and restart: " + e);
92 | } catch (SocketException e) {
93 | throw new IllegalArgumentException("Unable to open Sockets: " + e);
94 | }
95 |
96 | // Start an RPC thread loop that listens for packets on the rpc socket, processes and calls
97 | // onRpc with the processed method and payload.
98 | rpcListenerThread = new RpcThread();
99 | rpcListenerThread.start();
100 | }
101 |
102 | /**
103 | * Returns leader synchronized time in nanoseconds. This is in the clock domain of the leader's
104 | * localClock (SystemClock.elapsedRealtimeNanos())
105 | */
106 | public long getLeaderTimeNs() {
107 | return leaderTimeForLocalTimeNs(localClock.read());
108 | }
109 |
110 | /**
111 | * Calculates the leader time associated with the given local time in nanoseconds. The local time
112 | * must be in the SystemClock.elapsedRealTimeNanos() localClock domain, nanosecond units. This
113 | * includes timestamps such as the sensor timestamp from the camera. leader_time =
114 | * local_elapsed_time_ns + leader_from_local_ns.
115 | *
116 | * @param localTimeNs given local time (local clock SystemClock.elapsedRealtimeNanos() domain).
117 | * @return leader synchronized time in nanoseconds.
118 | */
119 | @Override
120 | public long leaderTimeForLocalTimeNs(long localTimeNs) {
121 | return localTimeNs - leaderFromLocalNs;
122 | }
123 |
124 | public String getName() {
125 | return localClientInfo.name();
126 | }
127 |
128 | ClientInfo getLocalClientInfo() {
129 | return localClientInfo;
130 | }
131 |
132 | public InetAddress getLeaderAddress() {
133 | return leaderAddress;
134 | }
135 |
136 | /**
137 | * Returns get the localClock offsetNs between this devices local elapsed time and the leader in
138 | * nanoseconds.
139 | */
140 | public long getLeaderFromLocalNs() {
141 | return leaderFromLocalNs;
142 | }
143 |
144 | /** Set the offsetNs between this device's local elapsed time and the leader synchronized time. */
145 | void setLeaderFromLocalNs(long value) {
146 | leaderFromLocalNs = value;
147 | }
148 |
149 | void addPublicRpcCallbacks(Map callbacks) {
150 | for (Integer key : callbacks.keySet()) {
151 | if (key < SyncConstants.START_NON_SOFTWARESYNC_METHOD_IDS) {
152 | throw new IllegalArgumentException(
153 | String.format(
154 | "Given method id %s, User method ids must" + " be >= %s",
155 | key, SyncConstants.START_NON_SOFTWARESYNC_METHOD_IDS));
156 | }
157 | }
158 | rpcMap.putAll(callbacks);
159 | }
160 |
161 | /** Sends a message with arguments to the specified address over the rpc socket. */
162 | void sendRpc(int method, String arguments, InetAddress address) {
163 | byte[] messagePayload = arguments.getBytes();
164 | if (messagePayload.length + 4 > SyncConstants.RPC_BUFFER_SIZE) {
165 | throw new IllegalArgumentException(
166 | String.format(
167 | "RPC arguments too big %d v %d",
168 | messagePayload.length + 4, SyncConstants.RPC_BUFFER_SIZE));
169 | }
170 |
171 | byte[] fullPayload =
172 | ByteBuffer.allocate(messagePayload.length + 4).putInt(method).put(messagePayload).array();
173 |
174 | DatagramPacket packet = new DatagramPacket(fullPayload, fullPayload.length, address, rpcPort);
175 | try {
176 | rpcSocket.send(packet);
177 | } catch (IOException e) {
178 | throw new IllegalStateException("Error sending RPC packet.");
179 | }
180 | }
181 |
182 | /**
183 | * RPC thread loop that listens for packets on the rpc socket, processes and calls onRpc with the
184 | * processed method and payload.
185 | */
186 | private class RpcThread extends HandlerThread {
187 | private boolean running;
188 |
189 | RpcThread() {
190 | super("RpcListenerThread");
191 | }
192 |
193 | void stopRunning() {
194 | running = false;
195 | }
196 |
197 | @Override
198 | @SuppressWarnings("FutureReturnValueIgnored")
199 | public void run() {
200 | running = true;
201 |
202 | byte[] buf = new byte[SyncConstants.RPC_BUFFER_SIZE];
203 | while (running && !rpcSocket.isClosed()) {
204 | DatagramPacket packet = new DatagramPacket(buf, buf.length);
205 |
206 | try {
207 | // Wait for a client message.
208 | rpcSocket.receive(packet);
209 |
210 | // Separate packet string into int method and string payload
211 | // First 4 bytes is the integer method.
212 | ByteBuffer packetByteBuffer = ByteBuffer.wrap(packet.getData());
213 | int method = packetByteBuffer.getInt(); // From first 4 bytes.
214 | // Rest of the bytes are the payload.
215 | String payload = new String(packet.getData(), 4, packet.getLength() - 4);
216 |
217 | // Call onRpc with the method and payload in a separate thread.
218 | rpcExecutor.submit(() -> onRpc(method, payload));
219 |
220 | } catch (SocketTimeoutException e) {
221 | // Do nothing since this is a normal timeout of the receive.
222 | } catch (IOException e) {
223 | if (running || rpcSocket.isClosed()) {
224 | Log.w(TAG, "Shutdown arrived in middle of a socket receive, ignoring error.");
225 | } else {
226 | throw new IllegalStateException("Socket Receive/Send error: " + e);
227 | }
228 | }
229 | }
230 | }
231 | }
232 |
233 | /** Handle RPCs using the existing RPC map. */
234 | public void onRpc(int method, String payload) {
235 | RpcCallback callback = rpcMap.get(method);
236 | if (callback != null) {
237 | callback.call(payload);
238 | }
239 | }
240 |
241 | /**
242 | * Idempotent close that handles closing sockets, threads if they are open or running, etc. If a
243 | * user overrides this method it is expected make sure to call super as well.
244 | */
245 | @Override
246 | public void close() throws IOException {
247 | rpcListenerThread.stopRunning();
248 | rpcSocket.close();
249 | sntpSocket.close();
250 | }
251 | }
252 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SoftwareSyncClient.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.util.Log;
20 | import java.io.IOException;
21 | import java.net.InetAddress;
22 | import java.util.Map;
23 | import java.util.concurrent.Executors;
24 | import java.util.concurrent.ScheduledExecutorService;
25 | import java.util.concurrent.TimeUnit;
26 |
27 | /**
28 | * Client which registers and synchronizes clocks with SoftwareSyncLeader. This allows it to receive
29 | * messages and timestamps at which to simultaneously process commands such as capturing a photo at
30 | * the same time as the leader device.
31 | *
32 | *
The client can be in one of internal three states: Waiting for leader, registered but not
33 | * synced, and synced.
34 | *
35 | *
On instantiation, the client attempts to register with the leader, and is waiting for the
36 | * leader to respond.
37 | *
38 | *
Then, once it is registered but not yet synced, it requests SNTP synchronization with the
39 | * leader. Here it listens for offsetNs update rpc messages and sets it's offsetNs to that, thereby
40 | * synchronizing it's clock with leader to the precision requested.
41 | *
42 | *
Finally, once the leader responds with an time correction offsetNs, it enters the synced
43 | * state.
44 | *
45 | *
>User should handle thrown IOExceptions for networking. The most common cause for a thrown
46 | * exception is when the user closes the client down, but there is still a socket receive or a
47 | * periodic heartbeat send still in progress.
48 | */
49 | public class SoftwareSyncClient extends SoftwareSyncBase {
50 |
51 | /** Tracks the state of client synchronization with the leader. */
52 | private boolean synced;
53 |
54 | private final Object syncLock = new Object();
55 | private final ScheduledExecutorService heartbeatScheduler = Executors.newScheduledThreadPool(1);
56 |
57 | /**
58 | * Time of last leader response received in the clock domain of the leader's
59 | * SystemClock.elapsedRealTimeNanos().
60 | */
61 | private long lastLeaderResponseTimeNs;
62 |
63 | private long lastLeaderOffsetResponseTimeNs;
64 |
65 | private SntpListener sntpThread;
66 |
67 | public SoftwareSyncClient(
68 | String name,
69 | InetAddress address,
70 | InetAddress leaderAddress,
71 | Map rpcCallbacks) {
72 | this(name, new SystemTicker(), address, leaderAddress, rpcCallbacks);
73 | }
74 |
75 | @SuppressWarnings("FutureReturnValueIgnored")
76 | private SoftwareSyncClient(
77 | String name,
78 | Ticker localClock,
79 | InetAddress address,
80 | InetAddress leaderAddress,
81 | Map rpcCallbacks) {
82 | super(name, localClock, address, leaderAddress);
83 |
84 | // Add client-specific RPC callbacks.
85 | rpcMap.put(
86 | SyncConstants.METHOD_HEARTBEAT_ACK,
87 | payload -> {
88 | // Leader responded to heartbeat. update last response and change sync status as needed.
89 | lastLeaderResponseTimeNs = localClock.read();
90 | Log.v(TAG, "Heartbeat acknowledge received from leader.");
91 | updateState();
92 | });
93 | rpcMap.put(
94 | SyncConstants.METHOD_OFFSET_UPDATE,
95 | payload -> {
96 | lastLeaderOffsetResponseTimeNs = localClock.read();
97 |
98 | Log.d(TAG, "Received offsetNs update: (" + payload + "), stopping sntp sync request.");
99 | // Set the time offsetNs to the offsetNs passed in by the leader and update state.
100 | setLeaderFromLocalNs(Long.parseLong(payload));
101 | updateState();
102 | onRpc(SyncConstants.METHOD_MSG_OFFSET_UPDATED, Long.toString(getLeaderFromLocalNs()));
103 | });
104 |
105 | // Add callbacks passed by user.
106 | addPublicRpcCallbacks(rpcCallbacks);
107 |
108 | // Initial state is waiting to register with leader.
109 | reset();
110 |
111 | // Start periodically sending out a heartbeat to the leader.
112 | heartbeatScheduler.scheduleAtFixedRate(
113 | this::sendHeartbeat, 0, SyncConstants.HEARTBEAT_PERIOD_NS, TimeUnit.NANOSECONDS);
114 | }
115 |
116 | /* Resets the client synchronization state. */
117 | private void reset() {
118 | lastLeaderResponseTimeNs = 0;
119 | lastLeaderOffsetResponseTimeNs = 0;
120 | maybeStartSntpThread();
121 | updateState();
122 | }
123 |
124 | /**
125 | * Sends a heartbeat to the leader and waits for an acknowledge response. If there is a response,
126 | * it updates the last known leader response time. It then calls updateState() to see if there is
127 | * a state transition needed. It also posts a delayed request to heartbeatHandler to run this
128 | * again in HEARTBEAT_TIME_MS.
129 | */
130 | private void sendHeartbeat() {
131 | // First update current client state based on time since last response.
132 | updateState();
133 |
134 | // Generate heartbeat message containing the client address and the
135 | // string value of the synchronization state.
136 | final String heartbeatMsg;
137 | synchronized (syncLock) {
138 | // Note: Send messages using strings for simplicity.
139 | heartbeatMsg =
140 | String.format(
141 | "%s,%s,%s",
142 | getLocalClientInfo().name(),
143 | getLocalClientInfo().address().getHostAddress(),
144 | Boolean.toString(synced));
145 | }
146 |
147 | // Send heartbeat RPC to leader, expecting a METHOD_HEARTBEAT_ACK rpc back from leader.
148 | sendRpc(SyncConstants.METHOD_HEARTBEAT, heartbeatMsg, getLeaderAddress());
149 | }
150 |
151 | /**
152 | * Propagate state machine depending on lastLeaderResponseTimeNs and currentState. This should be
153 | * called periodically, such as every time a heartbeat is sent, and after it receives an offsetNs
154 | * update RPC.
155 | */
156 | private void updateState() {
157 | final long timestamp = localClock.read();
158 | final long timeSinceLastLeaderResponseNs = timestamp - lastLeaderResponseTimeNs;
159 | final long timeSinceLastLeaderOffsetResponseNs = timestamp - lastLeaderOffsetResponseTimeNs;
160 | updateState(timeSinceLastLeaderResponseNs, timeSinceLastLeaderOffsetResponseNs);
161 | }
162 |
163 | /**
164 | * SoftwareSyncClient only has two states: WAITING_FOR_LEADER (false) and SYNCED (true).
165 | *
166 | *
The state is determined based on the time since the last leader heartbeat acknowledge and
167 | * the time since the last offsetNs was given by the leader.
168 | *
169 | *
If the time since the leader has responded is longer than STALE_TIME_NS or the last offsetNs
170 | * received happened longer than STALE_OFFSET_TIME_NS ago, then transition to the not synced
171 | * WAITING_FOR_LEADER state. Otherwise, transition to the SYNCED state.
172 | */
173 | private void updateState(
174 | final long timeSinceLastLeaderResponseNs, final long timeSinceLastLeaderOffsetResponseNs) {
175 | final boolean newSyncState =
176 | (lastLeaderResponseTimeNs != 0
177 | && lastLeaderOffsetResponseTimeNs != 0
178 | && timeSinceLastLeaderResponseNs < SyncConstants.STALE_TIME_NS
179 | && timeSinceLastLeaderOffsetResponseNs < SyncConstants.STALE_OFFSET_TIME_NS);
180 | synchronized (syncLock) {
181 | if (newSyncState == synced) {
182 | return; // No state change, do nothing.
183 | }
184 |
185 | // Update synchronization state.
186 | synced = newSyncState;
187 |
188 | if (synced) { // WAITING_FOR_LEADER -> SYNCED.
189 | onRpc(SyncConstants.METHOD_MSG_SYNCING, null);
190 | } else { // SYNCED -> WAITING_FOR_LEADER.
191 | onRpc(SyncConstants.METHOD_MSG_WAITING_FOR_LEADER, null);
192 | }
193 | }
194 | }
195 |
196 | /** Start SNTP thread if it's not already running. */
197 | private void maybeStartSntpThread() {
198 | if (sntpThread == null || !sntpThread.isAlive()) {
199 | // Set up SNTP thread.
200 | sntpThread = new SntpListener(localClock, sntpSocket, sntpPort);
201 | sntpThread.start();
202 | }
203 | }
204 |
205 | /** Blocking stop of SNTP Thread if it's not already stopped. */
206 | private void maybeStopSntpThread() {
207 | if (sntpThread != null && sntpThread.isAlive()) {
208 | sntpThread.stopRunning();
209 | // Wait for thread to finish.
210 | try {
211 | sntpThread.join();
212 | } catch (InterruptedException e) {
213 | throw new IllegalStateException("SNTP Thread didn't close gracefully: " + e);
214 | }
215 | }
216 | }
217 |
218 | @Override
219 | public void close() throws IOException {
220 | maybeStopSntpThread();
221 | // Stop the heartbeat scheduler.
222 | heartbeatScheduler.shutdown();
223 | try {
224 | heartbeatScheduler.awaitTermination(1, TimeUnit.SECONDS);
225 | } catch (InterruptedException e) {
226 | Thread.currentThread().interrupt(); // Restore the interrupted status.
227 | // Should only happen on app shutdown, fall out and continue.
228 | }
229 |
230 | super.close();
231 | }
232 | }
233 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SoftwareSyncLeader.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.util.Log;
20 | import java.io.IOException;
21 | import java.net.InetAddress;
22 | import java.net.UnknownHostException;
23 | import java.util.Arrays;
24 | import java.util.Collections;
25 | import java.util.HashMap;
26 | import java.util.Iterator;
27 | import java.util.List;
28 | import java.util.Map;
29 | import java.util.Map.Entry;
30 | import java.util.concurrent.ExecutorService;
31 | import java.util.concurrent.Executors;
32 | import java.util.concurrent.ScheduledExecutorService;
33 | import java.util.concurrent.TimeUnit;
34 |
35 | /**
36 | * Leader which listens for registrations from SoftwareSyncClients, allowing it to broadcast times
37 | * at which both itself and clients will simultaneously perform actions.
38 | *
39 | *
SoftwareSync assumes that clients are connected to a leader wifi hotspot created by the device
40 | * this instance is running on.
41 | *
42 | *
The leader listens for client registrations and keeps track of connected clients. It also
43 | * listens for SNTP synchronization requests and processes them in a queue. Once it has determined
44 | * the offsetNs it sends an rpc message to the client, thereby synchronizing the client's clock with
45 | * the leader's to the precision requested.
46 | */
47 | public class SoftwareSyncLeader extends SoftwareSyncBase {
48 | /** List of connected clients. */
49 | private final Map clients = new HashMap<>();
50 |
51 | private final Object clientsLock = new Object();
52 |
53 | /** Keeps track of how long since each client heartbeat was received, removing when stale. */
54 | private final ScheduledExecutorService staleClientChecker = Executors.newScheduledThreadPool(1);
55 |
56 | /** Send RPC messages on a separate thread, avoiding Network on Main Thread exceptions. */
57 | private final ExecutorService rpcMessageExecutor = Executors.newSingleThreadExecutor();
58 |
59 | /** Manages SNTP synchronization of clients. */
60 | private final SimpleNetworkTimeProtocol sntp;
61 |
62 | public SoftwareSyncLeader(
63 | String name, long initialTime, InetAddress address, Map rpcCallbacks) {
64 | this(name, new SystemTicker(), initialTime, address, rpcCallbacks);
65 | }
66 |
67 | @SuppressWarnings("FutureReturnValueIgnored")
68 | private SoftwareSyncLeader(
69 | String name,
70 | Ticker localClock,
71 | long initialTime,
72 | InetAddress address,
73 | Map rpcCallbacks) {
74 | // Note: Leader address is required to be the same as local address.
75 | super(name, localClock, address, address);
76 |
77 | // Set up the offsetNs so that the leader synchronized time (via getLeaderTimeNs()) on all
78 | // devices
79 | // runs starting from the initial time given. When initialTimeNs is zero the
80 | // leader synchronized time is the default of localClock, ie. the time since boot of the leader
81 | // device.
82 | // For convenience, all devices could instead be shifted to the leader device UTC time,
83 | // ex. initialTimeNs = TimeUtils.millisToNanos(System.currentTimeMillis())
84 | setLeaderFromLocalNs(localClock.read() - initialTime);
85 |
86 | // Add client-specific RPC callbacks.
87 | rpcMap.put(
88 | SyncConstants.METHOD_HEARTBEAT,
89 | payload -> {
90 | // Received heartbeat from client, send back an acknowledge and then
91 | // check the client state and add to sntp queue if needed.
92 | Log.v(TAG, "Heartbeat received from client: " + payload);
93 | try {
94 | processHeartbeatRpc(payload);
95 | } catch (UnknownHostException e) {
96 | Log.e(TAG, "Processed heartbeat with corrupt host address: " + payload);
97 | }
98 | });
99 |
100 | // Add callbacks passed by user.
101 | addPublicRpcCallbacks(rpcCallbacks);
102 |
103 | // Set up SNTP instance for synchronizing with clients.
104 | sntp = new SimpleNetworkTimeProtocol(localClock, sntpSocket, SyncConstants.SNTP_PORT, this);
105 |
106 | // Start periodically checking for stale clients and removing as needed.
107 | staleClientChecker.scheduleAtFixedRate(
108 | this::removeStaleClients, 0, SyncConstants.STALE_TIME_NS, TimeUnit.NANOSECONDS);
109 | }
110 |
111 | public Map getClients() {
112 | synchronized (clientsLock) {
113 | return Collections.unmodifiableMap(clients);
114 | }
115 | }
116 |
117 | /**
118 | * Checks if the address is already associated with one of the clients in the list of tracked
119 | * clients. If so, just update the last heartbeat, otherwise create a new client entry in the
120 | * list.
121 | */
122 | private void addOrUpdateClient(String name, InetAddress address) {
123 | // Check if it's a new client, so we don't add again.
124 | synchronized (clientsLock) {
125 | boolean clientExists = clients.containsKey(address);
126 | // Add or replace entry with an updated ClientInfo.
127 | long offsetNs = 0;
128 | long syncAccuracyNs = 0;
129 | if (clientExists) {
130 | offsetNs = clients.get(address).offset();
131 | syncAccuracyNs = clients.get(address).syncAccuracy();
132 | }
133 | ClientInfo updatedClient =
134 | ClientInfo.create(name, address, offsetNs, syncAccuracyNs, localClock.read());
135 | clients.put(address, updatedClient);
136 |
137 | if (!clientExists) {
138 | // Notify via message on interface if client is new.
139 | onRpc(SyncConstants.METHOD_MSG_ADDED_CLIENT, updatedClient.name());
140 | }
141 | }
142 | }
143 |
144 | /** Removes clients whose last heartbeat was longer than STALE_TIME_NS ago. */
145 | private void removeStaleClients() {
146 | long t = localClock.read();
147 | synchronized (clientsLock) {
148 | // Use iterator to avoid concurrent modification exception.
149 | Iterator> clientIterator = clients.entrySet().iterator();
150 | while (clientIterator.hasNext()) {
151 | ClientInfo client = clientIterator.next().getValue();
152 | long timeSince = t - client.lastHeartbeat();
153 | if (timeSince > SyncConstants.STALE_TIME_NS) {
154 | Log.w(
155 | TAG,
156 | String.format(
157 | "Stale client %s : time since %,d seconds",
158 | client.name(), TimeUtils.nanosToSeconds(timeSince)));
159 |
160 | // Remove entry from the client list first.
161 | clientIterator.remove();
162 | // Client hasn't responded in a while, remove from list.
163 | onRpc(SyncConstants.METHOD_MSG_REMOVED_CLIENT, client.name());
164 | }
165 | }
166 | }
167 | }
168 |
169 | /** Finds and updates client sync accuracy within list. */
170 | void updateClientWithOffsetResponse(InetAddress clientAddress, SntpOffsetResponse response) {
171 | // Update client sync accuracy locally.
172 | synchronized (clientsLock) {
173 | if (!clients.containsKey(clientAddress)) {
174 | Log.w(TAG, "Tried to update a client info that is no longer in the list, Skipping.");
175 | return;
176 | }
177 | final ClientInfo client = clients.get(clientAddress);
178 | ClientInfo updatedClient =
179 | ClientInfo.create(
180 | client.name(),
181 | client.address(),
182 | response.offsetNs(),
183 | response.syncAccuracyNs(),
184 | client.lastHeartbeat());
185 | clients.put(client.address(), updatedClient);
186 | }
187 | }
188 |
189 | /**
190 | * Sends an RPC to every client in the leader's clients list.
191 | *
192 | * @param method int type of RPC (in {@link SyncConstants}).
193 | * @param payload String payload.
194 | */
195 | @SuppressWarnings("FutureReturnValueIgnored")
196 | private void internalBroadcastRpc(int method, String payload) {
197 | // Send RPC message to all clients and call onRPC of self as well.
198 | synchronized (clientsLock) {
199 | for (InetAddress address : clients.keySet()) {
200 | rpcMessageExecutor.submit(() -> sendRpc(method, payload, address));
201 | }
202 | }
203 |
204 | // Also call onRpc for self (leader).
205 | onRpc(method, payload);
206 | }
207 |
208 | /**
209 | * Public-facing broadcast RPC to all current clients, for non-softwaresync RPC methods only.
210 | *
211 | * @param method int type of RPC, must be greater than {@link
212 | * SyncConstants#START_NON_SOFTWARESYNC_METHOD_IDS}.
213 | * @param payload String payload.
214 | */
215 | public void broadcastRpc(int method, String payload) {
216 | if (method < SyncConstants.START_NON_SOFTWARESYNC_METHOD_IDS) {
217 | throw new IllegalArgumentException(
218 | String.format(
219 | "Given method id %s, User method ids must" + " be >= %s",
220 | method, SyncConstants.START_NON_SOFTWARESYNC_METHOD_IDS));
221 | }
222 | internalBroadcastRpc(method, payload);
223 | }
224 |
225 | @Override
226 | public void close() throws IOException {
227 | sntp.close();
228 | staleClientChecker.shutdown();
229 | try {
230 | // Wait up to 0.5 seconds for this to close.
231 | staleClientChecker.awaitTermination(500, TimeUnit.MILLISECONDS);
232 | } catch (InterruptedException e) {
233 | Thread.currentThread().interrupt(); // Restore the interrupted status.
234 | // Should only happen on app shutdown, fall out and continue.
235 | }
236 | super.close();
237 | }
238 |
239 | /**
240 | * Process a heartbeat rpc call from a client by responding with a heartbeat acknowledge, adding
241 | * or updating the client in the tracked clients list, and submitting a new SNTP sync request if
242 | * the client state is not yet synchronized.
243 | *
244 | * @param payload format of "ClientName,ClientAddress,ClientState"
245 | */
246 | private void processHeartbeatRpc(String payload) throws UnknownHostException {
247 | List parts = Arrays.asList(payload.split(","));
248 | if (parts.size() != 3) {
249 | Log.e(
250 | TAG,
251 | "Heartbeat message has the wrong format, expected 3 comma-delimitted parts: "
252 | + payload
253 | + ". Skipping.");
254 | return;
255 | }
256 | String clientName = parts.get(0);
257 | InetAddress clientAddress = InetAddress.getByName(parts.get(1));
258 | boolean clientSyncState = Boolean.parseBoolean(parts.get(2));
259 |
260 | // Send heartbeat acknowledge RPC back to client first, containing the same payload.
261 | sendRpc(SyncConstants.METHOD_HEARTBEAT_ACK, payload, clientAddress);
262 |
263 | // Add or update client in clients.
264 | addOrUpdateClient(clientName, clientAddress);
265 |
266 | // If the client state is not yet synchronized, add it to the SNTP queue.
267 | if (!clientSyncState) {
268 | sntp.submitNewSyncRequest(clientAddress);
269 | }
270 | }
271 | }
272 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SyncConstants.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /** Ports and other constants used by SoftwareSync. */
20 | public class SyncConstants {
21 | public static final int SOCKET_WAIT_TIME_MS = 500;
22 |
23 | /** Heartbeat period between clients and leader. */
24 | public static final long HEARTBEAT_PERIOD_NS = TimeUtils.secondsToNanos(1);
25 |
26 | /**
27 | * Time until a lack of a heartbeat acknowledge from leader means a lost connection. Similarly the
28 | * time until a lack of a heartbeat from a client means the client is stale.
29 | */
30 | public static final long STALE_TIME_NS = 2 * HEARTBEAT_PERIOD_NS;
31 |
32 | /** Time until a given offsetNs by the leader is considered stale. */
33 | public static final long STALE_OFFSET_TIME_NS = TimeUtils.secondsToNanos(60 * 60);
34 |
35 | /** RPC. */
36 | public static final int RPC_PORT = 8244;
37 | public static final int RPC_BUFFER_SIZE = 1024;
38 |
39 | /** RPC Method ids.
40 | * [0 - 9,999] Reserved for SoftwareSync.
41 | * - [0 - 99] Synchronization-related.
42 | * - [100 - 199] Messages.
43 | * [10,000+ ] Available to user applications.
44 | */
45 | public static final int METHOD_HEARTBEAT = 1;
46 | public static final int METHOD_HEARTBEAT_ACK = 2;
47 | public static final int METHOD_OFFSET_UPDATE = 3;
48 |
49 | /* Define user RPC method ids using values greater or equal to this. */
50 | public static final int START_NON_SOFTWARESYNC_METHOD_IDS = 1_000;
51 |
52 | public static final int METHOD_MSG_ADDED_CLIENT = 1_101;
53 | public static final int METHOD_MSG_REMOVED_CLIENT = 1_102;
54 | public static final int METHOD_MSG_WAITING_FOR_LEADER = 1_103;
55 | public static final int METHOD_MSG_SYNCING = 1_104;
56 | public static final int METHOD_MSG_OFFSET_UPDATED = 1_105;
57 |
58 |
59 | /** Clock Sync - Simple Network Time Protocol (SNTP). */
60 | public static final int SNTP_PORT = 9428;
61 | public static final int SNTP_BUFFER_SIZE = 512;
62 | public static final int NUM_SNTP_CYCLES = 300;
63 | public static final long MIN_ROUND_TRIP_LATENCY_NS = TimeUtils.millisToNanos(1);
64 |
65 | private SyncConstants() {}
66 | }
67 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/SystemTicker.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | import android.os.SystemClock;
20 |
21 | /** Simple implementation of Ticker interface using the SystemClock elapsed realtime clock. */
22 | public class SystemTicker implements Ticker {
23 | @Override
24 | public long read() {
25 | return SystemClock.elapsedRealtimeNanos();
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/Ticker.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /** Interface for getting the time in nanoseconds, abstracting out accessing system time. */
20 | public interface Ticker {
21 | /* Returns the time in nanoseconds. */
22 | long read();
23 | }
24 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeDomainConverter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /**
20 | * Interface used to manage time domain conversion, implemented by {@link SoftwareSyncBase}. This
21 | * allows {@link ResultProcessor} to convert sensor timestamps to the synchronized time domain
22 | * without needing full access to the softwaresync object.
23 | */
24 | public interface TimeDomainConverter {
25 |
26 | /**
27 | * Calculates the leader time associated with the given local time in nanoseconds. The local time
28 | * must be in the SystemClock.elapsedRealTimeNanos() localClock domain, nanosecond units. This
29 | * includes timestamps such as the sensor timestamp from the camera. leader_time =
30 | * local_elapsed_time_ns + leader_from_local_ns.
31 | *
32 | * @param localTimeNs given local time (local clock SystemClock.elapsedRealtimeNanos() domain).
33 | * @return leader synchronized time in nanoseconds.
34 | */
35 | long leaderTimeForLocalTimeNs(long localTimeNs);
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync;
18 |
19 | /** Helper conversions between time scales. */
20 | public final class TimeUtils {
21 |
22 | public static double nanosToMillis(double nanos) {
23 | return nanos / 1_000_000L;
24 | }
25 |
26 | public static long nanosToSeconds(long nanos) {
27 | return nanos / 1_000_000_000L;
28 | }
29 |
30 | public static double nanosToSeconds(double nanos) {
31 | return nanos / 1_000_000_000L;
32 | }
33 |
34 | public static long millisToNanos(long millis) {
35 | return millis * 1_000_000L;
36 | }
37 |
38 | public static long secondsToNanos(int seconds) {
39 | return seconds * 1_000_000_000L;
40 | }
41 |
42 | private TimeUtils() {}
43 | }
44 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/phasealign/PeriodCalculator.java:
--------------------------------------------------------------------------------
1 | package com.googleresearch.capturesync.softwaresync.phasealign;
2 |
3 | import java.util.ArrayList;
4 | import java.util.Collections;
5 | import java.util.Comparator;
6 | import java.util.List;
7 | import java.util.Timer;
8 | import java.util.TimerTask;
9 | import java.util.concurrent.CountDownLatch;
10 | import java.util.stream.Collectors;
11 |
12 | public class PeriodCalculator {
13 | private final static long CALC_DURATION_MS = 10000;
14 | private volatile boolean shouldRegister;
15 | private ArrayList registeredTimestamps;
16 |
17 | public PeriodCalculator() {
18 | registeredTimestamps = new ArrayList<>();
19 | }
20 |
21 | // Blocking call, returns 0 in case of error
22 | public long getPeriodNs() throws InterruptedException {
23 | // Start recording timestamps
24 | registeredTimestamps = new ArrayList<>();
25 | shouldRegister = true;
26 | final CountDownLatch latch = new CountDownLatch(1);
27 | TimerTask task = new TimerTask() {
28 | public void run() {
29 | // Stop recording timestamps and calculate period
30 | shouldRegister = false;
31 | latch.countDown();
32 | }
33 | };
34 | Timer timer = new Timer("Timer");
35 |
36 | timer.schedule(task, CALC_DURATION_MS);
37 | latch.await();
38 | return calcPeriodNsClusters(getDiff(registeredTimestamps));
39 | }
40 |
41 | private ArrayList getDiff(ArrayList arrayList) {
42 | Long prev = 0L;
43 | ArrayList result = new ArrayList<>();
44 | for (Long aLong : arrayList) {
45 | if (prev == 0L) {
46 | prev = aLong;
47 | } else {
48 | result.add(aLong - prev);
49 | prev = aLong;
50 | }
51 | }
52 | return result;
53 | }
54 |
55 | private long calcPeriodNsClusters(ArrayList numArray) {
56 | long initEstimate = Collections.min(numArray);
57 | long nClust = Math.round(1.0 * Collections.max(numArray) / initEstimate);
58 | double weightedSum = 0L;
59 | for (int i = 0; i < nClust; i++) {
60 | int finalI = i;
61 | ArrayList clust = (ArrayList)numArray.stream().filter(
62 | x -> (x > (finalI + 0.5)*initEstimate) && (x < (finalI + 1.5)*initEstimate)
63 | ).collect(Collectors.toList());
64 | if (clust.size() > 0) {
65 | weightedSum += 1.0 * median(clust) / (i + 1) * clust.size();
66 | }
67 | }
68 | return Math.round(weightedSum / numArray.size());
69 | }
70 |
71 | private long calcPeriodNsMedian(ArrayList numArray) {
72 | return median(numArray);
73 | }
74 |
75 | private long median(ArrayList numArray) {
76 | Collections.sort(numArray);
77 | int middle = numArray.size() / 2;
78 | middle = middle > 0 && middle % 2 == 0 ? middle - 1 : middle;
79 | return numArray.get(middle);
80 | }
81 |
82 | public void onFrameTimestamp(long timestampNs) {
83 | // Register timestamp
84 | if (shouldRegister) {
85 | registeredTimestamps.add(timestampNs);
86 | }
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/phasealign/PhaseAligner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync.phasealign;
18 |
19 | /**
20 | * Calculates the current camera phase and returns the necessary exposure offset needed to align.
21 | * Runs in the clock domain of the timestamps passed in to `passTimestamp`.
22 | *
23 | *
Note: A single instance of PhaseAligner expects a constant period. Generally this will be for
24 | * an image stream with a constant frame duration and therefore a constant period and phase. This is
25 | * normally found for exposures under 33ms, where frame duration is clamped to 33.33ms. PhaseAligner
26 | * expects the user to have a fixed frame duration and associated period when running phase
27 | * alignment.
28 | *
29 | *
Users are expected to instantiate PhaseAligner instances from configuration files specific to
30 | * their device hardware using `newWithJSONConfig(...)`, or manually creating a config using
31 | * `loadFromConfig()`.
32 | *
33 | *
Users pass timestamps from the image sequence and are returned a PhaseResponse, which provides
34 | * both the current phase state and the estimated exposure and frame duration needed to align. By
35 | * inserting a new frame into the sequence with the estimated exposure and/or frame duration the
36 | * sequence should align closer to the desired phase. Given that the actual frame duration of the
37 | * inserted frame may be different from the estimated, several iterations may be required, and the
38 | * user can check to stope via when the phase response is aligned with `isAligned`.
39 | */
40 | public final class PhaseAligner {
41 | private final PhaseConfig config;
42 |
43 | /** Instantiate phase aligner using configuration options from a PhaseConfig proto. */
44 | public PhaseAligner(PhaseConfig config) {
45 | this.config = config;
46 | }
47 |
48 | /**
49 | * Given the latest image sequence timestamp, Responds with phase alignment information.
50 | *
51 | *
The response contains an estimated sensor exposure time or frame duration needed to align
52 | * align future frames to the desired goal phase, as well as the current alignment state.
53 | *
54 | * @param timestampNs timestamp in the same clock domain as used in the phase configuration. This
55 | * can be either the local clock domain or the software synchronized leader clock domain, as
56 | * long as it stays consistent for the duration of the phase aligner instance.
57 | * @return PhaseResponse containing the current phase state as well as the estimated sensor
58 | * exposure time and frame duration needed to align.
59 | */
60 | public final PhaseResponse passTimestamp(long timestampNs) {
61 | long phaseNs = timestampNs % config.periodNs();
62 | long diffFromGoalNs = config.goalPhaseNs() - phaseNs;
63 | boolean isAligned = Math.abs(diffFromGoalNs) < config.alignThresholdNs();
64 |
65 | /* Stop early if already aligned. */
66 | if (isAligned) {
67 | return PhaseResponse.builder()
68 | .setPhaseNs(phaseNs)
69 | .setExposureTimeToShiftNs(0)
70 | .setFrameDurationToShiftNs(0)
71 | .setDiffFromGoalNs(diffFromGoalNs)
72 | .setIsAligned(isAligned)
73 | .build();
74 | }
75 |
76 | /* Since we can only shift phase into the future, shift negative offsets over by one period. */
77 | long desiredPhaseOffsetNs = diffFromGoalNs;
78 | if (diffFromGoalNs < 0) {
79 | desiredPhaseOffsetNs += config.periodNs();
80 | }
81 |
82 | /*
83 | * Calculate the frame duration needed to align to the `goalPhaseNs`, using the linear
84 | * relationship between offset and phase shift. Since durations <= period have no effect, add
85 | * another period to the duration.
86 | */
87 | long frameDurationNsToShift = desiredPhaseOffsetNs / 2 + config.periodNs();
88 |
89 | /*
90 | * Convert to estimated shift exposure time by removing the average overheadNs. Note: The
91 | * majority of noise in phase alignment is due to this varying estimated overheadNs.
92 | *
93 | *
Due to the indirect control of frame duration, choosing offsets <= minExposure causes no
94 | * change in phase. To avoid this, a minimum offset is manually chosen for the specific device
95 | * architecture.
96 | */
97 | long exposureTimeNsToShift =
98 | Math.max(config.minExposureNs(), frameDurationNsToShift - config.overheadNs());
99 |
100 | return PhaseResponse.builder()
101 | .setPhaseNs(phaseNs)
102 | .setExposureTimeToShiftNs(exposureTimeNsToShift)
103 | .setFrameDurationToShiftNs(frameDurationNsToShift)
104 | .setDiffFromGoalNs(diffFromGoalNs)
105 | .setIsAligned(isAligned)
106 | .build();
107 | }
108 |
109 | /** Returns the configuration options used to set up the phase aligner. */
110 | public final PhaseConfig getConfig() {
111 | return config;
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/phasealign/PhaseConfig.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync.phasealign;
18 |
19 | import org.json.JSONException;
20 | import org.json.JSONObject;
21 |
22 | /**
23 | * A unique PhaseConfig is associated with a type of mobile device such as a Pixel 3, and a specific
24 | * capture mode, such as for a repeating request with exposures less than 33 ms.
25 | */
26 | public final class PhaseConfig {
27 | private long periodNs;
28 | private final long goalPhaseNs;
29 | private final long alignThresholdNs;
30 | private final long overheadNs;
31 | private final long minExposureNs;
32 |
33 | private PhaseConfig(
34 | long periodNs, long goalPhaseNs, long alignThresholdNs, long overheadNs, long minExposureNs) {
35 | this.periodNs = periodNs;
36 | this.goalPhaseNs = goalPhaseNs;
37 | this.alignThresholdNs = alignThresholdNs;
38 | this.overheadNs = overheadNs;
39 | this.minExposureNs = minExposureNs;
40 | }
41 |
42 | public void setPeriodNs(long periodNs) {
43 | this.periodNs = periodNs;
44 | }
45 |
46 | /** Parse from a given JSON. */
47 | public static PhaseConfig parseFromJSON(JSONObject json) throws JSONException {
48 | if (!json.has("periodNs")) {
49 | throw new IllegalArgumentException("Missing PeriodNs in JSON.");
50 | }
51 | if (!json.has("goalPhaseNs")) {
52 | throw new IllegalArgumentException("Missing GoalPhaseNs in JSON.");
53 | }
54 | if (!json.has("alignThresholdNs")) {
55 | throw new IllegalArgumentException("Missing AlignThresholdNs in JSON.");
56 | }
57 | if (!json.has("overheadNs")) {
58 | throw new IllegalArgumentException("Missing OverheadNs in JSON.");
59 | }
60 | if (!json.has("minExposureNs")) {
61 | throw new IllegalArgumentException("Missing MinExposureNs in JSON.");
62 | }
63 | return new PhaseConfig(
64 | json.getLong("periodNs"),
65 | json.getLong("goalPhaseNs"),
66 | json.getLong("alignThresholdNs"),
67 | json.getLong("overheadNs"),
68 | json.getLong("minExposureNs"));
69 | }
70 |
71 | /**
72 | * Nominal period between two frames in the image sequence. This is usually very close to the
73 | * `SENSOR_FRAME_DURATION`. The period is assumed to be constant for the duration of phase
74 | * alignment. ie. generally only in situations where exposure is < 33ms. If the exposure is fixed
75 | * at a longer exposure, it may also work with a tuned config.
76 | */
77 | public long periodNs() {
78 | return periodNs;
79 | }
80 |
81 | /* The target phase to align to, usually chosen as half the period. */
82 | public long goalPhaseNs() {
83 | return goalPhaseNs;
84 | }
85 |
86 | /* The threshold on the absolute difference between the goal and current
87 | * phases to be considered aligned. */
88 | public long alignThresholdNs() {
89 | return alignThresholdNs;
90 | }
91 |
92 | /**
93 | * Measured average difference between frame duration and sensor exposure time for exposures
94 | * greater than 33ms.
95 | */
96 | public long overheadNs() {
97 | return overheadNs;
98 | }
99 |
100 | /* Lower bound sensor exposure time that still causes a phase shift.
101 | * This is usually slightly less than the period by the overhead. In practice
102 | * this must be tuned. */
103 | public long minExposureNs() {
104 | return minExposureNs;
105 | }
106 |
107 | public String toString() {
108 | return "PhaseConfig{"
109 | + "periodNs="
110 | + periodNs
111 | + ", "
112 | + "goalPhaseNs="
113 | + goalPhaseNs
114 | + ", "
115 | + "alignThresholdNs="
116 | + alignThresholdNs
117 | + ", "
118 | + "overheadNs="
119 | + overheadNs
120 | + ", "
121 | + "minExposureNs="
122 | + minExposureNs
123 | + "}";
124 | }
125 | }
126 |
--------------------------------------------------------------------------------
/app/src/main/java/com/googleresearch/capturesync/softwaresync/phasealign/PhaseResponse.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2019 The Google Research Authors.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.googleresearch.capturesync.softwaresync.phasealign;
18 |
19 | /** PhaseAligner response providing image stream phase alignment results. */
20 | public final class PhaseResponse {
21 | private final long phaseNs;
22 | private final long exposureTimeToShiftNs;
23 | private final long frameDurationToShiftNs;
24 | private final long diffFromGoalNs;
25 | private final boolean isAligned;
26 |
27 | private PhaseResponse(
28 | long phaseNs,
29 | long exposureTimeToShiftNs,
30 | long frameDurationToShiftNs,
31 | long diffFromGoalNs,
32 | boolean isAligned) {
33 | this.phaseNs = phaseNs;
34 | this.exposureTimeToShiftNs = exposureTimeToShiftNs;
35 | this.frameDurationToShiftNs = frameDurationToShiftNs;
36 | this.diffFromGoalNs = diffFromGoalNs;
37 | this.isAligned = isAligned;
38 | }
39 |
40 | static Builder builder() {
41 | return new PhaseResponse.Builder();
42 | }
43 |
44 | /** Builder for new PhaseResponses. All parameters are required. */
45 | static final class Builder {
46 | private Long phaseNs;
47 | private Long exposureTimeToShiftNs;
48 | private Long frameDurationToShiftNs;
49 | private Long diffFromGoalNs;
50 | private Boolean isAligned;
51 |
52 | Builder() {}
53 |
54 | Builder setPhaseNs(long phaseNs) {
55 | this.phaseNs = phaseNs;
56 | return this;
57 | }
58 |
59 | Builder setExposureTimeToShiftNs(long exposureTimeToShiftNs) {
60 | this.exposureTimeToShiftNs = exposureTimeToShiftNs;
61 | return this;
62 | }
63 |
64 | Builder setFrameDurationToShiftNs(long frameDurationToShiftNs) {
65 | this.frameDurationToShiftNs = frameDurationToShiftNs;
66 | return this;
67 | }
68 |
69 | Builder setDiffFromGoalNs(long diffFromGoalNs) {
70 | this.diffFromGoalNs = diffFromGoalNs;
71 | return this;
72 | }
73 |
74 | Builder setIsAligned(boolean isAligned) {
75 | this.isAligned = isAligned;
76 | return this;
77 | }
78 |
79 | PhaseResponse build() {
80 | String missing = "";
81 | if (this.phaseNs == null) {
82 | missing += " phaseNs";
83 | }
84 | if (this.exposureTimeToShiftNs == null) {
85 | missing += " exposureTimeToShiftNs";
86 | }
87 | if (this.frameDurationToShiftNs == null) {
88 | missing += " frameDurationToShiftNs";
89 | }
90 | if (this.diffFromGoalNs == null) {
91 | missing += " diffFromGoalNs";
92 | }
93 | if (this.isAligned == null) {
94 | missing += " isAligned";
95 | }
96 |
97 | if (!missing.isEmpty()) {
98 | throw new IllegalStateException("Missing required properties:" + missing);
99 | }
100 |
101 | return new PhaseResponse(
102 | this.phaseNs,
103 | this.exposureTimeToShiftNs,
104 | this.frameDurationToShiftNs,
105 | this.diffFromGoalNs,
106 | this.isAligned);
107 | }
108 | }
109 |
110 | /** The measured phase in this response. */
111 | public long phaseNs() {
112 | return phaseNs;
113 | }
114 |
115 | /**
116 | * Estimated sensor exposure time needed in an inserted frame to align the phase to the goal
117 | * phase. This should be used to set CaptureRequest.SENSOR_EXPOSURE_TIME. This currently overrides
118 | * setting frame duration and is the only method for introducing phase shift.
119 | */
120 | public long exposureTimeToShiftNs() {
121 | return exposureTimeToShiftNs;
122 | }
123 |
124 | /** Difference between the goal phase and the phase in this response. */
125 | public long diffFromGoalNs() {
126 | return diffFromGoalNs;
127 | }
128 |
129 | /** True if the current phase is within the threshold of the goal phase. */
130 | public boolean isAligned() {
131 | return isAligned;
132 | }
133 |
134 | @Override
135 | public String toString() {
136 | return String.format(
137 | "PhaseResponse{phaseNs=%d, exposureTimeToShiftNs=%d, frameDurationToShiftNs=%d,"
138 | + " diffFromGoalNs=%d, isAligned=%s}",
139 | phaseNs, exposureTimeToShiftNs, frameDurationToShiftNs, diffFromGoalNs, isAligned);
140 | }
141 | }
142 |
--------------------------------------------------------------------------------
/app/src/main/res/layout/activity_main.xml:
--------------------------------------------------------------------------------
1 |
10 |
11 |
15 |
16 |
20 |
21 |
22 |
23 |
27 |
28 |
33 |
34 |
40 |
41 |
42 |
43 |
48 |
53 |
54 |
60 |
61 |
62 |
67 |
72 |
73 |
79 |
80 |
86 |
87 |
93 |
94 |
100 |
101 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
--------------------------------------------------------------------------------
/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MobileRoboticsSkoltech/RecSync-android/47fef0734f66bba545c191f31dcdbe6a72c5733b/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png
--------------------------------------------------------------------------------
/app/src/main/res/raw/default_phaseconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "periodNs": 33327307,
3 | "goalPhaseNs": 15000000,
4 | "alignThresholdNs": 100000,
5 | "overheadNs": 200000,
6 | "minExposureNs": 33370000
7 | }
--------------------------------------------------------------------------------
/app/src/main/res/raw/pixel1_phaseconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "periodNs": 33325000,
3 | "goalPhaseNs": 15000000,
4 | "alignThresholdNs": 200000,
5 | "overheadNs": 200000,
6 | "minExposureNs": 33170000
7 | }
8 |
--------------------------------------------------------------------------------
/app/src/main/res/raw/pixel2_phaseconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "periodNs": 33319419,
3 | "goalPhaseNs": 15000000,
4 | "alignThresholdNs": 200000,
5 | "overheadNs": 200000,
6 | "minExposureNs": 33170000
7 | }
8 |
--------------------------------------------------------------------------------
/app/src/main/res/raw/pixel3_240fps_phaseconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "periodNs": 4164011,
3 | "goalPhaseNs": 2000000,
4 | "alignThresholdNs": 100000,
5 | "overheadNs": 196499,
6 | "minExposureNs": 4164011
7 | }
8 |
--------------------------------------------------------------------------------
/app/src/main/res/raw/pixel3_60fps_phaseconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "periodNs": 16630105,
3 | "goalPhaseNs": 7000000,
4 | "alignThresholdNs": 100000,
5 | "overheadNs": 196499,
6 | "minExposureNs": 16530105
7 | }
8 |
--------------------------------------------------------------------------------
/app/src/main/res/values/colors.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | #008577
4 | #00574B
5 | #D81B60
6 |
7 |
--------------------------------------------------------------------------------
/app/src/main/res/values/dimens.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 16dp
5 | 16dp
6 |
7 |
8 |
--------------------------------------------------------------------------------
/app/src/main/res/values/strings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | RecSync
6 |
7 | Record video
8 |
9 | Align Phases
10 |
11 | Align 2A
12 |
13 |
14 | RecSync Android
15 |
16 | Phase
17 |
18 | Exposure:
19 |
20 | Sensitivity:
21 |
22 |
23 | You must grant camera, read, and write permissions
24 |
25 | Failed to open Camera2 service
26 |
27 | Failed to capture viewfinder frame
28 |
29 | No image outputs chosen
30 |
31 | Buffers
32 |
33 |
34 |
--------------------------------------------------------------------------------
/app/src/main/res/values/styles.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
7 |
14 |
15 |
16 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/build.gradle:
--------------------------------------------------------------------------------
1 | // Top-level build file where you can add configuration options common to all sub-projects/modules.
2 |
3 | buildscript {
4 | repositories {
5 | google()
6 | jcenter()
7 |
8 | }
9 | dependencies {
10 | classpath 'com.android.tools.build:gradle:3.5.1'
11 |
12 | // NOTE: Do not place your application dependencies here; they belong
13 | // in the individual module build.gradle files
14 | }
15 | }
16 |
17 | allprojects {
18 | repositories {
19 | google()
20 | jcenter()
21 |
22 | }
23 | }
24 |
25 | task clean(type: Delete) {
26 | delete rootProject.buildDir
27 | }
28 |
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MobileRoboticsSkoltech/RecSync-android/47fef0734f66bba545c191f31dcdbe6a72c5733b/gradle/wrapper/gradle-wrapper.jar
--------------------------------------------------------------------------------
/gradle/wrapper/gradle-wrapper.properties:
--------------------------------------------------------------------------------
1 | distributionBase=GRADLE_USER_HOME
2 | distributionPath=wrapper/dists
3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip
4 | zipStoreBase=GRADLE_USER_HOME
5 | zipStorePath=wrapper/dists
6 |
--------------------------------------------------------------------------------
/gradlew:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env sh
2 |
3 | #
4 | # Copyright 2015 the original author or authors.
5 | #
6 | # Licensed under the Apache License, Version 2.0 (the "License");
7 | # you may not use this file except in compliance with the License.
8 | # You may obtain a copy of the License at
9 | #
10 | # https://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # Unless required by applicable law or agreed to in writing, software
13 | # distributed under the License is distributed on an "AS IS" BASIS,
14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | # See the License for the specific language governing permissions and
16 | # limitations under the License.
17 | #
18 |
19 | ##############################################################################
20 | ##
21 | ## Gradle start up script for UN*X
22 | ##
23 | ##############################################################################
24 |
25 | # Attempt to set APP_HOME
26 | # Resolve links: $0 may be a link
27 | PRG="$0"
28 | # Need this for relative symlinks.
29 | while [ -h "$PRG" ] ; do
30 | ls=`ls -ld "$PRG"`
31 | link=`expr "$ls" : '.*-> \(.*\)$'`
32 | if expr "$link" : '/.*' > /dev/null; then
33 | PRG="$link"
34 | else
35 | PRG=`dirname "$PRG"`"/$link"
36 | fi
37 | done
38 | SAVED="`pwd`"
39 | cd "`dirname \"$PRG\"`/" >/dev/null
40 | APP_HOME="`pwd -P`"
41 | cd "$SAVED" >/dev/null
42 |
43 | APP_NAME="Gradle"
44 | APP_BASE_NAME=`basename "$0"`
45 |
46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
48 |
49 | # Use the maximum available, or set MAX_FD != -1 to use that value.
50 | MAX_FD="maximum"
51 |
52 | warn () {
53 | echo "$*"
54 | }
55 |
56 | die () {
57 | echo
58 | echo "$*"
59 | echo
60 | exit 1
61 | }
62 |
63 | # OS specific support (must be 'true' or 'false').
64 | cygwin=false
65 | msys=false
66 | darwin=false
67 | nonstop=false
68 | case "`uname`" in
69 | CYGWIN* )
70 | cygwin=true
71 | ;;
72 | Darwin* )
73 | darwin=true
74 | ;;
75 | MINGW* )
76 | msys=true
77 | ;;
78 | NONSTOP* )
79 | nonstop=true
80 | ;;
81 | esac
82 |
83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
84 |
85 |
86 | # Determine the Java command to use to start the JVM.
87 | if [ -n "$JAVA_HOME" ] ; then
88 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
89 | # IBM's JDK on AIX uses strange locations for the executables
90 | JAVACMD="$JAVA_HOME/jre/sh/java"
91 | else
92 | JAVACMD="$JAVA_HOME/bin/java"
93 | fi
94 | if [ ! -x "$JAVACMD" ] ; then
95 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
96 |
97 | Please set the JAVA_HOME variable in your environment to match the
98 | location of your Java installation."
99 | fi
100 | else
101 | JAVACMD="java"
102 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
103 |
104 | Please set the JAVA_HOME variable in your environment to match the
105 | location of your Java installation."
106 | fi
107 |
108 | # Increase the maximum file descriptors if we can.
109 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
110 | MAX_FD_LIMIT=`ulimit -H -n`
111 | if [ $? -eq 0 ] ; then
112 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
113 | MAX_FD="$MAX_FD_LIMIT"
114 | fi
115 | ulimit -n $MAX_FD
116 | if [ $? -ne 0 ] ; then
117 | warn "Could not set maximum file descriptor limit: $MAX_FD"
118 | fi
119 | else
120 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
121 | fi
122 | fi
123 |
124 | # For Darwin, add options to specify how the application appears in the dock
125 | if $darwin; then
126 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
127 | fi
128 |
129 | # For Cygwin or MSYS, switch paths to Windows format before running java
130 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
131 | APP_HOME=`cygpath --path --mixed "$APP_HOME"`
132 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
133 |
134 | JAVACMD=`cygpath --unix "$JAVACMD"`
135 |
136 | # We build the pattern for arguments to be converted via cygpath
137 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
138 | SEP=""
139 | for dir in $ROOTDIRSRAW ; do
140 | ROOTDIRS="$ROOTDIRS$SEP$dir"
141 | SEP="|"
142 | done
143 | OURCYGPATTERN="(^($ROOTDIRS))"
144 | # Add a user-defined pattern to the cygpath arguments
145 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then
146 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
147 | fi
148 | # Now convert the arguments - kludge to limit ourselves to /bin/sh
149 | i=0
150 | for arg in "$@" ; do
151 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
152 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
153 |
154 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
155 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
156 | else
157 | eval `echo args$i`="\"$arg\""
158 | fi
159 | i=`expr $i + 1`
160 | done
161 | case $i in
162 | 0) set -- ;;
163 | 1) set -- "$args0" ;;
164 | 2) set -- "$args0" "$args1" ;;
165 | 3) set -- "$args0" "$args1" "$args2" ;;
166 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
167 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
168 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
169 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
170 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
171 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
172 | esac
173 | fi
174 |
175 | # Escape application args
176 | save () {
177 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
178 | echo " "
179 | }
180 | APP_ARGS=`save "$@"`
181 |
182 | # Collect all arguments for the java command, following the shell quoting and substitution rules
183 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
184 |
185 | exec "$JAVACMD" "$@"
186 |
--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
1 | @rem
2 | @rem Copyright 2015 the original author or authors.
3 | @rem
4 | @rem Licensed under the Apache License, Version 2.0 (the "License");
5 | @rem you may not use this file except in compliance with the License.
6 | @rem You may obtain a copy of the License at
7 | @rem
8 | @rem https://www.apache.org/licenses/LICENSE-2.0
9 | @rem
10 | @rem Unless required by applicable law or agreed to in writing, software
11 | @rem distributed under the License is distributed on an "AS IS" BASIS,
12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | @rem See the License for the specific language governing permissions and
14 | @rem limitations under the License.
15 | @rem
16 |
17 | @if "%DEBUG%" == "" @echo off
18 | @rem ##########################################################################
19 | @rem
20 | @rem Gradle startup script for Windows
21 | @rem
22 | @rem ##########################################################################
23 |
24 | @rem Set local scope for the variables with windows NT shell
25 | if "%OS%"=="Windows_NT" setlocal
26 |
27 | set DIRNAME=%~dp0
28 | if "%DIRNAME%" == "" set DIRNAME=.
29 | set APP_BASE_NAME=%~n0
30 | set APP_HOME=%DIRNAME%
31 |
32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter.
33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
34 |
35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
37 |
38 | @rem Find java.exe
39 | if defined JAVA_HOME goto findJavaFromJavaHome
40 |
41 | set JAVA_EXE=java.exe
42 | %JAVA_EXE% -version >NUL 2>&1
43 | if "%ERRORLEVEL%" == "0" goto init
44 |
45 | echo.
46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
47 | echo.
48 | echo Please set the JAVA_HOME variable in your environment to match the
49 | echo location of your Java installation.
50 |
51 | goto fail
52 |
53 | :findJavaFromJavaHome
54 | set JAVA_HOME=%JAVA_HOME:"=%
55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
56 |
57 | if exist "%JAVA_EXE%" goto init
58 |
59 | echo.
60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
61 | echo.
62 | echo Please set the JAVA_HOME variable in your environment to match the
63 | echo location of your Java installation.
64 |
65 | goto fail
66 |
67 | :init
68 | @rem Get command-line arguments, handling Windows variants
69 |
70 | if not "%OS%" == "Windows_NT" goto win9xME_args
71 |
72 | :win9xME_args
73 | @rem Slurp the command line arguments.
74 | set CMD_LINE_ARGS=
75 | set _SKIP=2
76 |
77 | :win9xME_args_slurp
78 | if "x%~1" == "x" goto execute
79 |
80 | set CMD_LINE_ARGS=%*
81 |
82 | :execute
83 | @rem Setup the command line
84 |
85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
86 |
87 |
88 | @rem Execute Gradle
89 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
90 |
91 | :end
92 | @rem End local scope for the variables with windows NT shell
93 | if "%ERRORLEVEL%"=="0" goto mainEnd
94 |
95 | :fail
96 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
97 | rem the _cmd.exe /c_ return code!
98 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
99 | exit /b 1
100 |
101 | :mainEnd
102 | if "%OS%"=="Windows_NT" endlocal
103 |
104 | :omega
105 |
--------------------------------------------------------------------------------
/scripts/yuv2rgb.py:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | # Copyright 2019 The Google Research Authors.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | """Process raw NV21 images to jpg or png images using ffmpeg.
17 |
18 | This script requires ffmpeg (https://www.ffmpeg.org/download.html).
19 |
20 | Run with:
21 | python3 yuv2rgb.py img_.nv21 nv21_metadata_.txt
22 | out..
23 | """
24 |
25 | import argparse
26 | import subprocess
27 |
28 |
29 | def parse_meta(path):
30 | with open(path, 'r') as f:
31 | lines = [l.strip() for l in f.readlines()]
32 | img_width = lines[0].split(' ')[1]
33 | img_height = lines[1].split(' ')[1]
34 | img_pixel_format = lines[2].split(' ')[1]
35 | return img_width, img_height, img_pixel_format
36 |
37 |
38 | if __name__ == '__main__':
39 | parser = argparse.ArgumentParser()
40 | parser.add_argument(
41 | '-y', '--overwrite', help='Overwrite output.', action='store_true')
42 | parser.add_argument('input')
43 | parser.add_argument('meta')
44 | parser.add_argument('output', help='output filename (ending in .jpg or .png)')
45 | args = parser.parse_args()
46 |
47 | if not args.output.endswith('.jpg') and not args.output.endswith('.png'):
48 | raise ValueError('output must end in jpg or png.')
49 |
50 | width, height, pixel_format = parse_meta(args.meta)
51 | pixel_format = pixel_format.lower()
52 |
53 | overwrite_flag = '-y' if args.overwrite else '-n'
54 |
55 | cmd = [
56 | 'ffmpeg', overwrite_flag, '-f', 'image2', '-vcodec', 'rawvideo',
57 | '-pix_fmt', pixel_format, '-s', f'{width}x{height}', '-i', args.input,
58 | args.output
59 | ]
60 |
61 | subprocess.call(cmd)
62 |
--------------------------------------------------------------------------------
/settings.gradle:
--------------------------------------------------------------------------------
1 | include ':app'
2 | rootProject.name='CaptureSync'
3 |
--------------------------------------------------------------------------------
/utils/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 |
131 | .idea/
132 |
133 | # Other
134 | *.png
135 |
136 | *.pcd
137 |
138 | *.pto
139 |
140 | *.jpg
141 |
142 | *.csv
143 |
144 | *.tif
145 |
146 | *.mp4
147 |
148 | *.npy
149 |
150 | *.yaml
151 |
--------------------------------------------------------------------------------
/utils/README.md:
--------------------------------------------------------------------------------
1 | # Panoramic video demo
2 |
3 | Uses hujin CLI interface and ffmpeg commands to merge two synchronized smartphone videos, recorded with RecSync.
4 | TODO: describe
5 |
--------------------------------------------------------------------------------
/utils/extract.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import argparse
16 | from src.extraction_utils import extract_frame_data
17 |
18 |
19 | def main():
20 | parser = argparse.ArgumentParser(
21 | description="Extracts frames"
22 | )
23 | parser.add_argument(
24 | "--output",
25 | required=True
26 | )
27 | parser.add_argument('--frame_dir',
28 | help=' Smartphone frames directory')
29 | parser.add_argument('--vid', help=' Smartphone video path')
30 |
31 | args = parser.parse_args()
32 | # TODO: args assertion for dir and vid
33 | print("Extracting smartphone video frame data..")
34 | extract_frame_data(args.frame_dir, args.vid)
35 |
36 |
37 | if __name__ == '__main__':
38 | main()
39 |
--------------------------------------------------------------------------------
/utils/extract.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | set -eo pipefail
17 |
18 | SMARTPHONE_VIDEO_PATH=$1
19 |
20 | DATA_DIR="output/"$2
21 |
22 |
23 | ## Create a subdirectory for extraction
24 | rm -rf "$DATA_DIR"
25 | mkdir -p "$DATA_DIR"
26 |
27 | # SMARTPHONE_VIDEO_DIR="${SMARTPHONE_VIDEO_PATH%/*}"
28 |
29 | # Check if video exists
30 | echo "$SMARTPHONE_VIDEO_PATH"
31 | if [ ! -f "$SMARTPHONE_VIDEO_PATH" ]; then
32 | >&2 echo "Provided smartphone video doesn't exist"
33 | else
34 | DIR="$DATA_DIR"
35 | rm -rf "$DIR"
36 | mkdir "$DIR"
37 | ffmpeg -i "$SMARTPHONE_VIDEO_PATH" -vsync 0 "$DIR/frame-%d.png"
38 | python extract.py --output "$DIR" \
39 | --frame_dir "$DIR" --vid "$SMARTPHONE_VIDEO_PATH"
40 | fi
41 |
42 |
--------------------------------------------------------------------------------
/utils/get_match.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import glob
16 | import os
17 | import pandas as pd
18 |
19 |
20 | def main():
21 | vid_1 = './output/2'
22 | vid_2 = './output/1'
23 | match(vid_1, vid_2)
24 |
25 |
26 | def match(vid_1, vid_2):
27 | out_images_1 = sorted(glob.glob(vid_1 + "/*"))
28 | out_images_2 = sorted(glob.glob(vid_2 + "/*"))
29 | image_timestamps_1 = (list(map(
30 | lambda x: int(os.path.splitext(os.path.basename(x))[0]),
31 | out_images_1)))
32 | image_timestamps_2 = (list(map(
33 | lambda x: int(os.path.splitext(os.path.basename(x))[0]),
34 | out_images_2)))
35 |
36 | THRESHOLD_NS = 100000
37 |
38 | left = pd.DataFrame({'t': image_timestamps_1,
39 | 'left': image_timestamps_1}, dtype=int)
40 | # TODO: change this quick hack to prevent pandas from
41 | # converting ints to floats
42 | right = pd.DataFrame({'t': image_timestamps_2,
43 | 'right_int': image_timestamps_2,
44 | 'right': list(map(str, image_timestamps_2))},
45 | )
46 | print(right.dtypes)
47 | # align by nearest, because we need to account for frame drops
48 | df = pd.merge_asof(left, right, on='t',
49 | tolerance=THRESHOLD_NS,
50 | allow_exact_matches=True,
51 | direction='nearest')
52 | df = df.dropna()
53 | df = df.drop('t', axis='columns')
54 | df = df.drop('right_int', axis='columns')
55 |
56 | df = df.reset_index(drop=True)
57 | print(df.head())
58 | print(df.dtypes)
59 |
60 | df.to_csv('./output/match.csv')
61 |
62 |
63 | if __name__ == '__main__':
64 | main()
65 |
--------------------------------------------------------------------------------
/utils/make_demo.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | set -eo
17 |
18 | # Split two videos to frames
19 | ./extract.sh "$1" 1
20 | ./extract.sh "$2" 2
21 |
22 | # Find matching frames
23 | python ./get_match.py
24 |
25 | # Stitch panoramas
26 | python stitch.py --matcher ./output/match.csv --target ./output
27 |
28 | # Convert new images to video
29 | ./stitching_demo/convert.sh
--------------------------------------------------------------------------------
/utils/match.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | set -eo
17 |
18 | # Split two videos to frames
19 | ./extract.sh "$1" 1
20 | ./extract.sh "$2" 2
21 |
22 | # Find matching frames
23 | python ./get_match.py
24 |
--------------------------------------------------------------------------------
/utils/requirements.txt:
--------------------------------------------------------------------------------
1 | rosbag
2 | numpy
3 | opencv-python==4.2.0.32
4 | cv_bridge
5 | pyyaml
6 | pycryptodomex
7 | gnupg
8 | rospkg
9 | sensor_msgs
10 | pypcd
11 | pandas
12 |
--------------------------------------------------------------------------------
/utils/split.py:
--------------------------------------------------------------------------------
1 | # Copyright 2020 Mobile Robotics Lab. at Skoltech
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import argparse
16 | import os
17 | from shutil import copyfile
18 | from src.alignment_utils import ALLOWED_EXTENSIONS
19 | from src.rosbag_extraction_utils import make_dir_if_needed
20 |
21 |
22 | def main():
23 | parser = argparse.ArgumentParser(
24 | description="Split extracted data"
25 | )
26 | parser.add_argument(
27 | "--target_dir",
28 | required=True
29 | )
30 | parser.add_argument(
31 | "--data_dir",
32 | required=True
33 | )
34 | parser.add_argument('--timestamps', nargs='+', help=' List of sequence timestamps')
35 | args = parser.parse_args()
36 | split(args.target_dir, args.data_dir, list(map(lambda x: int(x), args.timestamps)))
37 |
38 |
39 | def split(target_dir, data_dir, timestamps):
40 | print("Splitting sequences...")
41 |
42 | filename_timestamps = list(map(
43 | lambda x: (x, int(os.path.splitext(x)[0])),
44 | filter(
45 | lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS,
46 | os.listdir(target_dir)
47 | )
48 | ))
49 | filename_timestamps.sort(key=lambda tup: tup[1])
50 | sequences = []
51 | prev = 0
52 | for timestamp in timestamps:
53 | sequences.append(list(filter(lambda x: x[1] < timestamp and x[1] >= prev, filename_timestamps)))
54 | prev = timestamp
55 | sequences.append(list(filter(lambda x: x[1] >= timestamp, filename_timestamps)))
56 | for i, seq in enumerate(sequences):
57 | print("Copying sequence %d..." % i)
58 | new_dir = os.path.join(data_dir, "seq_%d" % i, os.path.split(target_dir)[-1])
59 | make_dir_if_needed(new_dir)
60 | for filename, _ in seq:
61 | copyfile(os.path.join(target_dir, filename), os.path.join(new_dir, filename))
62 |
63 |
64 | if __name__ == '__main__':
65 | main()
66 |
--------------------------------------------------------------------------------
/utils/src/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MobileRoboticsSkoltech/RecSync-android/47fef0734f66bba545c191f31dcdbe6a72c5733b/utils/src/__init__.py
--------------------------------------------------------------------------------
/utils/src/extraction_utils.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import re
16 | import os
17 | ALLOWED_EXTENSIONS = ['.jpg', '.jpeg', '.npy', '.png', '.pcd']
18 |
19 |
20 | def make_dir_if_needed(dir_path):
21 | if not os.path.exists(dir_path):
22 | os.makedirs(dir_path)
23 |
24 |
25 | def get_timestamp_filename(timestamp, extension):
26 | return "%d.%s" % (timestamp.secs * 1e9 + timestamp.nsecs, extension)
27 |
28 |
29 | def extract_frame_data(target_dir, video_path):
30 | # load frame timestamps csv, rename frames according to it
31 | video_root, video_filename = os.path.split(video_path)
32 | video_name, _ = os.path.splitext(video_filename)
33 | video_date = re.sub(r"VID_((\d|_)*)", r"\1", video_name)
34 |
35 | video_parent_dir = os.path.abspath(os.path.join(video_root, os.pardir))
36 |
37 | with open(os.path.join(video_parent_dir, video_date + ".csv"))\
38 | as frame_timestamps_file:
39 | filename_timestamps = list(map(
40 | lambda x: (x.strip('\n'), int(x)), frame_timestamps_file.readlines()
41 | ))
42 | length = len(list(filter(
43 | lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS,
44 | os.listdir(target_dir)
45 | )))
46 | # frame number assertion
47 | # assert len(filename_timestamps) == len(list(filter(
48 | # lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS,
49 | # os.listdir(target_dir)
50 | # ))), "Frame number in video %d and timestamp files %d did not match" % (l, len(filename_timestamps))
51 |
52 | _, extension = os.path.splitext(os.listdir(target_dir)[0])
53 | for i in range(length):
54 | timestamp = filename_timestamps[i]
55 | os.rename(
56 | os.path.join(target_dir, "frame-%d.png" % (i + 1)),
57 | os.path.join(target_dir, timestamp[0] + extension)
58 | )
--------------------------------------------------------------------------------
/utils/stitch.py:
--------------------------------------------------------------------------------
1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | import pandas as pd
16 | import argparse
17 | import os
18 | import subprocess
19 |
20 |
21 | def main():
22 | parser = argparse.ArgumentParser(
23 | description="Stitch multiple images"
24 | )
25 |
26 | parser.add_argument(
27 | "--matcher",
28 | required=True
29 | )
30 | parser.add_argument(
31 | "--target",
32 | required=True
33 | )
34 | args = parser.parse_args()
35 |
36 | target = args.target
37 | matcher = args.matcher
38 | stitch(target, matcher)
39 |
40 |
41 | def stitch(target, matcher):
42 | with open(matcher, 'r') as csvfile:
43 | print(csvfile.name)
44 | df = pd.read_csv(csvfile)
45 | # TODO: change to csv lib usage instead of pandas
46 | for index, row in df.iterrows():
47 | # Read matching from csv
48 | print(row)
49 | right = row['right']
50 | left = row['left']
51 | right_img = os.path.join(target, '1', f'{right}.png')
52 | left_img = os.path.join(target, '2', f'{left}.png')
53 | print(f"Stitching {left_img} and {right_img}")
54 | # Launch the script for stitching two images, save w first name
55 | bashCommand = f" ./stitching_demo/stitch_two.sh {left_img} \
56 | {right_img} {left}"
57 |
58 | p = subprocess.Popen(bashCommand, shell=True)
59 |
60 | # and you can block util the cmd execute finish
61 | p.wait()
62 |
63 |
64 | if __name__ == '__main__':
65 | main()
66 |
--------------------------------------------------------------------------------
/utils/stitching_demo/stitch_two.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | set -evo pipefail
17 |
18 | IMAGE_1=$1
19 | IMAGE_2=$2
20 | # TODO fix this
21 | NAME=$3
22 | PROJECT='./stitching_demo/project.pto'
23 | echo "$IMAGE_1" - "$IMAGE_2"
24 | pto_gen --projection=0 --fov=50 -o "$PROJECT" "$IMAGE_1" "$IMAGE_2"
25 | pto_lensstack --new-lens i0,i1 -o "$PROJECT" "$PROJECT"
26 | cd "./stitching_demo" || exit
27 | PROJECT='project.pto'
28 | # cpfind -o "$PROJECT" --multirow --celeste "$PROJECT"
29 | # celeste_standalone -i project.pto -o project.pto
30 | # cpclean -o "$PROJECT" "$PROJECT"
31 | # linefind -o "$PROJECT" "$PROJECT"
32 | # --opt=v,a,b,c,d,e,g,t,y,p,r,TrX,TrY,TrZ,Tpy,Tpp
33 | pto_var --anchor=1 --opt=Vb0 --set=y0=37.086,p0=-0.295,r0=3.013,y1=0,p1=0,r1=0,TrX=0,TrY=0,TrZ=0,Tpy=0,Tpp=0 -o "$PROJECT" "$PROJECT"
34 | autooptimiser -n -o "$PROJECT" "$PROJECT"
35 | pano_modify --crop=878,3000,39,710 --canvas=3000x750 --fov=120x30 -o "$PROJECT" "$PROJECT"
36 |
37 | vig_optimize -o "$PROJECT" "$PROJECT"
38 | hugin_executor --stitching --prefix="$NAME" "$PROJECT"
--------------------------------------------------------------------------------