├── utils ├── src │ ├── __init__.py │ └── extraction_utils.py ├── README.md ├── requirements.txt ├── match.sh ├── make_demo.sh ├── extract.sh ├── extract.py ├── stitching_demo │ └── stitch_two.sh ├── stitch.py ├── get_match.py ├── split.py └── .gitignore ├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── app ├── src │ └── main │ │ ├── res │ │ ├── mipmap-xxxhdpi │ │ │ └── ic_launcher.png │ │ ├── raw │ │ │ ├── default_phaseconfig.json │ │ │ ├── pixel1_phaseconfig.json │ │ │ ├── pixel2_phaseconfig.json │ │ │ ├── pixel3_240fps_phaseconfig.json │ │ │ └── pixel3_60fps_phaseconfig.json │ │ ├── values │ │ │ ├── colors.xml │ │ │ ├── dimens.xml │ │ │ ├── styles.xml │ │ │ └── strings.xml │ │ └── layout │ │ │ └── activity_main.xml │ │ ├── java │ │ └── com │ │ │ └── googleresearch │ │ │ └── capturesync │ │ │ ├── softwaresync │ │ │ ├── Ticker.java │ │ │ ├── SystemTicker.java │ │ │ ├── RpcCallback.java │ │ │ ├── TimeUtils.java │ │ │ ├── TimeDomainConverter.java │ │ │ ├── SntpOffsetResponse.java │ │ │ ├── SyncConstants.java │ │ │ ├── ClientInfo.java │ │ │ ├── phasealign │ │ │ │ ├── PeriodCalculator.java │ │ │ │ ├── PhaseConfig.java │ │ │ │ ├── PhaseResponse.java │ │ │ │ └── PhaseAligner.java │ │ │ ├── NetworkHelpers.java │ │ │ ├── SntpListener.java │ │ │ ├── CSVLogger.java │ │ │ ├── SoftwareSyncClient.java │ │ │ ├── SoftwareSyncBase.java │ │ │ ├── SimpleNetworkTimeProtocol.java │ │ │ └── SoftwareSyncLeader.java │ │ │ ├── Frame.java │ │ │ ├── Constants.java │ │ │ ├── AutoFitSurfaceView.java │ │ │ ├── CaptureRequestFactory.java │ │ │ ├── PhaseAlignController.java │ │ │ ├── CameraController.java │ │ │ └── SoftwareSyncController.java │ │ └── AndroidManifest.xml ├── proguard-rules.pro ├── build.gradle └── app.iml ├── .gitignore ├── CaptureSync.iml ├── CONTRIBUTING.md ├── .github └── workflows │ └── assemble.yml ├── scripts └── yuv2rgb.py ├── README.md ├── gradlew.bat ├── gradlew ├── LICENSE └── LICENSE_GOOGLE_RESEARCH /utils/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | rootProject.name='CaptureSync' 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MobileRoboticsSkoltech/RecSync-android/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MobileRoboticsSkoltech/RecSync-android/HEAD/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /utils/README.md: -------------------------------------------------------------------------------- 1 | # Panoramic video demo 2 | 3 | Uses hujin CLI interface and ffmpeg commands to merge two synchronized smartphone videos, recorded with RecSync. 4 | TODO: describe 5 | -------------------------------------------------------------------------------- /utils/requirements.txt: -------------------------------------------------------------------------------- 1 | rosbag 2 | numpy 3 | opencv-python==4.2.0.32 4 | cv_bridge 5 | pyyaml 6 | pycryptodomex 7 | gnupg 8 | rospkg 9 | sensor_msgs 10 | pypcd 11 | pandas 12 | -------------------------------------------------------------------------------- /app/src/main/res/raw/default_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33327307, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33370000 7 | } -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel1_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33325000, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 200000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33170000 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel2_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33319419, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 200000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33170000 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel3_240fps_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 4164011, 3 | "goalPhaseNs": 2000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 196499, 6 | "minExposureNs": 4164011 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel3_60fps_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 16630105, 3 | "goalPhaseNs": 7000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 196499, 6 | "minExposureNs": 16530105 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/values/colors.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | #008577 4 | #00574B 5 | #D81B60 6 | 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /app/src/main/res/values/dimens.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16dp 5 | 16dp 6 | 7 | 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _archive/ 2 | _devdocs/ 3 | _docs/*.jpg 4 | _docs/ads.txt 5 | _docs/app-ads.txt 6 | _docs/gplay.html 7 | _other/ 8 | _saved/ 9 | 10 | .gradle/ 11 | 12 | .idea/* 13 | !.idea/inspectionProfiles 14 | 15 | build/ 16 | app/release/ 17 | gfx/ 18 | testdata/ 19 | 20 | *.db 21 | *.iml 22 | *.apk 23 | *.ap_ 24 | 25 | local.properties 26 | uninstall.bat 27 | 28 | *.keystore 29 | 30 | opencamera-extended-firebase-adminsdk-yv5yz-e33a8ce5c1.json 31 | 32 | *.csv 33 | 34 | **/*.png 35 | **/*.zip 36 | **/*.mp4 37 | **/*.tif 38 | 39 | *.mp4 40 | -------------------------------------------------------------------------------- /app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 14 | 15 | 16 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /utils/match.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -eo 17 | 18 | # Split two videos to frames 19 | ./extract.sh "$1" 1 20 | ./extract.sh "$2" 2 21 | 22 | # Find matching frames 23 | python ./get_match.py 24 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile 22 | -------------------------------------------------------------------------------- /CaptureSync.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/Ticker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Interface for getting the time in nanoseconds, abstracting out accessing system time. */ 20 | public interface Ticker { 21 | /* Returns the time in nanoseconds. */ 22 | long read(); 23 | } 24 | -------------------------------------------------------------------------------- /utils/make_demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -eo 17 | 18 | # Split two videos to frames 19 | ./extract.sh "$1" 1 20 | ./extract.sh "$2" 2 21 | 22 | # Find matching frames 23 | python ./get_match.py 24 | 25 | # Stitch panoramas 26 | python stitch.py --matcher ./output/match.csv --target ./output 27 | 28 | # Convert new images to video 29 | ./stitching_demo/convert.sh -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 29 5 | buildToolsVersion "29.0.2" 6 | defaultConfig { 7 | applicationId "com.googleresearch.capturesync" 8 | minSdkVersion 28 9 | targetSdkVersion 29 10 | versionCode 1 11 | versionName "1.0" 12 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 13 | } 14 | buildTypes { 15 | release { 16 | minifyEnabled false 17 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' 18 | } 19 | } 20 | compileOptions { 21 | sourceCompatibility = 1.8 22 | targetCompatibility = 1.8 23 | } 24 | } 25 | 26 | dependencies { 27 | implementation fileTree(dir: 'libs', include: ['*.jar']) 28 | implementation 'androidx.appcompat:appcompat:1.0.2' 29 | testImplementation 'junit:junit:4.12' 30 | androidTestImplementation 'androidx.test.ext:junit:1.1.0' 31 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1' 32 | } 33 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SystemTicker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.os.SystemClock; 20 | 21 | /** Simple implementation of Ticker interface using the SystemClock elapsed realtime clock. */ 22 | public class SystemTicker implements Ticker { 23 | @Override 24 | public long read() { 25 | return SystemClock.elapsedRealtimeNanos(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/RpcCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Interface for RPC callbacks, the base methods used for communicating between devices. */ 20 | public interface RpcCallback { 21 | 22 | /** 23 | * The callback method called when an RPC is received. 24 | * 25 | * @param payload Contains the payload sent by the RPC. 26 | */ 27 | void call(String payload); 28 | } 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 17 | 18 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /utils/extract.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -eo pipefail 17 | 18 | SMARTPHONE_VIDEO_PATH=$1 19 | 20 | DATA_DIR="output/"$2 21 | 22 | 23 | ## Create a subdirectory for extraction 24 | rm -rf "$DATA_DIR" 25 | mkdir -p "$DATA_DIR" 26 | 27 | # SMARTPHONE_VIDEO_DIR="${SMARTPHONE_VIDEO_PATH%/*}" 28 | 29 | # Check if video exists 30 | echo "$SMARTPHONE_VIDEO_PATH" 31 | if [ ! -f "$SMARTPHONE_VIDEO_PATH" ]; then 32 | >&2 echo "Provided smartphone video doesn't exist" 33 | else 34 | DIR="$DATA_DIR" 35 | rm -rf "$DIR" 36 | mkdir "$DIR" 37 | ffmpeg -i "$SMARTPHONE_VIDEO_PATH" -vsync 0 "$DIR/frame-%d.png" 38 | python extract.py --output "$DIR" \ 39 | --frame_dir "$DIR" --vid "$SMARTPHONE_VIDEO_PATH" 40 | fi 41 | 42 | -------------------------------------------------------------------------------- /utils/extract.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | from src.extraction_utils import extract_frame_data 17 | 18 | 19 | def main(): 20 | parser = argparse.ArgumentParser( 21 | description="Extracts frames" 22 | ) 23 | parser.add_argument( 24 | "--output", 25 | required=True 26 | ) 27 | parser.add_argument('--frame_dir', 28 | help=' Smartphone frames directory') 29 | parser.add_argument('--vid', help=' Smartphone video path') 30 | 31 | args = parser.parse_args() 32 | # TODO: args assertion for dir and vid 33 | print("Extracting smartphone video frame data..") 34 | extract_frame_data(args.frame_dir, args.vid) 35 | 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Helper conversions between time scales. */ 20 | public final class TimeUtils { 21 | 22 | public static double nanosToMillis(double nanos) { 23 | return nanos / 1_000_000L; 24 | } 25 | 26 | public static long nanosToSeconds(long nanos) { 27 | return nanos / 1_000_000_000L; 28 | } 29 | 30 | public static double nanosToSeconds(double nanos) { 31 | return nanos / 1_000_000_000L; 32 | } 33 | 34 | public static long millisToNanos(long millis) { 35 | return millis * 1_000_000L; 36 | } 37 | 38 | public static long secondsToNanos(int seconds) { 39 | return seconds * 1_000_000_000L; 40 | } 41 | 42 | private TimeUtils() {} 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/assemble.yml: -------------------------------------------------------------------------------- 1 | name: Assemble 2 | 3 | on: [push, workflow_dispatch] 4 | 5 | jobs: 6 | assemble: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | - uses: actions/setup-java@v2 13 | with: 14 | distribution: 'adopt' 15 | java-version: '8' 16 | 17 | - name: Cache Gradle packages 18 | uses: actions/cache@v2 19 | with: 20 | path: | 21 | ~/.gradle/caches 22 | ~/.gradle/wrapper 23 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 24 | restore-keys: | 25 | ${{ runner.os }}-gradle- 26 | 27 | - name: Assemble release 28 | run: ./gradlew assembleRelease 29 | 30 | - name: Run lint for release 31 | run: ./gradlew lintRelease 32 | 33 | - name: Assemble debug 34 | run: ./gradlew assembleDebug 35 | 36 | - name: Run lint for debug 37 | run: ./gradlew lintDebug 38 | 39 | - name: Upload lint results 40 | uses: actions/upload-artifact@v2 41 | with: 42 | name: lint-results 43 | path: app/build/reports/lint-results-*.* 44 | 45 | - name: Cleanup Gradle Cache 46 | # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions. 47 | # Restoring these files from a GitHub Actions cache might cause problems for future builds. 48 | run: | 49 | rm -f ~/.gradle/caches/modules-2/modules-2.lock 50 | rm -f ~/.gradle/caches/modules-2/gc.properties 51 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/Frame.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.hardware.camera2.CaptureResult; 20 | import android.media.Image; 21 | import com.googleresearch.capturesync.ImageMetadataSynchronizer.Output; 22 | import java.io.Closeable; 23 | 24 | /** A holder for a CaptureResult with multiple outputs. */ 25 | public class Frame implements Closeable { 26 | public final CaptureResult result; 27 | public final Output output; 28 | private boolean closed = false; 29 | 30 | public Frame(CaptureResult result, Output output) { 31 | this.result = result; 32 | this.output = output; 33 | } 34 | 35 | @Override 36 | public void close() { 37 | if (closed) { 38 | throw new IllegalStateException("This Frame is already closed"); 39 | } 40 | for (Image image : output.images) { 41 | image.close(); 42 | } 43 | output.close(); 44 | closed = true; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /utils/stitching_demo/stitch_two.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | set -evo pipefail 17 | 18 | IMAGE_1=$1 19 | IMAGE_2=$2 20 | # TODO fix this 21 | NAME=$3 22 | PROJECT='./stitching_demo/project.pto' 23 | echo "$IMAGE_1" - "$IMAGE_2" 24 | pto_gen --projection=0 --fov=50 -o "$PROJECT" "$IMAGE_1" "$IMAGE_2" 25 | pto_lensstack --new-lens i0,i1 -o "$PROJECT" "$PROJECT" 26 | cd "./stitching_demo" || exit 27 | PROJECT='project.pto' 28 | # cpfind -o "$PROJECT" --multirow --celeste "$PROJECT" 29 | # celeste_standalone -i project.pto -o project.pto 30 | # cpclean -o "$PROJECT" "$PROJECT" 31 | # linefind -o "$PROJECT" "$PROJECT" 32 | # --opt=v,a,b,c,d,e,g,t,y,p,r,TrX,TrY,TrZ,Tpy,Tpp 33 | pto_var --anchor=1 --opt=Vb0 --set=y0=37.086,p0=-0.295,r0=3.013,y1=0,p1=0,r1=0,TrX=0,TrY=0,TrZ=0,Tpy=0,Tpp=0 -o "$PROJECT" "$PROJECT" 34 | autooptimiser -n -o "$PROJECT" "$PROJECT" 35 | pano_modify --crop=878,3000,39,710 --canvas=3000x750 --fov=120x30 -o "$PROJECT" "$PROJECT" 36 | 37 | vig_optimize -o "$PROJECT" "$PROJECT" 38 | hugin_executor --stitching --prefix="$NAME" "$PROJECT" -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeDomainConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** 20 | * Interface used to manage time domain conversion, implemented by {@link SoftwareSyncBase}. This 21 | * allows {@link ResultProcessor} to convert sensor timestamps to the synchronized time domain 22 | * without needing full access to the softwaresync object. 23 | */ 24 | public interface TimeDomainConverter { 25 | 26 | /** 27 | * Calculates the leader time associated with the given local time in nanoseconds. The local time 28 | * must be in the SystemClock.elapsedRealTimeNanos() localClock domain, nanosecond units. This 29 | * includes timestamps such as the sensor timestamp from the camera. leader_time = 30 | * local_elapsed_time_ns + leader_from_local_ns. 31 | * 32 | * @param localTimeNs given local time (local clock SystemClock.elapsedRealtimeNanos() domain). 33 | * @return leader synchronized time in nanoseconds. 34 | */ 35 | long leaderTimeForLocalTimeNs(long localTimeNs); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | RecSync 6 | 7 | Record video 8 | 9 | Align Phases 10 | 11 | Align 2A 12 | 13 | 14 | RecSync Android 15 | 16 | Phase 17 | 18 | Exposure: 19 | 20 | Sensitivity: 21 | 22 | 23 | You must grant camera, read, and write permissions 24 | 25 | Failed to open Camera2 service 26 | 27 | Failed to capture viewfinder frame 28 | 29 | No image outputs chosen 30 | 31 | Buffers 32 | 33 | 34 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/Constants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.hardware.camera2.CameraCharacteristics; 20 | import com.googleresearch.capturesync.softwaresync.TimeUtils; 21 | 22 | /** Constants including what type of images to save and the save directory. */ 23 | public final class Constants { 24 | public static final int DEFAULT_CAMERA_FACING = CameraCharacteristics.LENS_FACING_BACK; 25 | 26 | /** 27 | * Delay from capture button press to capture, giving network time to send messages to clients. 28 | */ 29 | public static final long FUTURE_TRIGGER_DELAY_NS = TimeUtils.millisToNanos(500); 30 | 31 | /* Set at least one of {SAVE_YUV, SAVE_RAW} to true to save any data. */ 32 | public static final boolean SAVE_YUV = true; 33 | 34 | // TODO: Implement saving ImageFormat.RAW10 to DNG. 35 | // DngCreator works with ImageFormat.RAW_SENSOR but it is slow and power-hungry. 36 | public static final boolean SAVE_RAW = false; 37 | 38 | // TODO(samansari): Turn SAVE_JPG_FROM_YUV into a checkbox instead. 39 | /* Set true to save a JPG to the gallery for preview. This is slow but gives you a "postview". */ 40 | public static final boolean SAVE_JPG_FROM_YUV = true; 41 | public static final int JPG_QUALITY = 95; 42 | 43 | public static final boolean USE_FULL_SCREEN_IMMERSIVE = false; 44 | 45 | private Constants() {} 46 | } 47 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpOffsetResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** AutoValue class for SNTP offsetNs, synchronization accuracy and status. */ 20 | public final class SntpOffsetResponse { 21 | private final long offsetNs; 22 | private final long syncAccuracyNs; 23 | private final boolean status; 24 | 25 | static SntpOffsetResponse create(long offset, long syncAccuracy, boolean status) { 26 | return new SntpOffsetResponse(offset, syncAccuracy, status); 27 | } 28 | 29 | private SntpOffsetResponse(long offsetNs, long syncAccuracyNs, boolean status) { 30 | this.offsetNs = offsetNs; 31 | this.syncAccuracyNs = syncAccuracyNs; 32 | this.status = status; 33 | } 34 | 35 | /** 36 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain. 37 | * 38 | *

The client can take their local_time to get leader_time via: local_time (leader - client) = 39 | * leader_time. 40 | */ 41 | public long offsetNs() { 42 | return offsetNs; 43 | } 44 | 45 | /** 46 | * The worst case error in the clock domains between leader and client for this response, in 47 | * nanoseconds of the AP SystemClock domain. 48 | */ 49 | public long syncAccuracyNs() { 50 | return syncAccuracyNs; 51 | } 52 | 53 | /** The success status of this response. */ 54 | public boolean status() { 55 | return status; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /utils/stitch.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import pandas as pd 16 | import argparse 17 | import os 18 | import subprocess 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser( 23 | description="Stitch multiple images" 24 | ) 25 | 26 | parser.add_argument( 27 | "--matcher", 28 | required=True 29 | ) 30 | parser.add_argument( 31 | "--target", 32 | required=True 33 | ) 34 | args = parser.parse_args() 35 | 36 | target = args.target 37 | matcher = args.matcher 38 | stitch(target, matcher) 39 | 40 | 41 | def stitch(target, matcher): 42 | with open(matcher, 'r') as csvfile: 43 | print(csvfile.name) 44 | df = pd.read_csv(csvfile) 45 | # TODO: change to csv lib usage instead of pandas 46 | for index, row in df.iterrows(): 47 | # Read matching from csv 48 | print(row) 49 | right = row['right'] 50 | left = row['left'] 51 | right_img = os.path.join(target, '1', f'{right}.png') 52 | left_img = os.path.join(target, '2', f'{left}.png') 53 | print(f"Stitching {left_img} and {right_img}") 54 | # Launch the script for stitching two images, save w first name 55 | bashCommand = f" ./stitching_demo/stitch_two.sh {left_img} \ 56 | {right_img} {left}" 57 | 58 | p = subprocess.Popen(bashCommand, shell=True) 59 | 60 | # and you can block util the cmd execute finish 61 | p.wait() 62 | 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /scripts/yuv2rgb.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Process raw NV21 images to jpg or png images using ffmpeg. 17 | 18 | This script requires ffmpeg (https://www.ffmpeg.org/download.html). 19 | 20 | Run with: 21 | python3 yuv2rgb.py img_.nv21 nv21_metadata_.txt 22 | out.. 23 | """ 24 | 25 | import argparse 26 | import subprocess 27 | 28 | 29 | def parse_meta(path): 30 | with open(path, 'r') as f: 31 | lines = [l.strip() for l in f.readlines()] 32 | img_width = lines[0].split(' ')[1] 33 | img_height = lines[1].split(' ')[1] 34 | img_pixel_format = lines[2].split(' ')[1] 35 | return img_width, img_height, img_pixel_format 36 | 37 | 38 | if __name__ == '__main__': 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument( 41 | '-y', '--overwrite', help='Overwrite output.', action='store_true') 42 | parser.add_argument('input') 43 | parser.add_argument('meta') 44 | parser.add_argument('output', help='output filename (ending in .jpg or .png)') 45 | args = parser.parse_args() 46 | 47 | if not args.output.endswith('.jpg') and not args.output.endswith('.png'): 48 | raise ValueError('output must end in jpg or png.') 49 | 50 | width, height, pixel_format = parse_meta(args.meta) 51 | pixel_format = pixel_format.lower() 52 | 53 | overwrite_flag = '-y' if args.overwrite else '-n' 54 | 55 | cmd = [ 56 | 'ffmpeg', overwrite_flag, '-f', 'image2', '-vcodec', 'rawvideo', 57 | '-pix_fmt', pixel_format, '-s', f'{width}x{height}', '-i', args.input, 58 | args.output 59 | ] 60 | 61 | subprocess.call(cmd) 62 | -------------------------------------------------------------------------------- /utils/get_match.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import glob 16 | import os 17 | import pandas as pd 18 | 19 | 20 | def main(): 21 | vid_1 = './output/2' 22 | vid_2 = './output/1' 23 | match(vid_1, vid_2) 24 | 25 | 26 | def match(vid_1, vid_2): 27 | out_images_1 = sorted(glob.glob(vid_1 + "/*")) 28 | out_images_2 = sorted(glob.glob(vid_2 + "/*")) 29 | image_timestamps_1 = (list(map( 30 | lambda x: int(os.path.splitext(os.path.basename(x))[0]), 31 | out_images_1))) 32 | image_timestamps_2 = (list(map( 33 | lambda x: int(os.path.splitext(os.path.basename(x))[0]), 34 | out_images_2))) 35 | 36 | THRESHOLD_NS = 100000 37 | 38 | left = pd.DataFrame({'t': image_timestamps_1, 39 | 'left': image_timestamps_1}, dtype=int) 40 | # TODO: change this quick hack to prevent pandas from 41 | # converting ints to floats 42 | right = pd.DataFrame({'t': image_timestamps_2, 43 | 'right_int': image_timestamps_2, 44 | 'right': list(map(str, image_timestamps_2))}, 45 | ) 46 | print(right.dtypes) 47 | # align by nearest, because we need to account for frame drops 48 | df = pd.merge_asof(left, right, on='t', 49 | tolerance=THRESHOLD_NS, 50 | allow_exact_matches=True, 51 | direction='nearest') 52 | df = df.dropna() 53 | df = df.drop('t', axis='columns') 54 | df = df.drop('right_int', axis='columns') 55 | 56 | df = df.reset_index(drop=True) 57 | print(df.head()) 58 | print(df.dtypes) 59 | 60 | df.to_csv('./output/match.csv') 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /utils/split.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Mobile Robotics Lab. at Skoltech 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import os 17 | from shutil import copyfile 18 | from src.alignment_utils import ALLOWED_EXTENSIONS 19 | from src.rosbag_extraction_utils import make_dir_if_needed 20 | 21 | 22 | def main(): 23 | parser = argparse.ArgumentParser( 24 | description="Split extracted data" 25 | ) 26 | parser.add_argument( 27 | "--target_dir", 28 | required=True 29 | ) 30 | parser.add_argument( 31 | "--data_dir", 32 | required=True 33 | ) 34 | parser.add_argument('--timestamps', nargs='+', help=' List of sequence timestamps') 35 | args = parser.parse_args() 36 | split(args.target_dir, args.data_dir, list(map(lambda x: int(x), args.timestamps))) 37 | 38 | 39 | def split(target_dir, data_dir, timestamps): 40 | print("Splitting sequences...") 41 | 42 | filename_timestamps = list(map( 43 | lambda x: (x, int(os.path.splitext(x)[0])), 44 | filter( 45 | lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS, 46 | os.listdir(target_dir) 47 | ) 48 | )) 49 | filename_timestamps.sort(key=lambda tup: tup[1]) 50 | sequences = [] 51 | prev = 0 52 | for timestamp in timestamps: 53 | sequences.append(list(filter(lambda x: x[1] < timestamp and x[1] >= prev, filename_timestamps))) 54 | prev = timestamp 55 | sequences.append(list(filter(lambda x: x[1] >= timestamp, filename_timestamps))) 56 | for i, seq in enumerate(sequences): 57 | print("Copying sequence %d..." % i) 58 | new_dir = os.path.join(data_dir, "seq_%d" % i, os.path.split(target_dir)[-1]) 59 | make_dir_if_needed(new_dir) 60 | for filename, _ in seq: 61 | copyfile(os.path.join(target_dir, filename), os.path.join(new_dir, filename)) 62 | 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /utils/src/extraction_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Mobile Robotics Lab. at Skoltech 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import re 16 | import os 17 | ALLOWED_EXTENSIONS = ['.jpg', '.jpeg', '.npy', '.png', '.pcd'] 18 | 19 | 20 | def make_dir_if_needed(dir_path): 21 | if not os.path.exists(dir_path): 22 | os.makedirs(dir_path) 23 | 24 | 25 | def get_timestamp_filename(timestamp, extension): 26 | return "%d.%s" % (timestamp.secs * 1e9 + timestamp.nsecs, extension) 27 | 28 | 29 | def extract_frame_data(target_dir, video_path): 30 | # load frame timestamps csv, rename frames according to it 31 | video_root, video_filename = os.path.split(video_path) 32 | video_name, _ = os.path.splitext(video_filename) 33 | video_date = re.sub(r"VID_((\d|_)*)", r"\1", video_name) 34 | 35 | video_parent_dir = os.path.abspath(os.path.join(video_root, os.pardir)) 36 | 37 | with open(os.path.join(video_parent_dir, video_date + ".csv"))\ 38 | as frame_timestamps_file: 39 | filename_timestamps = list(map( 40 | lambda x: (x.strip('\n'), int(x)), frame_timestamps_file.readlines() 41 | )) 42 | length = len(list(filter( 43 | lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS, 44 | os.listdir(target_dir) 45 | ))) 46 | # frame number assertion 47 | # assert len(filename_timestamps) == len(list(filter( 48 | # lambda x: os.path.splitext(x)[1] in ALLOWED_EXTENSIONS, 49 | # os.listdir(target_dir) 50 | # ))), "Frame number in video %d and timestamp files %d did not match" % (l, len(filename_timestamps)) 51 | 52 | _, extension = os.path.splitext(os.listdir(target_dir)[0]) 53 | for i in range(length): 54 | timestamp = filename_timestamps[i] 55 | os.rename( 56 | os.path.join(target_dir, "frame-%d.png" % (i + 1)), 57 | os.path.join(target_dir, timestamp[0] + extension) 58 | ) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Logo](https://imgur.com/YtJA0E2.png) 2 | 3 | If you use this application, please cite [Sub-millisecond Video Synchronization of Multiple Android Smartphones](https://arxiv.org/abs/2107.00987): 4 | ``` 5 | @misc{akhmetyanov2021submillisecond, 6 | title={Sub-millisecond Video Synchronization of Multiple Android Smartphones}, 7 | author={Azat Akhmetyanov and Anastasiia Kornilova and Marsel Faizullin and David Pozo and Gonzalo Ferrer}, 8 | year={2021}, 9 | eprint={2107.00987}, 10 | archivePrefix={arXiv}, 11 | primaryClass={cs.CV} 12 | } 13 | ``` 14 | ### Usage: 15 | 16 | 17 | #### Leader smartphone setup 18 | 19 | 1. Start a Wi-Fi hotspot. 20 | 2. The app should display connected clients and buttons for recording control 21 | 22 | #### Client smartphones setup 23 | 24 | 1. Enable WiFi and connect to the Wi-Fi hotspot. 25 | 26 | #### Recording video 27 | 28 | 1. [Optional step] Press the ```calculate period``` button. The app will analyze frame stream and use the calculated frame period in further synchronization steps. 29 | 2. Adjust exposure and ISO to your needs. 30 | 3. Press the ```phase align``` button. 31 | 4. Press the ```record video``` button to start synchronized video recording. 32 | 5. Get videos from RecSync folder in smartphone root directory. 33 | 34 | #### Extraction and matching of the frames 35 | 36 | ``` 37 | Requirements: 38 | 39 | - Python 40 | - ffmpeg 41 | ``` 42 | 43 | 1. Navigate to ```utils``` directory in the repository. 44 | 2. Run ```./match.sh ```. 45 | 3. Frames will be extracted to directories ```output/1``` and ```output/2``` with timestamps in filenames, output directory will also contain ```match.csv``` file in the following format: 46 | ``` 47 | timestamp_1(ns) timestamp_2(ns) 48 | ``` 49 | 50 | ### Our contribution: 51 | 52 | - Integrated **synchronized video recording** 53 | - Scripts for extraction, alignment and processing of video frames 54 | - Experiment with flash blinking to evaluate video frames synchronization accuracy 55 | - Panoramic video demo with automated Hugin stitching 56 | 57 | ### Panoramic video stitching demo 58 | 59 | ### [Link to youtube demo video](https://youtu.be/W6iANtCuQ-o) 60 | 61 | - We provide scripts to **stitch 2 syncronized smatphone videos** with Hujin panorama CLI tools 62 | - Usage: 63 | - Run ```./make_demo.sh {VIDEO_LEFT} {VIDEO_RIGHT}``` 64 | 65 | ### This work is based on "Wireless Software Synchronization of Multiple Distributed Cameras" 66 | 67 | Reference code for the paper 68 | [Wireless Software Synchronization of Multiple Distributed Cameras](https://arxiv.org/abs/1812.09366). 69 | _Sameer Ansari, Neal Wadhwa, Rahul Garg, Jiawen Chen_, ICCP 2019. 70 | -------------------------------------------------------------------------------- /utils/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .idea/ 132 | 133 | # Other 134 | *.png 135 | 136 | *.pcd 137 | 138 | *.pto 139 | 140 | *.jpg 141 | 142 | *.csv 143 | 144 | *.tif 145 | 146 | *.mp4 147 | 148 | *.npy 149 | 150 | *.yaml 151 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SyncConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Ports and other constants used by SoftwareSync. */ 20 | public class SyncConstants { 21 | public static final int SOCKET_WAIT_TIME_MS = 500; 22 | 23 | /** Heartbeat period between clients and leader. */ 24 | public static final long HEARTBEAT_PERIOD_NS = TimeUtils.secondsToNanos(1); 25 | 26 | /** 27 | * Time until a lack of a heartbeat acknowledge from leader means a lost connection. Similarly the 28 | * time until a lack of a heartbeat from a client means the client is stale. 29 | */ 30 | public static final long STALE_TIME_NS = 2 * HEARTBEAT_PERIOD_NS; 31 | 32 | /** Time until a given offsetNs by the leader is considered stale. */ 33 | public static final long STALE_OFFSET_TIME_NS = TimeUtils.secondsToNanos(60 * 60); 34 | 35 | /** RPC. */ 36 | public static final int RPC_PORT = 8244; 37 | public static final int RPC_BUFFER_SIZE = 1024; 38 | 39 | /** RPC Method ids. 40 | * [0 - 9,999] Reserved for SoftwareSync. 41 | * - [0 - 99] Synchronization-related. 42 | * - [100 - 199] Messages. 43 | * [10,000+ ] Available to user applications. 44 | */ 45 | public static final int METHOD_HEARTBEAT = 1; 46 | public static final int METHOD_HEARTBEAT_ACK = 2; 47 | public static final int METHOD_OFFSET_UPDATE = 3; 48 | 49 | /* Define user RPC method ids using values greater or equal to this. */ 50 | public static final int START_NON_SOFTWARESYNC_METHOD_IDS = 1_000; 51 | 52 | public static final int METHOD_MSG_ADDED_CLIENT = 1_101; 53 | public static final int METHOD_MSG_REMOVED_CLIENT = 1_102; 54 | public static final int METHOD_MSG_WAITING_FOR_LEADER = 1_103; 55 | public static final int METHOD_MSG_SYNCING = 1_104; 56 | public static final int METHOD_MSG_OFFSET_UPDATED = 1_105; 57 | 58 | 59 | /** Clock Sync - Simple Network Time Protocol (SNTP). */ 60 | public static final int SNTP_PORT = 9428; 61 | public static final int SNTP_BUFFER_SIZE = 512; 62 | public static final int NUM_SNTP_CYCLES = 300; 63 | public static final long MIN_ROUND_TRIP_LATENCY_NS = TimeUtils.millisToNanos(1); 64 | 65 | private SyncConstants() {} 66 | } 67 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/ClientInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import java.net.InetAddress; 20 | 21 | /** 22 | * Utility immutable class for providing accessors for client name, address, ip local address 23 | * ending, current best accuracy, and last known heartbeat. 24 | */ 25 | public final class ClientInfo { 26 | private final String name; 27 | private final InetAddress address; 28 | private final long offsetNs; 29 | private final long syncAccuracyNs; 30 | private final long lastHeartbeatNs; 31 | 32 | static ClientInfo create( 33 | String name, InetAddress address, long offset, long syncAccuracy, long lastHeartbeat) { 34 | return new ClientInfo(name, address, offset, syncAccuracy, lastHeartbeat); 35 | } 36 | 37 | static ClientInfo create(String name, InetAddress address) { 38 | return new ClientInfo( 39 | name, address, /*offsetNs=*/ 0, /*syncAccuracyNs=*/ 0, /*lastHeartbeatNs=*/ 0); 40 | } 41 | 42 | private ClientInfo( 43 | String name, InetAddress address, long offsetNs, long syncAccuracyNs, long lastHeartbeatNs) { 44 | this.name = name; 45 | this.address = address; 46 | this.offsetNs = offsetNs; 47 | this.syncAccuracyNs = syncAccuracyNs; 48 | this.lastHeartbeatNs = lastHeartbeatNs; 49 | } 50 | 51 | public String name() { 52 | return name; 53 | } 54 | 55 | public InetAddress address() { 56 | return address; 57 | } 58 | 59 | /** 60 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain. The client can 61 | * take their local_time to get leader_time via: local_time (leader - client) = leader_time. 62 | */ 63 | public long offset() { 64 | return offsetNs; 65 | } 66 | 67 | /** 68 | * The worst case error in the clock domains between leader and client for this response, in 69 | * nanoseconds of the AP SystemClock domain. 70 | */ 71 | public long syncAccuracy() { 72 | return syncAccuracyNs; 73 | } 74 | 75 | /* The last time a client heartbeat was detected. */ 76 | public long lastHeartbeat() { 77 | return lastHeartbeatNs; 78 | } 79 | 80 | @Override 81 | public String toString() { 82 | return String.format("%s[%.2f ms]", name(), TimeUtils.nanosToMillis((double) syncAccuracy())); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/AutoFitSurfaceView.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.content.Context; 20 | import android.util.AttributeSet; 21 | import android.util.Log; 22 | import android.view.SurfaceView; 23 | 24 | /** A {@link SurfaceView} that can be adjusted to a specified aspect ratio. */ 25 | public class AutoFitSurfaceView extends SurfaceView { 26 | 27 | private static final String TAG = "AutoFitSurfaceView"; 28 | 29 | private int ratioWidth = 0; 30 | private int ratioHeight = 0; 31 | 32 | public AutoFitSurfaceView(Context context) { 33 | this(context, null); 34 | } 35 | 36 | public AutoFitSurfaceView(Context context, AttributeSet attrs) { 37 | this(context, attrs, 0); 38 | } 39 | 40 | public AutoFitSurfaceView(Context context, AttributeSet attrs, int defStyle) { 41 | super(context, attrs, defStyle); 42 | } 43 | 44 | /** 45 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio 46 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that is, 47 | * calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 48 | * 49 | * @param width Relative horizontal size 50 | * @param height Relative vertical size 51 | */ 52 | public void setAspectRatio(int width, int height) { 53 | if (width < 0 || height < 0) { 54 | throw new IllegalArgumentException("Size cannot be negative."); 55 | } 56 | ratioWidth = width; 57 | ratioHeight = height; 58 | requestLayout(); 59 | } 60 | 61 | @Override 62 | protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { 63 | super.onMeasure(widthMeasureSpec, heightMeasureSpec); 64 | int width = MeasureSpec.getSize(widthMeasureSpec); 65 | int height = MeasureSpec.getSize(heightMeasureSpec); 66 | if (0 == ratioWidth || 0 == ratioHeight) { 67 | Log.d( 68 | TAG, 69 | String.format( 70 | "aspect ratio is 0 x 0 (uninitialized), setting measured" + " dimension to: %d x %d", 71 | width, height)); 72 | setMeasuredDimension(width, height); 73 | } else { 74 | if (width < height * ratioWidth / ratioHeight) { 75 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height)); 76 | setMeasuredDimension(width, width * ratioHeight / ratioWidth); 77 | } else { 78 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height)); 79 | setMeasuredDimension(height * ratioWidth / ratioHeight, height); 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/phasealign/PeriodCalculator.java: -------------------------------------------------------------------------------- 1 | package com.googleresearch.capturesync.softwaresync.phasealign; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.Comparator; 6 | import java.util.List; 7 | import java.util.Timer; 8 | import java.util.TimerTask; 9 | import java.util.concurrent.CountDownLatch; 10 | import java.util.stream.Collectors; 11 | 12 | public class PeriodCalculator { 13 | private final static long CALC_DURATION_MS = 10000; 14 | private volatile boolean shouldRegister; 15 | private ArrayList registeredTimestamps; 16 | 17 | public PeriodCalculator() { 18 | registeredTimestamps = new ArrayList<>(); 19 | } 20 | 21 | // Blocking call, returns 0 in case of error 22 | public long getPeriodNs() throws InterruptedException { 23 | // Start recording timestamps 24 | registeredTimestamps = new ArrayList<>(); 25 | shouldRegister = true; 26 | final CountDownLatch latch = new CountDownLatch(1); 27 | TimerTask task = new TimerTask() { 28 | public void run() { 29 | // Stop recording timestamps and calculate period 30 | shouldRegister = false; 31 | latch.countDown(); 32 | } 33 | }; 34 | Timer timer = new Timer("Timer"); 35 | 36 | timer.schedule(task, CALC_DURATION_MS); 37 | latch.await(); 38 | return calcPeriodNsClusters(getDiff(registeredTimestamps)); 39 | } 40 | 41 | private ArrayList getDiff(ArrayList arrayList) { 42 | Long prev = 0L; 43 | ArrayList result = new ArrayList<>(); 44 | for (Long aLong : arrayList) { 45 | if (prev == 0L) { 46 | prev = aLong; 47 | } else { 48 | result.add(aLong - prev); 49 | prev = aLong; 50 | } 51 | } 52 | return result; 53 | } 54 | 55 | private long calcPeriodNsClusters(ArrayList numArray) { 56 | long initEstimate = Collections.min(numArray); 57 | long nClust = Math.round(1.0 * Collections.max(numArray) / initEstimate); 58 | double weightedSum = 0L; 59 | for (int i = 0; i < nClust; i++) { 60 | int finalI = i; 61 | ArrayList clust = (ArrayList)numArray.stream().filter( 62 | x -> (x > (finalI + 0.5)*initEstimate) && (x < (finalI + 1.5)*initEstimate) 63 | ).collect(Collectors.toList()); 64 | if (clust.size() > 0) { 65 | weightedSum += 1.0 * median(clust) / (i + 1) * clust.size(); 66 | } 67 | } 68 | return Math.round(weightedSum / numArray.size()); 69 | } 70 | 71 | private long calcPeriodNsMedian(ArrayList numArray) { 72 | return median(numArray); 73 | } 74 | 75 | private long median(ArrayList numArray) { 76 | Collections.sort(numArray); 77 | int middle = numArray.size() / 2; 78 | middle = middle > 0 && middle % 2 == 0 ? middle - 1 : middle; 79 | return numArray.get(middle); 80 | } 81 | 82 | public void onFrameTimestamp(long timestampNs) { 83 | // Register timestamp 84 | if (shouldRegister) { 85 | registeredTimestamps.add(timestampNs); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/NetworkHelpers.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.net.wifi.WifiManager; 20 | import java.net.Inet4Address; 21 | import java.net.InetAddress; 22 | import java.net.NetworkInterface; 23 | import java.net.SocketException; 24 | import java.net.UnknownHostException; 25 | import java.util.Collections; 26 | import java.util.List; 27 | 28 | /** Helper functions for determining local IP address and host IP address on the network. */ 29 | public final class NetworkHelpers { 30 | private final WifiManager wifiManager; 31 | 32 | /** 33 | * Constructor providing the wifi manager for determining hotspot leader address. 34 | * 35 | * @param wifiManager via '(WifiManager) context.getSystemService(Context.WIFI_SERVICE);'' 36 | */ 37 | public NetworkHelpers(WifiManager wifiManager) { 38 | this.wifiManager = wifiManager; 39 | } 40 | 41 | /** 42 | * Returns the IP address of the hotspot host. Requires ACCESS_WIFI_STATE permission. Note: This 43 | * may not work on several devices. 44 | * 45 | * @return IP address of the hotspot host. 46 | */ 47 | public InetAddress getHotspotServerAddress() throws SocketException, UnknownHostException { 48 | if (wifiManager.isWifiEnabled()) { 49 | // Return the DHCP server address, which is the hotspot ip address. 50 | int serverAddress = wifiManager.getDhcpInfo().serverAddress; 51 | // DhcpInfo integer addresses are Little Endian and InetAddresses.fromInteger() are Big Endian 52 | // so reverse the bytes before converting from Integer. 53 | byte[] addressBytes = { 54 | (byte) (0xff & serverAddress), 55 | (byte) (0xff & (serverAddress >> 8)), 56 | (byte) (0xff & (serverAddress >> 16)), 57 | (byte) (0xff & (serverAddress >> 24)) 58 | }; 59 | return InetAddress.getByAddress(addressBytes); 60 | } 61 | // If wifi is disabled, then this is the hotspot host, return the local ip address. 62 | return getIPAddress(); 63 | } 64 | 65 | /** 66 | * Finds this devices's IPv4 address that is not localhost and not on a dummy interface. 67 | * 68 | * @return the String IP address on success. 69 | * @throws SocketException on failure to find a suitable IP address. 70 | */ 71 | public static InetAddress getIPAddress() throws SocketException { 72 | List interfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); 73 | for (NetworkInterface intf : interfaces) { 74 | for (InetAddress addr : Collections.list(intf.getInetAddresses())) { 75 | if (!addr.isLoopbackAddress() 76 | && !intf.getName().equals("dummy0") 77 | && addr instanceof Inet4Address) { 78 | return addr; 79 | } 80 | } 81 | } 82 | throw new SocketException("No viable IP Network addresses found."); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | 88 | @rem Execute Gradle 89 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 90 | 91 | :end 92 | @rem End local scope for the variables with windows NT shell 93 | if "%ERRORLEVEL%"=="0" goto mainEnd 94 | 95 | :fail 96 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 97 | rem the _cmd.exe /c_ return code! 98 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 99 | exit /b 1 100 | 101 | :mainEnd 102 | if "%OS%"=="Windows_NT" endlocal 103 | 104 | :omega 105 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.util.Log; 20 | import java.io.IOException; 21 | import java.net.DatagramPacket; 22 | import java.net.DatagramSocket; 23 | import java.net.SocketTimeoutException; 24 | import java.nio.ByteBuffer; 25 | 26 | /** 27 | * SNTP listener thread, which is expected to only run while the client is waiting for the leader to 28 | * synchronize with it (WAITING_FOR_LEADER state). The {@link SimpleNetworkTimeProtocol} class is 29 | * used by the leader to send the SNTP messages that this thread listens for. 30 | */ 31 | public class SntpListener extends Thread { 32 | 33 | private static final String TAG = "SntpListener"; 34 | private boolean running; 35 | private final DatagramSocket nptpSocket; 36 | private final int nptpPort; 37 | private final Ticker localClock; 38 | 39 | public SntpListener(Ticker localClock, DatagramSocket nptpSocket, int nptpPort) { 40 | this.localClock = localClock; 41 | this.nptpSocket = nptpSocket; 42 | this.nptpPort = nptpPort; 43 | } 44 | 45 | public void stopRunning() { 46 | running = false; 47 | } 48 | 49 | @Override 50 | public void run() { 51 | running = true; 52 | 53 | Log.w(TAG, "Starting SNTP Listener thread."); 54 | 55 | byte[] buf = new byte[SyncConstants.SNTP_BUFFER_SIZE]; 56 | while (running && !nptpSocket.isClosed()) { 57 | DatagramPacket packet = new DatagramPacket(buf, buf.length); 58 | try { 59 | // Listen for PTP messages. 60 | nptpSocket.receive(packet); 61 | 62 | // 2 (B) - Recv UDP message with t0 at time t0'. 63 | long t0r = localClock.read(); 64 | 65 | final int longSize = Long.SIZE / Byte.SIZE; 66 | 67 | if (packet.getLength() != longSize) { 68 | Log.e( 69 | TAG, 70 | "Received UDP message with incorrect packet length " 71 | + packet.getLength() 72 | + ", skipping."); 73 | continue; 74 | } 75 | 76 | // 3 (B) - Send UDP message with t0,t0',t1 at time t1. 77 | long t1 = localClock.read(); 78 | ByteBuffer buffer = ByteBuffer.allocate(3 * longSize); 79 | buffer.put(packet.getData(), 0, longSize); 80 | buffer.putLong(longSize, t0r); 81 | buffer.putLong(2 * longSize, t1); 82 | byte[] bufferArray = buffer.array(); 83 | 84 | // Send SNTP response back. 85 | DatagramPacket response = 86 | new DatagramPacket(bufferArray, bufferArray.length, packet.getAddress(), nptpPort); 87 | nptpSocket.send(response); 88 | } catch (SocketTimeoutException e) { 89 | // It is normal to time out most of the time, continue. 90 | } catch (IOException e) { 91 | if (nptpSocket.isClosed()) { 92 | // Stop here if socket is closed. 93 | return; 94 | } 95 | throw new IllegalStateException("SNTP Thread didn't close gracefully: " + e); 96 | } 97 | } 98 | Log.w(TAG, "SNTP Listener thread finished."); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/CSVLogger.java: -------------------------------------------------------------------------------- 1 | package com.googleresearch.capturesync.softwaresync; 2 | 3 | import android.content.Context; 4 | import android.hardware.camera2.CameraAccessException; 5 | import android.hardware.camera2.CameraCharacteristics; 6 | import android.hardware.camera2.CameraManager; 7 | import android.hardware.camera2.CameraMetadata; 8 | import android.os.Build; 9 | import android.os.Environment; 10 | 11 | import com.googleresearch.capturesync.MainActivity; 12 | 13 | import java.io.BufferedWriter; 14 | import java.io.File; 15 | import java.io.FileWriter; 16 | import java.io.IOException; 17 | import java.nio.file.Files; 18 | import java.nio.file.Path; 19 | import java.nio.file.Paths; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | public class CSVLogger { 24 | private final BufferedWriter writer; 25 | 26 | public boolean isClosed() { 27 | return isClosed; 28 | } 29 | 30 | private volatile boolean isClosed; 31 | 32 | public CSVLogger(String dirName, String filename, MainActivity context) throws IOException { 33 | isClosed = true; 34 | File sdcard = Environment.getExternalStorageDirectory(); 35 | Path dir = Files.createDirectories(Paths.get(sdcard.getAbsolutePath(), dirName)); 36 | File file = new File(dir.toFile(), filename); 37 | writer = new BufferedWriter(new FileWriter(file, true)); 38 | 39 | // Important: adding comment with metadata before isClosed is changed 40 | // writer.write("# " + Build.MODEL); 41 | // writer.write("\n"); 42 | // writer.write("# " + Build.VERSION.SDK_INT); 43 | // writer.write("\n"); 44 | // 45 | // 46 | // CameraManager manager = (CameraManager) context.getSystemService(Context.CAMERA_SERVICE); 47 | // try { 48 | // String[] idList = manager.getCameraIdList(); 49 | // 50 | // Map levels = new HashMap() {{ 51 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY, "LEGACY"); 52 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED, "LIMITED"); 53 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL, "FULL"); 54 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_EXTERNAL, "EXTERNAL"); 55 | // put(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_3, "LEVEL_3"); 56 | // }}; 57 | // 58 | // 59 | // int maxCameraCnt = idList.length; 60 | // writer.write("# " + maxCameraCnt); 61 | // writer.write("\n"); 62 | // for (int index = 0; index < maxCameraCnt; index++) { 63 | // String cameraId = manager.getCameraIdList()[index]; 64 | // CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); 65 | // String deviceLevel = levels.get(characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)); 66 | // String source = characteristics.get( 67 | // CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE 68 | // ) == CameraMetadata.SENSOR_INFO_TIMESTAMP_SOURCE_REALTIME ? "REALTIME" : "UNKNOWN"; 69 | // writer.write("# " + source + " " + deviceLevel); 70 | // writer.write("\n"); 71 | // } 72 | // } catch (CameraAccessException e) { 73 | // e.printStackTrace(); 74 | // } 75 | 76 | 77 | isClosed = false; 78 | } 79 | 80 | public void logLine(String line) throws IOException { 81 | writer.write(line); 82 | writer.write("\n"); 83 | }; 84 | 85 | public void close() { 86 | try { 87 | isClosed = true; 88 | writer.close(); 89 | } catch (IOException e) { 90 | e.printStackTrace(); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 10 | 11 | 15 | 16 | 20 | 21 | 22 | 23 | 27 | 28 | 33 | 34 | 40 | 41 | 42 | 43 | 48 |