├── settings.gradle ├── app ├── src │ └── main │ │ ├── res │ │ ├── mipmap-hdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ ├── mipmap-mdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ ├── mipmap-xhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ ├── mipmap-xxxhdpi │ │ │ ├── ic_launcher.png │ │ │ └── ic_launcher_round.png │ │ ├── raw │ │ │ ├── pixel1_phaseconfig.json │ │ │ ├── pixel2_phaseconfig.json │ │ │ ├── pixel3_240fps_phaseconfig.json │ │ │ ├── pixel3_phaseconfig.json │ │ │ └── pixel3_60fps_phaseconfig.json │ │ ├── values │ │ │ ├── colors.xml │ │ │ ├── dimens.xml │ │ │ ├── styles.xml │ │ │ └── strings.xml │ │ ├── mipmap-anydpi-v26 │ │ │ ├── ic_launcher.xml │ │ │ └── ic_launcher_round.xml │ │ ├── drawable-v24 │ │ │ └── ic_launcher_foreground.xml │ │ ├── layout │ │ │ └── activity_main.xml │ │ └── drawable │ │ │ └── ic_launcher_background.xml │ │ ├── java │ │ └── com │ │ │ └── googleresearch │ │ │ └── capturesync │ │ │ ├── softwaresync │ │ │ ├── Ticker.java │ │ │ ├── SystemTicker.java │ │ │ ├── RpcCallback.java │ │ │ ├── TimeUtils.java │ │ │ ├── TimeDomainConverter.java │ │ │ ├── SntpOffsetResponse.java │ │ │ ├── SyncConstants.java │ │ │ ├── ClientInfo.java │ │ │ ├── NetworkHelpers.java │ │ │ ├── SntpListener.java │ │ │ ├── phasealign │ │ │ │ ├── PhaseConfig.java │ │ │ │ ├── PhaseResponse.java │ │ │ │ └── PhaseAligner.java │ │ │ ├── SoftwareSyncClient.java │ │ │ ├── SoftwareSyncBase.java │ │ │ ├── SimpleNetworkTimeProtocol.java │ │ │ └── SoftwareSyncLeader.java │ │ │ ├── Frame.java │ │ │ ├── Constants.java │ │ │ ├── AutoFitSurfaceView.java │ │ │ ├── CaptureRequestFactory.java │ │ │ ├── PhaseAlignController.java │ │ │ ├── CameraController.java │ │ │ ├── SoftwareSyncController.java │ │ │ ├── ResultProcessor.java │ │ │ └── ImageMetadataSynchronizer.java │ │ └── AndroidManifest.xml ├── proguard-rules.pro ├── build.gradle └── app.iml ├── CaptureSync.iml ├── CONTRIBUTING.md ├── scripts └── yuv2rgb.py ├── LICENSE └── README.md /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':app' 2 | rootProject.name='CaptureSync' 3 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-hdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-mdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-hdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-hdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-mdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-mdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/libsoftwaresync/HEAD/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel1_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33325000, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 200000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33170000 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel2_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33319419, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 200000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33170000 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel3_240fps_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 4164011, 3 | "goalPhaseNs": 2000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 196499, 6 | "minExposureNs": 4164011 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel3_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 33320000, 3 | "goalPhaseNs": 15000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 200000, 6 | "minExposureNs": 33370000 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/raw/pixel3_60fps_phaseconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "periodNs": 16630105, 3 | "goalPhaseNs": 7000000, 4 | "alignThresholdNs": 100000, 5 | "overheadNs": 196499, 6 | "minExposureNs": 16530105 7 | } 8 | -------------------------------------------------------------------------------- /app/src/main/res/values/colors.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | #008577 4 | #00574B 5 | #D81B60 6 | 7 | -------------------------------------------------------------------------------- /app/src/main/res/values/dimens.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16dp 5 | 16dp 6 | 7 | 8 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /app/src/main/res/values/styles.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 14 | 15 | 16 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /app/proguard-rules.pro: -------------------------------------------------------------------------------- 1 | # Add project specific ProGuard rules here. 2 | # You can control the set of applied configuration files using the 3 | # proguardFiles setting in build.gradle. 4 | # 5 | # For more details, see 6 | # http://developer.android.com/guide/developing/tools/proguard.html 7 | 8 | # If your project uses WebView with JS, uncomment the following 9 | # and specify the fully qualified class name to the JavaScript interface 10 | # class: 11 | #-keepclassmembers class fqcn.of.javascript.interface.for.webview { 12 | # public *; 13 | #} 14 | 15 | # Uncomment this to preserve the line number information for 16 | # debugging stack traces. 17 | #-keepattributes SourceFile,LineNumberTable 18 | 19 | # If you keep the line number information, uncomment this to 20 | # hide the original source file name. 21 | #-renamesourcefileattribute SourceFile 22 | -------------------------------------------------------------------------------- /CaptureSync.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/Ticker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Interface for getting the time in nanoseconds, abstracting out accessing system time. */ 20 | public interface Ticker { 21 | /* Returns the time in nanoseconds. */ 22 | long read(); 23 | } 24 | -------------------------------------------------------------------------------- /app/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'com.android.application' 2 | 3 | android { 4 | compileSdkVersion 29 5 | buildToolsVersion "29.0.2" 6 | defaultConfig { 7 | applicationId "com.googleresearch.capturesync" 8 | minSdkVersion 28 9 | targetSdkVersion 29 10 | versionCode 1 11 | versionName "1.0" 12 | testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" 13 | } 14 | buildTypes { 15 | release { 16 | minifyEnabled false 17 | proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' 18 | } 19 | } 20 | compileOptions { 21 | sourceCompatibility = 1.8 22 | targetCompatibility = 1.8 23 | } 24 | } 25 | 26 | dependencies { 27 | implementation fileTree(dir: 'libs', include: ['*.jar']) 28 | implementation 'androidx.appcompat:appcompat:1.0.2' 29 | testImplementation 'junit:junit:4.12' 30 | androidTestImplementation 'androidx.test.ext:junit:1.1.0' 31 | androidTestImplementation 'androidx.test.espresso:espresso-core:3.1.1' 32 | } 33 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SystemTicker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.os.SystemClock; 20 | 21 | /** Simple implementation of Ticker interface using the SystemClock elapsed realtime clock. */ 22 | public class SystemTicker implements Ticker { 23 | @Override 24 | public long read() { 25 | return SystemClock.elapsedRealtimeNanos(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/RpcCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Interface for RPC callbacks, the base methods used for communicating between devices. */ 20 | public interface RpcCallback { 21 | 22 | /** 23 | * The callback method called when an RPC is received. 24 | * 25 | * @param payload Contains the payload sent by the RPC. 26 | */ 27 | void call(String payload); 28 | } 29 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /app/src/main/AndroidManifest.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 17 | 18 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Helper conversions between time scales. */ 20 | public final class TimeUtils { 21 | 22 | public static double nanosToMillis(double nanos) { 23 | return nanos / 1_000_000L; 24 | } 25 | 26 | public static long nanosToSeconds(long nanos) { 27 | return nanos / 1_000_000_000L; 28 | } 29 | 30 | public static double nanosToSeconds(double nanos) { 31 | return nanos / 1_000_000_000L; 32 | } 33 | 34 | public static long millisToNanos(long millis) { 35 | return millis * 1_000_000L; 36 | } 37 | 38 | public static long secondsToNanos(int seconds) { 39 | return seconds * 1_000_000_000L; 40 | } 41 | 42 | private TimeUtils() {} 43 | } 44 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/Frame.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.hardware.camera2.CaptureResult; 20 | import android.media.Image; 21 | import com.googleresearch.capturesync.ImageMetadataSynchronizer.Output; 22 | import java.io.Closeable; 23 | 24 | /** A holder for a CaptureResult with multiple outputs. */ 25 | public class Frame implements Closeable { 26 | public final CaptureResult result; 27 | public final Output output; 28 | private boolean closed = false; 29 | 30 | public Frame(CaptureResult result, Output output) { 31 | this.result = result; 32 | this.output = output; 33 | } 34 | 35 | @Override 36 | public void close() { 37 | if (closed) { 38 | throw new IllegalStateException("This Frame is already closed"); 39 | } 40 | for (Image image : output.images) { 41 | image.close(); 42 | } 43 | output.close(); 44 | closed = true; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/TimeDomainConverter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** 20 | * Interface used to manage time domain conversion, implemented by {@link SoftwareSyncBase}. This 21 | * allows {@link ResultProcessor} to convert sensor timestamps to the synchronized time domain 22 | * without needing full access to the softwaresync object. 23 | */ 24 | public interface TimeDomainConverter { 25 | 26 | /** 27 | * Calculates the leader time associated with the given local time in nanoseconds. The local time 28 | * must be in the SystemClock.elapsedRealTimeNanos() localClock domain, nanosecond units. This 29 | * includes timestamps such as the sensor timestamp from the camera. leader_time = 30 | * local_elapsed_time_ns + leader_from_local_ns. 31 | * 32 | * @param localTimeNs given local time (local clock SystemClock.elapsedRealtimeNanos() domain). 33 | * @return leader synchronized time in nanoseconds. 34 | */ 35 | long leaderTimeForLocalTimeNs(long localTimeNs); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /app/src/main/res/values/strings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CaptureSync 6 | 7 | Capture Still 8 | 9 | Align Phases 10 | 11 | Align 2A 12 | 13 | 14 | SoftwareSync Demo App 15 | 16 | Phase 17 | 18 | Exposure: 19 | 20 | Sensitivity: 21 | 22 | 23 | You must grant camera, read, and write permissions 24 | 25 | Failed to open Camera2 service 26 | 27 | Failed to capture viewfinder frame 28 | 29 | No image outputs chosen 30 | 31 | Buffers 32 | 33 | 34 | -------------------------------------------------------------------------------- /app/src/main/res/drawable-v24/ic_launcher_foreground.xml: -------------------------------------------------------------------------------- 1 | 7 | 12 | 13 | 19 | 22 | 25 | 26 | 27 | 28 | 34 | 35 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/Constants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.hardware.camera2.CameraCharacteristics; 20 | import com.googleresearch.capturesync.softwaresync.TimeUtils; 21 | 22 | /** Constants including what type of images to save and the save directory. */ 23 | public final class Constants { 24 | public static final int DEFAULT_CAMERA_FACING = CameraCharacteristics.LENS_FACING_BACK; 25 | 26 | /** 27 | * Delay from capture button press to capture, giving network time to send messages to clients. 28 | */ 29 | public static final long FUTURE_TRIGGER_DELAY_NS = TimeUtils.millisToNanos(500); 30 | 31 | /* Set at least one of {SAVE_YUV, SAVE_RAW} to true to save any data. */ 32 | public static final boolean SAVE_YUV = true; 33 | 34 | // TODO: Implement saving ImageFormat.RAW10 to DNG. 35 | // DngCreator works with ImageFormat.RAW_SENSOR but it is slow and power-hungry. 36 | public static final boolean SAVE_RAW = false; 37 | 38 | // TODO(samansari): Turn SAVE_JPG_FROM_YUV into a checkbox instead. 39 | /* Set true to save a JPG to the gallery for preview. This is slow but gives you a "postview". */ 40 | public static final boolean SAVE_JPG_FROM_YUV = true; 41 | public static final int JPG_QUALITY = 95; 42 | 43 | public static final boolean USE_FULL_SCREEN_IMMERSIVE = false; 44 | 45 | private Constants() {} 46 | } 47 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpOffsetResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** AutoValue class for SNTP offsetNs, synchronization accuracy and status. */ 20 | public final class SntpOffsetResponse { 21 | private final long offsetNs; 22 | private final long syncAccuracyNs; 23 | private final boolean status; 24 | 25 | static SntpOffsetResponse create(long offset, long syncAccuracy, boolean status) { 26 | return new SntpOffsetResponse(offset, syncAccuracy, status); 27 | } 28 | 29 | private SntpOffsetResponse(long offsetNs, long syncAccuracyNs, boolean status) { 30 | this.offsetNs = offsetNs; 31 | this.syncAccuracyNs = syncAccuracyNs; 32 | this.status = status; 33 | } 34 | 35 | /** 36 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain. 37 | * 38 | *

The client can take their local_time to get leader_time via: local_time (leader - client) = 39 | * leader_time. 40 | */ 41 | public long offsetNs() { 42 | return offsetNs; 43 | } 44 | 45 | /** 46 | * The worst case error in the clock domains between leader and client for this response, in 47 | * nanoseconds of the AP SystemClock domain. 48 | */ 49 | public long syncAccuracyNs() { 50 | return syncAccuracyNs; 51 | } 52 | 53 | /** The success status of this response. */ 54 | public boolean status() { 55 | return status; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /scripts/yuv2rgb.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2019 The Google Research Authors. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """Process raw NV21 images to jpg or png images using ffmpeg. 17 | 18 | This script requires ffmpeg (https://www.ffmpeg.org/download.html). 19 | 20 | Run with: 21 | python3 yuv2rgb.py img_.nv21 nv21_metadata_.txt 22 | out.. 23 | """ 24 | 25 | import argparse 26 | import subprocess 27 | 28 | 29 | def parse_meta(path): 30 | with open(path, 'r') as f: 31 | lines = [l.strip() for l in f.readlines()] 32 | img_width = lines[0].split(' ')[1] 33 | img_height = lines[1].split(' ')[1] 34 | img_pixel_format = lines[2].split(' ')[1] 35 | return img_width, img_height, img_pixel_format 36 | 37 | 38 | if __name__ == '__main__': 39 | parser = argparse.ArgumentParser() 40 | parser.add_argument( 41 | '-y', '--overwrite', help='Overwrite output.', action='store_true') 42 | parser.add_argument('input') 43 | parser.add_argument('meta') 44 | parser.add_argument('output', help='output filename (ending in .jpg or .png)') 45 | args = parser.parse_args() 46 | 47 | if not args.output.endswith('.jpg') and not args.output.endswith('.png'): 48 | raise ValueError('output must end in jpg or png.') 49 | 50 | width, height, pixel_format = parse_meta(args.meta) 51 | pixel_format = pixel_format.lower() 52 | 53 | overwrite_flag = '-y' if args.overwrite else '-n' 54 | 55 | cmd = [ 56 | 'ffmpeg', overwrite_flag, '-f', 'image2', '-vcodec', 'rawvideo', 57 | '-pix_fmt', pixel_format, '-s', f'{width}x{height}', '-i', args.input, 58 | args.output 59 | ] 60 | 61 | subprocess.call(cmd) 62 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SyncConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | /** Ports and other constants used by SoftwareSync. */ 20 | public class SyncConstants { 21 | public static final int SOCKET_WAIT_TIME_MS = 500; 22 | 23 | /** Heartbeat period between clients and leader. */ 24 | public static final long HEARTBEAT_PERIOD_NS = TimeUtils.secondsToNanos(1); 25 | 26 | /** 27 | * Time until a lack of a heartbeat acknowledge from leader means a lost connection. Similarly the 28 | * time until a lack of a heartbeat from a client means the client is stale. 29 | */ 30 | public static final long STALE_TIME_NS = 2 * HEARTBEAT_PERIOD_NS; 31 | 32 | /** Time until a given offsetNs by the leader is considered stale. */ 33 | public static final long STALE_OFFSET_TIME_NS = TimeUtils.secondsToNanos(60 * 60); 34 | 35 | /** RPC. */ 36 | public static final int RPC_PORT = 8244; 37 | public static final int RPC_BUFFER_SIZE = 1024; 38 | 39 | /** RPC Method ids. 40 | * [0 - 9,999] Reserved for SoftwareSync. 41 | * - [0 - 99] Synchronization-related. 42 | * - [100 - 199] Messages. 43 | * [10,000+ ] Available to user applications. 44 | */ 45 | public static final int METHOD_HEARTBEAT = 1; 46 | public static final int METHOD_HEARTBEAT_ACK = 2; 47 | public static final int METHOD_OFFSET_UPDATE = 3; 48 | 49 | /* Define user RPC method ids using values greater or equal to this. */ 50 | public static final int START_NON_SOFTWARESYNC_METHOD_IDS = 1_000; 51 | 52 | public static final int METHOD_MSG_ADDED_CLIENT = 1_101; 53 | public static final int METHOD_MSG_REMOVED_CLIENT = 1_102; 54 | public static final int METHOD_MSG_WAITING_FOR_LEADER = 1_103; 55 | public static final int METHOD_MSG_SYNCING = 1_104; 56 | public static final int METHOD_MSG_OFFSET_UPDATED = 1_105; 57 | 58 | /** Clock Sync - Simple Network Time Protocol (SNTP). */ 59 | public static final int SNTP_PORT = 9428; 60 | public static final int SNTP_BUFFER_SIZE = 512; 61 | public static final int NUM_SNTP_CYCLES = 300; 62 | public static final long MIN_ROUND_TRIP_LATENCY_NS = TimeUtils.millisToNanos(1); 63 | 64 | private SyncConstants() {} 65 | } 66 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/ClientInfo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import java.net.InetAddress; 20 | 21 | /** 22 | * Utility immutable class for providing accessors for client name, address, ip local address 23 | * ending, current best accuracy, and last known heartbeat. 24 | */ 25 | public final class ClientInfo { 26 | private final String name; 27 | private final InetAddress address; 28 | private final long offsetNs; 29 | private final long syncAccuracyNs; 30 | private final long lastHeartbeatNs; 31 | 32 | static ClientInfo create( 33 | String name, InetAddress address, long offset, long syncAccuracy, long lastHeartbeat) { 34 | return new ClientInfo(name, address, offset, syncAccuracy, lastHeartbeat); 35 | } 36 | 37 | static ClientInfo create(String name, InetAddress address) { 38 | return new ClientInfo( 39 | name, address, /*offsetNs=*/ 0, /*syncAccuracyNs=*/ 0, /*lastHeartbeatNs=*/ 0); 40 | } 41 | 42 | private ClientInfo( 43 | String name, InetAddress address, long offsetNs, long syncAccuracyNs, long lastHeartbeatNs) { 44 | this.name = name; 45 | this.address = address; 46 | this.offsetNs = offsetNs; 47 | this.syncAccuracyNs = syncAccuracyNs; 48 | this.lastHeartbeatNs = lastHeartbeatNs; 49 | } 50 | 51 | public String name() { 52 | return name; 53 | } 54 | 55 | public InetAddress address() { 56 | return address; 57 | } 58 | 59 | /** 60 | * The time delta (leader - client) in nanoseconds of the AP SystemClock domain. The client can 61 | * take their local_time to get leader_time via: local_time (leader - client) = leader_time. 62 | */ 63 | public long offset() { 64 | return offsetNs; 65 | } 66 | 67 | /** 68 | * The worst case error in the clock domains between leader and client for this response, in 69 | * nanoseconds of the AP SystemClock domain. 70 | */ 71 | public long syncAccuracy() { 72 | return syncAccuracyNs; 73 | } 74 | 75 | /* The last time a client heartbeat was detected. */ 76 | public long lastHeartbeat() { 77 | return lastHeartbeatNs; 78 | } 79 | 80 | @Override 81 | public String toString() { 82 | return String.format("%s[%.2f ms]", name(), TimeUtils.nanosToMillis((double) syncAccuracy())); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/AutoFitSurfaceView.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync; 18 | 19 | import android.content.Context; 20 | import android.util.AttributeSet; 21 | import android.util.Log; 22 | import android.view.SurfaceView; 23 | 24 | /** A {@link SurfaceView} that can be adjusted to a specified aspect ratio. */ 25 | public class AutoFitSurfaceView extends SurfaceView { 26 | 27 | private static final String TAG = "AutoFitSurfaceView"; 28 | 29 | private int ratioWidth = 0; 30 | private int ratioHeight = 0; 31 | 32 | public AutoFitSurfaceView(Context context) { 33 | this(context, null); 34 | } 35 | 36 | public AutoFitSurfaceView(Context context, AttributeSet attrs) { 37 | this(context, attrs, 0); 38 | } 39 | 40 | public AutoFitSurfaceView(Context context, AttributeSet attrs, int defStyle) { 41 | super(context, attrs, defStyle); 42 | } 43 | 44 | /** 45 | * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio 46 | * calculated from the parameters. Note that the actual sizes of parameters don't matter, that is, 47 | * calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. 48 | * 49 | * @param width Relative horizontal size 50 | * @param height Relative vertical size 51 | */ 52 | public void setAspectRatio(int width, int height) { 53 | if (width < 0 || height < 0) { 54 | throw new IllegalArgumentException("Size cannot be negative."); 55 | } 56 | ratioWidth = width; 57 | ratioHeight = height; 58 | requestLayout(); 59 | } 60 | 61 | @Override 62 | protected void onMeasure(int widthMeasureSpec, int heightMeasureSpec) { 63 | super.onMeasure(widthMeasureSpec, heightMeasureSpec); 64 | int width = MeasureSpec.getSize(widthMeasureSpec); 65 | int height = MeasureSpec.getSize(heightMeasureSpec); 66 | if (0 == ratioWidth || 0 == ratioHeight) { 67 | Log.d( 68 | TAG, 69 | String.format( 70 | "aspect ratio is 0 x 0 (uninitialized), setting measured" + " dimension to: %d x %d", 71 | width, height)); 72 | setMeasuredDimension(width, height); 73 | } else { 74 | if (width < height * ratioWidth / ratioHeight) { 75 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height)); 76 | setMeasuredDimension(width, width * ratioHeight / ratioWidth); 77 | } else { 78 | Log.d(TAG, String.format("setting measured dimension to %d x %d", width, height)); 79 | setMeasuredDimension(height * ratioWidth / ratioHeight, height); 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/NetworkHelpers.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.net.wifi.WifiManager; 20 | import java.net.Inet4Address; 21 | import java.net.InetAddress; 22 | import java.net.NetworkInterface; 23 | import java.net.SocketException; 24 | import java.net.UnknownHostException; 25 | import java.util.Collections; 26 | import java.util.List; 27 | 28 | /** Helper functions for determining local IP address and host IP address on the network. */ 29 | public final class NetworkHelpers { 30 | private final WifiManager wifiManager; 31 | 32 | /** 33 | * Constructor providing the wifi manager for determining hotspot leader address. 34 | * 35 | * @param wifiManager via '(WifiManager) context.getSystemService(Context.WIFI_SERVICE);'' 36 | */ 37 | public NetworkHelpers(WifiManager wifiManager) { 38 | this.wifiManager = wifiManager; 39 | } 40 | 41 | /** 42 | * Returns the IP address of the hotspot host. Requires ACCESS_WIFI_STATE permission. Note: This 43 | * may not work on several devices. 44 | * 45 | * @return IP address of the hotspot host. 46 | */ 47 | public InetAddress getHotspotServerAddress() throws SocketException, UnknownHostException { 48 | if (wifiManager.isWifiEnabled()) { 49 | // Return the DHCP server address, which is the hotspot ip address. 50 | int serverAddress = wifiManager.getDhcpInfo().serverAddress; 51 | // DhcpInfo integer addresses are Little Endian and InetAddresses.fromInteger() are Big Endian 52 | // so reverse the bytes before converting from Integer. 53 | byte[] addressBytes = { 54 | (byte) (0xff & serverAddress), 55 | (byte) (0xff & (serverAddress >> 8)), 56 | (byte) (0xff & (serverAddress >> 16)), 57 | (byte) (0xff & (serverAddress >> 24)) 58 | }; 59 | return InetAddress.getByAddress(addressBytes); 60 | } 61 | // If wifi is disabled, then this is the hotspot host, return the local ip address. 62 | return getIPAddress(); 63 | } 64 | 65 | /** 66 | * Finds this devices's IPv4 address that is not localhost and not on a dummy interface. 67 | * 68 | * @return the String IP address on success. 69 | * @throws SocketException on failure to find a suitable IP address. 70 | */ 71 | public static InetAddress getIPAddress() throws SocketException { 72 | List interfaces = Collections.list(NetworkInterface.getNetworkInterfaces()); 73 | for (NetworkInterface intf : interfaces) { 74 | for (InetAddress addr : Collections.list(intf.getInetAddresses())) { 75 | if (!addr.isLoopbackAddress() 76 | && !intf.getName().equals("dummy0") 77 | && addr instanceof Inet4Address) { 78 | return addr; 79 | } 80 | } 81 | } 82 | throw new SocketException("No viable IP Network addresses found."); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /app/src/main/java/com/googleresearch/capturesync/softwaresync/SntpListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 The Google Research Authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.googleresearch.capturesync.softwaresync; 18 | 19 | import android.util.Log; 20 | import java.io.IOException; 21 | import java.net.DatagramPacket; 22 | import java.net.DatagramSocket; 23 | import java.net.SocketTimeoutException; 24 | import java.nio.ByteBuffer; 25 | 26 | /** 27 | * SNTP listener thread, which is expected to only run while the client is waiting for the leader to 28 | * synchronize with it (WAITING_FOR_LEADER state). The {@link SimpleNetworkTimeProtocol} class is 29 | * used by the leader to send the SNTP messages that this thread listens for. 30 | */ 31 | public class SntpListener extends Thread { 32 | 33 | private static final String TAG = "SntpListener"; 34 | private boolean running; 35 | private final DatagramSocket nptpSocket; 36 | private final int nptpPort; 37 | private final Ticker localClock; 38 | 39 | public SntpListener(Ticker localClock, DatagramSocket nptpSocket, int nptpPort) { 40 | this.localClock = localClock; 41 | this.nptpSocket = nptpSocket; 42 | this.nptpPort = nptpPort; 43 | } 44 | 45 | public void stopRunning() { 46 | running = false; 47 | } 48 | 49 | @Override 50 | public void run() { 51 | running = true; 52 | 53 | Log.w(TAG, "Starting SNTP Listener thread."); 54 | 55 | byte[] buf = new byte[SyncConstants.SNTP_BUFFER_SIZE]; 56 | while (running && !nptpSocket.isClosed()) { 57 | DatagramPacket packet = new DatagramPacket(buf, buf.length); 58 | try { 59 | // Listen for PTP messages. 60 | nptpSocket.receive(packet); 61 | 62 | // 2 (B) - Recv UDP message with t0 at time t0'. 63 | long t0r = localClock.read(); 64 | 65 | final int longSize = Long.SIZE / Byte.SIZE; 66 | 67 | if (packet.getLength() != longSize) { 68 | Log.e( 69 | TAG, 70 | "Received UDP message with incorrect packet length " 71 | + packet.getLength() 72 | + ", skipping."); 73 | continue; 74 | } 75 | 76 | // 3 (B) - Send UDP message with t0,t0',t1 at time t1. 77 | long t1 = localClock.read(); 78 | ByteBuffer buffer = ByteBuffer.allocate(3 * longSize); 79 | buffer.put(packet.getData(), 0, longSize); 80 | buffer.putLong(longSize, t0r); 81 | buffer.putLong(2 * longSize, t1); 82 | byte[] bufferArray = buffer.array(); 83 | 84 | // Send SNTP response back. 85 | DatagramPacket response = 86 | new DatagramPacket(bufferArray, bufferArray.length, packet.getAddress(), nptpPort); 87 | nptpSocket.send(response); 88 | } catch (SocketTimeoutException e) { 89 | // It is normal to time out most of the time, continue. 90 | } catch (IOException e) { 91 | if (nptpSocket.isClosed()) { 92 | // Stop here if socket is closed. 93 | return; 94 | } 95 | throw new IllegalStateException("SNTP Thread didn't close gracefully: " + e); 96 | } 97 | } 98 | Log.w(TAG, "SNTP Listener thread finished."); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /app/src/main/res/layout/activity_main.xml: -------------------------------------------------------------------------------- 1 | 10 | 11 | 15 | 16 | 20 | 21 | 22 | 23 | 27 | 28 | 33 | 34 | 40 | 41 | 42 | 43 | 48 |