├── Podfile.lock ├── Pods ├── Manifest.lock ├── Target Support Files │ └── Pods-AnySense │ │ ├── Pods-AnySense.modulemap │ │ ├── Pods-AnySense-dummy.m │ │ ├── Pods-AnySense-acknowledgements.markdown │ │ ├── Pods-AnySense-umbrella.h │ │ ├── Pods-AnySense.debug.xcconfig │ │ ├── Pods-AnySense.release.xcconfig │ │ ├── Pods-AnySense-acknowledgements.plist │ │ └── Pods-AnySense-Info.plist └── Pods.xcodeproj │ └── project.pbxproj ├── AnySense ├── Assets.xcassets │ ├── Contents.json │ ├── AppIcon.appiconset │ │ ├── logo.jpg │ │ └── Contents.json │ ├── AnySense logo.imageset │ │ ├── anysense_logo_scaled.png │ │ └── Contents.json │ ├── StartButton.imageset │ │ ├── START (1000 x 1000 px) (1).png │ │ └── Contents.json │ ├── AccentColor.colorset │ │ └── Contents.json │ ├── DeviceWordColor.colorset │ │ └── Contents.json │ ├── ViewButtonColor.colorset │ │ └── Contents.json │ ├── DeviceTopColor.colorset │ │ └── Contents.json │ ├── TabBackgroundColor.colorset │ │ └── Contents.json │ └── customizedBackground.colorset │ │ └── Contents.json ├── Info.plist ├── AnySenseApp.swift ├── MainPage.swift ├── Views │ ├── peripheralView.swift │ ├── ContentView.swift │ ├── accountView.swift │ └── readView.swift ├── AnySenseLaunchScreen.storyboard ├── dataStorage.swift └── Managers │ ├── USBManager.swift │ ├── BluetoothManager.swift │ └── ARViewContainer.swift ├── Podfile ├── AnySense.xcodeproj ├── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ │ └── IDEWorkspaceChecks.plist └── project.pbxproj ├── AnySense.xcworkspace └── contents.xcworkspacedata ├── LICENSE ├── Privacy-Policy.md ├── .gitignore └── README.md /Podfile.lock: -------------------------------------------------------------------------------- 1 | PODFILE CHECKSUM: 6b87639c1dec426e7f105f24069e220f15ed5fb6 2 | 3 | COCOAPODS: 1.16.2 4 | -------------------------------------------------------------------------------- /Pods/Manifest.lock: -------------------------------------------------------------------------------- 1 | PODFILE CHECKSUM: 6b87639c1dec426e7f105f24069e220f15ed5fb6 2 | 3 | COCOAPODS: 1.16.2 4 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "info" : { 3 | "author" : "xcode", 4 | "version" : 1 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/AppIcon.appiconset/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYU-robot-learning/AnySense/HEAD/AnySense/Assets.xcassets/AppIcon.appiconset/logo.jpg -------------------------------------------------------------------------------- /Podfile: -------------------------------------------------------------------------------- 1 | project 'AnySense.xcodeproj' 2 | 3 | target 'AnySense' do 4 | # Comment the next line if you don't want to use dynamic frameworks 5 | use_frameworks! 6 | 7 | end 8 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense.modulemap: -------------------------------------------------------------------------------- 1 | framework module Pods_AnySense { 2 | umbrella header "Pods-AnySense-umbrella.h" 3 | 4 | export * 5 | module * { export * } 6 | } 7 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/AnySense logo.imageset/anysense_logo_scaled.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYU-robot-learning/AnySense/HEAD/AnySense/Assets.xcassets/AnySense logo.imageset/anysense_logo_scaled.png -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense-dummy.m: -------------------------------------------------------------------------------- 1 | #import 2 | @interface PodsDummy_Pods_AnySense : NSObject 3 | @end 4 | @implementation PodsDummy_Pods_AnySense 5 | @end 6 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/StartButton.imageset/START (1000 x 1000 px) (1).png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NYU-robot-learning/AnySense/HEAD/AnySense/Assets.xcassets/StartButton.imageset/START (1000 x 1000 px) (1).png -------------------------------------------------------------------------------- /AnySense.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense-acknowledgements.markdown: -------------------------------------------------------------------------------- 1 | # Acknowledgements 2 | This application makes use of the following third party libraries: 3 | Generated by CocoaPods - https://cocoapods.org 4 | -------------------------------------------------------------------------------- /AnySense.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /AnySense.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense-umbrella.h: -------------------------------------------------------------------------------- 1 | #ifdef __OBJC__ 2 | #import 3 | #else 4 | #ifndef FOUNDATION_EXPORT 5 | #if defined(__cplusplus) 6 | #define FOUNDATION_EXPORT extern "C" 7 | #else 8 | #define FOUNDATION_EXPORT extern 9 | #endif 10 | #endif 11 | #endif 12 | 13 | 14 | FOUNDATION_EXPORT double Pods_AnySenseVersionNumber; 15 | FOUNDATION_EXPORT const unsigned char Pods_AnySenseVersionString[]; 16 | 17 | -------------------------------------------------------------------------------- /AnySense/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | UIFileSharingEnabled 6 | 7 | UILaunchScreen 8 | 9 | UILaunchScreen 10 | 11 | AnySenseLaunchScreen 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/AnySense logo.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "filename" : "anysense_logo_scaled.png", 5 | "idiom" : "universal", 6 | "scale" : "1x" 7 | }, 8 | { 9 | "idiom" : "universal", 10 | "scale" : "2x" 11 | }, 12 | { 13 | "idiom" : "universal", 14 | "scale" : "3x" 15 | } 16 | ], 17 | "info" : { 18 | "author" : "xcode", 19 | "version" : 1 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/StartButton.imageset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "filename" : "START (1000 x 1000 px) (1).png", 5 | "idiom" : "universal", 6 | "scale" : "1x" 7 | }, 8 | { 9 | "idiom" : "universal", 10 | "scale" : "2x" 11 | }, 12 | { 13 | "idiom" : "universal", 14 | "scale" : "3x" 15 | } 16 | ], 17 | "info" : { 18 | "author" : "xcode", 19 | "version" : 1 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense.debug.xcconfig: -------------------------------------------------------------------------------- 1 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO 2 | GCC_PREPROCESSOR_DEFINITIONS = $(inherited) COCOAPODS=1 3 | PODS_BUILD_DIR = ${BUILD_DIR} 4 | PODS_CONFIGURATION_BUILD_DIR = ${PODS_BUILD_DIR}/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME) 5 | PODS_PODFILE_DIR_PATH = ${SRCROOT}/. 6 | PODS_ROOT = ${SRCROOT}/Pods 7 | PODS_XCFRAMEWORKS_BUILD_DIR = $(PODS_CONFIGURATION_BUILD_DIR)/XCFrameworkIntermediates 8 | USE_RECURSIVE_SCRIPT_INPUTS_IN_SCRIPT_PHASES = YES 9 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense.release.xcconfig: -------------------------------------------------------------------------------- 1 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = NO 2 | GCC_PREPROCESSOR_DEFINITIONS = $(inherited) COCOAPODS=1 3 | PODS_BUILD_DIR = ${BUILD_DIR} 4 | PODS_CONFIGURATION_BUILD_DIR = ${PODS_BUILD_DIR}/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME) 5 | PODS_PODFILE_DIR_PATH = ${SRCROOT}/. 6 | PODS_ROOT = ${SRCROOT}/Pods 7 | PODS_XCFRAMEWORKS_BUILD_DIR = $(PODS_CONFIGURATION_BUILD_DIR)/XCFrameworkIntermediates 8 | USE_RECURSIVE_SCRIPT_INPUTS_IN_SCRIPT_PHASES = YES 9 | -------------------------------------------------------------------------------- /AnySense/AnySenseApp.swift: -------------------------------------------------------------------------------- 1 | // 2 | // AnySenseApp.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/5/22. 6 | // 7 | 8 | import SwiftUI 9 | import BackgroundTasks 10 | 11 | @main 12 | struct AnySenseApp: App { 13 | @StateObject var appStatus = AppInformation() 14 | @StateObject var bluetoothManager = BluetoothManager() 15 | var body: some Scene { 16 | WindowGroup { 17 | ContentView() 18 | .environmentObject(appStatus) 19 | .environmentObject(bluetoothManager) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "filename" : "logo.jpg", 5 | "idiom" : "universal", 6 | "platform" : "ios", 7 | "size" : "1024x1024" 8 | }, 9 | { 10 | "appearances" : [ 11 | { 12 | "appearance" : "luminosity", 13 | "value" : "dark" 14 | } 15 | ], 16 | "idiom" : "universal", 17 | "platform" : "ios", 18 | "size" : "1024x1024" 19 | }, 20 | { 21 | "appearances" : [ 22 | { 23 | "appearance" : "luminosity", 24 | "value" : "tinted" 25 | } 26 | ], 27 | "idiom" : "universal", 28 | "platform" : "ios", 29 | "size" : "1024x1024" 30 | } 31 | ], 32 | "info" : { 33 | "author" : "xcode", 34 | "version" : 1 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/AccentColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.965", 9 | "green" : "0.473", 10 | "red" : "0.204" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "display-p3", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "1.000", 27 | "green" : "1.000", 28 | "red" : "1.000" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/DeviceWordColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.000", 9 | "green" : "0.000", 10 | "red" : "0.000" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "srgb", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "1.000", 27 | "green" : "1.000", 28 | "red" : "1.000" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/ViewButtonColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.913", 9 | "green" : "0.913", 10 | "red" : "0.904" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "srgb", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "1.000", 27 | "green" : "1.000", 28 | "red" : "1.000" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/DeviceTopColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.969", 9 | "green" : "0.949", 10 | "red" : "0.949" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "display-p3", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "0.141", 27 | "green" : "0.129", 28 | "red" : "0.122" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/TabBackgroundColor.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.951", 9 | "green" : "0.951", 10 | "red" : "0.941" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "display-p3", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "0.180", 27 | "green" : "0.176", 28 | "red" : "0.173" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /AnySense/Assets.xcassets/customizedBackground.colorset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "colors" : [ 3 | { 4 | "color" : { 5 | "color-space" : "display-p3", 6 | "components" : { 7 | "alpha" : "1.000", 8 | "blue" : "0.969", 9 | "green" : "0.949", 10 | "red" : "0.949" 11 | } 12 | }, 13 | "idiom" : "universal" 14 | }, 15 | { 16 | "appearances" : [ 17 | { 18 | "appearance" : "luminosity", 19 | "value" : "dark" 20 | } 21 | ], 22 | "color" : { 23 | "color-space" : "display-p3", 24 | "components" : { 25 | "alpha" : "1.000", 26 | "blue" : "0.141", 27 | "green" : "0.129", 28 | "red" : "0.122" 29 | } 30 | }, 31 | "idiom" : "universal" 32 | } 33 | ], 34 | "info" : { 35 | "author" : "xcode", 36 | "version" : 1 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense-acknowledgements.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | PreferenceSpecifiers 6 | 7 | 8 | FooterText 9 | This application makes use of the following third party libraries: 10 | Title 11 | Acknowledgements 12 | Type 13 | PSGroupSpecifier 14 | 15 | 16 | FooterText 17 | Generated by CocoaPods - https://cocoapods.org 18 | Title 19 | 20 | Type 21 | PSGroupSpecifier 22 | 23 | 24 | StringsTable 25 | Acknowledgements 26 | Title 27 | Acknowledgements 28 | 29 | 30 | -------------------------------------------------------------------------------- /Pods/Target Support Files/Pods-AnySense/Pods-AnySense-Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | ${PODS_DEVELOPMENT_LANGUAGE} 7 | CFBundleExecutable 8 | ${EXECUTABLE_NAME} 9 | CFBundleIdentifier 10 | ${PRODUCT_BUNDLE_IDENTIFIER} 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | ${PRODUCT_NAME} 15 | CFBundlePackageType 16 | FMWK 17 | CFBundleShortVersionString 18 | 1.0.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | ${CURRENT_PROJECT_VERSION} 23 | NSPrincipalClass 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 NYU Generalizable Robotics and AI Lab (GRAIL) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Privacy-Policy.md: -------------------------------------------------------------------------------- 1 | # Privacy Policy 2 | 3 | ## Last Updated: February 21, 2025 4 | 5 | ### 1. Introduction 6 | Welcome to AnySense. Your privacy is important to us, and we are committed to protecting it. This Privacy Policy explains that we **do not collect, store, or share any personal data** from users. 7 | 8 | ### 2. Information We Do Not Collect 9 | We do not collect any: 10 | - Personal information (such as name, email, phone number, etc.). 11 | - Location data. 12 | - Biometric data. 13 | - Behavioral or analytics data. 14 | - Any other user-related data. 15 | 16 | ### 3. How We Handle Your Data 17 | Since we do not collect any personal data, there is no data to process, store, or share with third parties. 18 | 19 | ### 4. Third-Party Services 20 | Our app does not integrate with third-party analytics, advertising, or tracking services. 21 | 22 | ### 5. Children's Privacy 23 | Our application does not knowingly collect any personal data from children under 13 years of age. 24 | 25 | ### 6. Changes to This Policy 26 | We may update this Privacy Policy from time to time. Any changes will be posted on this page with the updated date. 27 | 28 | ### 7. Contact Us 29 | If you have any questions about this Privacy Policy, please contact us at: 30 | raunaqbhirangi@nyu.edu 31 | 32 | --- 33 | 34 | This privacy policy is provided for informational purposes and does not constitute legal advice. If you have specific concerns, consider consulting a legal expert. 35 | 36 | -------------------------------------------------------------------------------- /AnySense/MainPage.swift: -------------------------------------------------------------------------------- 1 | // 2 | // MainPage.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/5/22. 6 | // 7 | 8 | import SwiftUI 9 | 10 | struct MainPage: View { 11 | @EnvironmentObject private var appStatus : AppInformation 12 | @EnvironmentObject private var bluetoothManager: BluetoothManager 13 | @Environment(\.scenePhase) private var phase 14 | let arViewModel: ARViewModel 15 | // Start the default page be the read page 16 | @State private var selection = 1 17 | 18 | var body: some View { 19 | TabView(selection: $selection){ 20 | Group{ 21 | PeripheralView() 22 | .tabItem { 23 | Label("ble-device", systemImage: "iphone.gen1.radiowaves.left.and.right") 24 | } 25 | .tag(0) 26 | 27 | ReadView(arViewModel: arViewModel) 28 | .tabItem { 29 | Label("read", systemImage: "dot.scope") 30 | } 31 | .tag(1) 32 | 33 | 34 | SettingsView() 35 | .tabItem { 36 | Label("settings", systemImage: "gear") 37 | 38 | } 39 | .tag(2) 40 | } 41 | .toolbarBackground(.tabBackground, for: .tabBar) 42 | .toolbarBackground(.visible, for: .tabBar) 43 | } 44 | .accentColor(.accentColor) 45 | } 46 | 47 | } 48 | 49 | #Preview { 50 | MainPage(arViewModel: ARViewModel()) 51 | .environmentObject(AppInformation()) 52 | .environmentObject(BluetoothManager()) 53 | } 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Xcode 2 | # 3 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore 4 | 5 | .DS_Store 6 | build/ 7 | DerivedData/ 8 | 9 | ## User settings 10 | xcuserdata/ 11 | 12 | ## Obj-C/Swift specific 13 | *.hmap 14 | 15 | ## App packaging 16 | *.ipa 17 | *.dSYM.zip 18 | *.dSYM 19 | 20 | ## Playgrounds 21 | timeline.xctimeline 22 | playground.xcworkspace 23 | 24 | # Swift Package Manager 25 | # 26 | # Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. 27 | # Packages/ 28 | # Package.pins 29 | # Package.resolved 30 | # *.xcodeproj 31 | # 32 | # Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata 33 | # hence it is not needed unless you have added a package configuration file to your project 34 | # .swiftpm 35 | 36 | .build/ 37 | 38 | # CocoaPods 39 | # 40 | # We recommend against adding the Pods directory to your .gitignore. However 41 | # you should judge for yourself, the pros and cons are mentioned at: 42 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 43 | # 44 | # Pods/ 45 | # 46 | # Add this line if you want to avoid checking in source code from the Xcode workspace 47 | # *.xcworkspace 48 | 49 | # Carthage 50 | # 51 | # Add this line if you want to avoid checking in source code from Carthage dependencies. 52 | # Carthage/Checkouts 53 | 54 | Carthage/Build/ 55 | 56 | # fastlane 57 | # 58 | # It is recommended to not store the screenshots in the git repo. 59 | # Instead, use fastlane to re-generate the screenshots whenever they are needed. 60 | # For more information about the recommended setup visit: 61 | # https://docs.fastlane.tools/best-practices/source-control/#source-control 62 | 63 | fastlane/report.xml 64 | fastlane/Preview.html 65 | fastlane/screenshots/**/*.png 66 | fastlane/test_output 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # AnySense 3 | [AnySense](https://anysense.app) is an iPhone application that integrates the iPhone's sensory suite with external multisensory inputs via Bluetooth and wired interfaces, enabling both offline data collection and online streaming to robots. Currently, we record RGB and depth videos, metric depth frames, streamed Bluetooth data appended into a binary file and timestamped pose data as a `.txt` file. Example streaming code for streaming Bluetooth data can be found on [AnySkin](https://any-skin.github.io). We also allow for USB streaming by simply connecting the iPhone to your computer and using this [accompanying library](https://github.com/NYU-robot-learning/anysense-streaming) forked from the excellent [record3d](https://github.com/marek-simonik/record3d) library. 4 | 5 | ## App Screenshots 6 | sdfasdf 7 | asddsfa 8 | 9 | AnySense data storage format: 10 | - Streamed Bluetooth Data (.bin) 11 | - RGB video (.mp4) 12 | - Depth video (.mp4) 13 | - Metric depth frames (*.bin) 14 | - Pose Data (.txt) 15 | 16 | ## Setting up locally 17 | 1. Clone the github repository: 18 | ``` 19 | git clone https://github.com/NYU-robot-learning/AnySense.git 20 | ``` 21 | 2. Install [XCode](https://developer.apple.com/xcode/) and open the repository in XCode. 22 | 23 | 3. In the app settings, navigate to the "Signing & Capabilities" section. Make sure the "Automatically manage signing" checkbox is checked. Add your account and set a unique bundle identifier. 24 | 25 | 4. Plug in your IOS device to your Mac, and follow instructions for trusting the computer, [enabling Developer mode](https://developer.apple.com/documentation/xcode/enabling-developer-mode-on-a-device). You should now be able to build the app. 26 | 27 | ## Contact me 28 | Questions about AnySense? Contact: raunaqbhirangi@nyu.edu 29 | 30 | The team at Generalizable Robotics and AI Lab, NYU: [Raunaq Bhirangi](https://raunaqbhirangi.nyu.edu), Zeyu (Michael) Bian, [Venkatesh Pattabiraman](https://venkyp.com), [Haritheja Etukuru](https://haritheja.com), [Mehmet Enes Erciyes](https://eneserciyes.github.io), [Nur Muhammad Mahi Shafiullah](https://mahis.life), [Lerrel Pinto](https://www.lerrelpinto.com) 31 | -------------------------------------------------------------------------------- /AnySense/Views/peripheralView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // peripheralView.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/7/29. 6 | // 7 | 8 | import SwiftUI 9 | import CoreBluetooth 10 | 11 | struct singleBLEPeripheral: View { 12 | @EnvironmentObject var bluetoothManager: BluetoothManager 13 | @EnvironmentObject var appStatus: AppInformation 14 | @State private var isConnected = false 15 | let name: String 16 | let uuid: UUID 17 | 18 | var body: some View { 19 | HStack(alignment: .firstTextBaseline) { 20 | Text(name) 21 | .font(.headline) 22 | Spacer() 23 | if(!appStatus.ifBluetoothConnected || isConnected){ 24 | Button(action: toggleConnection) { 25 | Text(isConnected ? "Disconnect" : "Connect") 26 | .foregroundColor(isConnected ? .red : .blue) 27 | } 28 | .frame(minWidth: 120.0) 29 | .buttonStyle(.bordered) 30 | } 31 | } 32 | } 33 | 34 | private func toggleConnection() { 35 | UIImpactFeedbackGenerator(style: appStatus.hapticFeedbackLevel).impactOccurred() 36 | 37 | if !isConnected{ 38 | bluetoothManager.connectToPeripheral(withUUID: uuid) { result in 39 | switch result { 40 | case .success(let connectedPeripheral): 41 | print("Successfully connected to: \(connectedPeripheral.name ?? "Unknown Device")") 42 | case .failure(let error): 43 | print("Connection failed: \(error.localizedDescription)") 44 | } 45 | } 46 | appStatus.ifBluetoothConnected = true 47 | } else { 48 | appStatus.ifBluetoothConnected = false 49 | bluetoothManager.disconnectFromDevice() 50 | } 51 | 52 | isConnected = !isConnected 53 | } 54 | } 55 | 56 | struct PeripheralView: View { 57 | @EnvironmentObject var appStatus : AppInformation 58 | @EnvironmentObject var bluetoothManager: BluetoothManager 59 | var body: some View { 60 | VStack{ 61 | Text("Devices Detected") 62 | .font(.body) 63 | .frame(width: 500.0, height: 50) 64 | .ignoresSafeArea() 65 | .foregroundStyle(.deviceWord) 66 | .background(.deviceTop) 67 | .padding(.top, 5) 68 | List(Array(bluetoothManager.discoveredPeripherals.keys), id: \.self) { uuid in 69 | if let peripheral = bluetoothManager.discoveredPeripherals[uuid] { 70 | singleBLEPeripheral( 71 | name: peripheral.name ?? "Unknown Device", 72 | uuid: peripheral.identifier 73 | ) 74 | } 75 | } 76 | .scrollContentBackground(.hidden) 77 | .background(Color.customizedBackground) 78 | } 79 | .background(Color.customizedBackground) 80 | } 81 | } 82 | 83 | #Preview { 84 | PeripheralView().environmentObject(AppInformation()).environmentObject(BluetoothManager()) 85 | } 86 | -------------------------------------------------------------------------------- /AnySense/AnySenseLaunchScreen.storyboard: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /AnySense/Views/ContentView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ContentView.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/5/22. 6 | // 7 | 8 | import SwiftUI 9 | import CoreBluetooth 10 | import AVFoundation 11 | 12 | struct ContentView: View { 13 | @EnvironmentObject var appStatus : AppInformation 14 | @EnvironmentObject var bluetoothManager: BluetoothManager 15 | @StateObject private var arViewModel = ARViewModel() 16 | 17 | @State private var hasPermissions = false 18 | @State private var showPermissionAlert = false 19 | @State private var showMainPage = false 20 | 21 | var body: some View { 22 | ZStack{ 23 | Color.customizedBackground 24 | .ignoresSafeArea() 25 | VStack { 26 | Image("AnySense logo") 27 | .resizable() 28 | .frame(width:220.0, height: 220.0) 29 | .cornerRadius(30.0) 30 | Text("Welcome to AnySense") 31 | .font(.title) 32 | .multilineTextAlignment(.center) 33 | .bold() 34 | if hasPermissions { 35 | Button(action: {showMainPage = true}) { 36 | Image("StartButton") 37 | .resizable() 38 | .frame(width: 200, height: 200) 39 | } 40 | .padding(.top, 10.0) 41 | .background(.customizedBackground) 42 | } 43 | } 44 | .onAppear { 45 | checkPermissions() 46 | } 47 | .fullScreenCover(isPresented: $showMainPage) { 48 | MainPage(arViewModel: arViewModel) 49 | } 50 | .alert(isPresented: $showPermissionAlert) { 51 | Alert( 52 | title: Text("Camera Access Required"), 53 | message: Text("Please enable camera access in Settings to use AR features."), 54 | primaryButton: .default(Text("Settings"), action: openAppSettings), 55 | secondaryButton: .cancel() 56 | ) 57 | } 58 | } 59 | 60 | } 61 | 62 | private func checkPermissions() { 63 | PermissionsManager.checkCameraPermissions { granted in 64 | if granted { 65 | arViewModel.setupARSession() 66 | hasPermissions = true 67 | } else { 68 | showPermissionAlert = true 69 | } 70 | } 71 | } 72 | 73 | private func openAppSettings() { 74 | if let url = URL(string: UIApplication.openSettingsURLString) { 75 | UIApplication.shared.open(url) 76 | } 77 | } 78 | 79 | } 80 | 81 | class PermissionsManager { 82 | static func checkCameraPermissions(completion: @escaping (Bool) -> Void) { 83 | switch AVCaptureDevice.authorizationStatus(for: .video) { 84 | case .authorized: 85 | completion(true) 86 | case .notDetermined: 87 | AVCaptureDevice.requestAccess(for: .video) { granted in 88 | DispatchQueue.main.async { 89 | completion(granted) 90 | } 91 | } 92 | default: 93 | completion(false) 94 | } 95 | } 96 | } 97 | 98 | 99 | class AppInformation : ObservableObject{ 100 | @Published var animationFPS: Double = 30.0 101 | @Published var hapticFeedbackLevel: UIImpactFeedbackGenerator.FeedbackStyle = .medium 102 | @Published var rgbdVideoStreaming: StreamingMode = .off 103 | @Published var gridProjectionTrigger: GridMode = .off 104 | @Published var colorMapTrigger: Bool = false 105 | @Published var ifBluetoothConnected: Bool = false 106 | } 107 | 108 | 109 | #Preview { 110 | ContentView() 111 | .environmentObject(AppInformation()) 112 | } 113 | 114 | 115 | -------------------------------------------------------------------------------- /AnySense/Views/accountView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // accountView.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/5/27. 6 | // 7 | 8 | import SwiftUI 9 | 10 | struct SettingsView : View{ 11 | @EnvironmentObject var appStatus: AppInformation 12 | 13 | let frequencyOptions = ["0.1", "0.05", "0.033", "0.02", "0.017", "0.01"] // Frequency options 14 | 15 | var body : some View{ 16 | ZStack{ 17 | Color.customizedBackground 18 | .ignoresSafeArea() 19 | Form{ 20 | Section(header: Text("GENERAL")) { 21 | HStack { 22 | VStack(alignment: .leading, spacing: 8) { 23 | // Title and caption 24 | Text("Live RGBD Streaming") 25 | .font(.body) // Regular font 26 | .foregroundColor(.primary) 27 | Text("Stream to your computer") 28 | .font(.caption) 29 | .foregroundColor(.gray) 30 | } 31 | Spacer() 32 | let binding = Binding( 33 | get: { appStatus.rgbdVideoStreaming }, 34 | set: { newValue in 35 | if newValue != StreamingMode.wifi { // Disable Option wifi 36 | appStatus.rgbdVideoStreaming = newValue 37 | } 38 | } 39 | ) 40 | Picker("Streaming Options", selection: binding) { // Temporary fix to keep wifi option but disable it 41 | Text("Wi-Fi").tag(StreamingMode.wifi).opacity(0.5) 42 | Text("USB").tag(StreamingMode.usb) 43 | Text("Off").tag(StreamingMode.off) 44 | } 45 | .pickerStyle(SegmentedPickerStyle()) 46 | .frame(width: 125) // Adjust width for the picker 47 | } 48 | .padding(.vertical, 5) 49 | .padding(.vertical, 5) 50 | HStack{ 51 | Text("Buttons haptic feedback") 52 | .font(.body) 53 | .foregroundColor(.primary) 54 | // .padding(.leading, 20) 55 | Spacer() 56 | Picker("", selection: $appStatus.hapticFeedbackLevel) { 57 | Text("medium").tag(UIImpactFeedbackGenerator.FeedbackStyle.medium) 58 | Text("heavy").tag(UIImpactFeedbackGenerator.FeedbackStyle.heavy) 59 | Text("light").tag(UIImpactFeedbackGenerator.FeedbackStyle.light) 60 | } 61 | .pickerStyle(MenuPickerStyle()) // Dropdown style 62 | .frame(width: 110) 63 | // .padding(.leading, 75) 64 | } 65 | .padding(.vertical, 5) 66 | HStack{ 67 | VStack(alignment: .leading, spacing: 8){ 68 | Picker("Grid projection enabled", selection: $appStatus.gridProjectionTrigger){ 69 | Text("3x3").tag(GridMode._3x3) 70 | Text("5x5").tag(GridMode._5x5) 71 | Text("off").tag(GridMode.off) 72 | } 73 | .pickerStyle(MenuPickerStyle()) 74 | Text("Project grid lines to your camera") 75 | .font(.caption) 76 | .foregroundStyle(.gray) 77 | } 78 | } 79 | 80 | } 81 | // Section(header: Text("INFO")) { 82 | // NavigationLink { 83 | // InstructionView() 84 | // } label: { 85 | // HStack { 86 | // Text("How to use?") 87 | // .font(.body) 88 | // .foregroundColor(.black) 89 | // Spacer() 90 | // } 91 | // } 92 | // NavigationLink { 93 | // fileMarkdownView() 94 | // } label: { 95 | // HStack { 96 | // Text("About") 97 | // .font(.body) 98 | // .foregroundColor(.black) 99 | // Spacer() 100 | // } 101 | // } 102 | // } 103 | } 104 | .scrollContentBackground(.hidden) 105 | } 106 | } 107 | } 108 | 109 | enum StreamingMode: String { 110 | case off = "Off" 111 | case wifi = "Wi-Fi" 112 | case usb = "USB" 113 | } 114 | 115 | enum GridMode: Int { 116 | case off = 0 117 | case _3x3 = 3 118 | case _5x5 = 5 119 | } 120 | 121 | #Preview { 122 | SettingsView() 123 | .environmentObject(AppInformation()) 124 | } 125 | -------------------------------------------------------------------------------- /AnySense/dataStorage.swift: -------------------------------------------------------------------------------- 1 | // 2 | // dataStorage.swift 3 | // AnySense 4 | // 5 | // Created by Michael on 2025/2/1. 6 | // 7 | import SwiftUI 8 | import UniformTypeIdentifiers 9 | import Foundation 10 | 11 | struct TextFile: FileDocument { 12 | // tell the system we support only plain text 13 | static var readableContentTypes = [UTType.text] 14 | 15 | var text = "" 16 | var url : String 17 | 18 | init(url : String){ 19 | self.url = url 20 | } 21 | 22 | // this initializer loads data/Users/bianzeyu/Documents/PolySense/AnySense/dataStorage.swift that has been saved previously 23 | init(configuration: ReadConfiguration) throws { 24 | if let data = configuration.file.regularFileContents { 25 | text = String(decoding: data, as: UTF8.self) 26 | } 27 | url = "" 28 | } 29 | 30 | 31 | func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { 32 | let file = try! FileWrapper(url: URL(fileURLWithPath: url), options: .immediate) 33 | return file 34 | } 35 | } 36 | 37 | struct VideoFile: FileDocument{ 38 | var url: URL 39 | 40 | static var readableContentTypes: [UTType] { [.mpeg4Movie] } 41 | static var writableContentTypes: [UTType] { [.mpeg4Movie] } 42 | 43 | // Initialize with a given URL 44 | init(url: URL) { 45 | self.url = url 46 | } 47 | 48 | // This is called when the system wants to read the previously saved data 49 | init(configuration: ReadConfiguration) throws { 50 | self.url = URL(fileURLWithPath: "") 51 | } 52 | 53 | func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { 54 | return try FileWrapper(url: url, options: .immediate) 55 | } 56 | } 57 | 58 | struct ImageFile: FileDocument { 59 | var url: URL 60 | 61 | static var readableContentTypes: [UTType] { [.jpeg] } 62 | static var writableContentTypes: [UTType] { [.jpeg] } 63 | 64 | init(url: URL) { 65 | self.url = url 66 | } 67 | 68 | init(configuration: ReadConfiguration) throws { 69 | self.url = URL(fileURLWithPath: "") 70 | } 71 | 72 | func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { 73 | return try FileWrapper(url: url, options: .immediate) 74 | } 75 | 76 | 77 | } 78 | enum FileElement { 79 | case videoFile(VideoFile) 80 | case textFile(TextFile) 81 | case directory(SubLevelDirectory) 82 | } 83 | 84 | enum FileElementSub { 85 | case imageFile(ImageFile) 86 | } 87 | 88 | struct SubLevelDirectory: FileDocument{ 89 | var url: URL 90 | var containedFiles: [FileElementSub] 91 | 92 | static var readableContentTypes: [UTType] { [.folder] } 93 | static var writableContentTypes: [UTType] { [.folder] } 94 | 95 | init(url: URL) { 96 | self.url = url 97 | self.containedFiles = [] 98 | do{ 99 | let contents = try FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil) 100 | for content in contents{ 101 | if content.pathExtension.lowercased() == "ipeg" || content.pathExtension.lowercased() == "jpg" { 102 | self.containedFiles.append(.imageFile(ImageFile(url: content))) 103 | } 104 | } 105 | }catch{ 106 | print("Error when extract and append files to export directory") 107 | } 108 | } 109 | 110 | init(configuration: ReadConfiguration) throws { 111 | self.url = URL(fileURLWithPath: "") 112 | self.containedFiles = [] 113 | } 114 | 115 | func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { 116 | let dirWrapper = FileWrapper(directoryWithFileWrappers: [:]) 117 | for file in containedFiles { 118 | switch file { 119 | 120 | case .imageFile(let imageFile): 121 | let fileWrapper = try imageFile.fileWrapper(configuration: configuration) 122 | dirWrapper.addFileWrapper(fileWrapper) 123 | } 124 | } 125 | return dirWrapper 126 | } 127 | } 128 | 129 | 130 | struct DocumentaryFolder: FileDocument { 131 | var files: [FileElement] 132 | 133 | 134 | static var readableContentTypes: [UTType] { [.folder] } 135 | static var writableContentTypes: [UTType] { [.folder] } 136 | 137 | 138 | init(files: [FileElement]) { 139 | self.files = files 140 | } 141 | 142 | 143 | init(configuration: ReadConfiguration) throws { 144 | self.files = [] 145 | // Implement reading logic if needed 146 | } 147 | 148 | 149 | func fileWrapper(configuration: WriteConfiguration) throws -> FileWrapper { 150 | let folderWrapper = FileWrapper(directoryWithFileWrappers: [:]) 151 | 152 | 153 | for file in files { 154 | switch file { 155 | case .videoFile(let videoFile): 156 | let fileWrapper = try videoFile.fileWrapper(configuration: configuration) 157 | fileWrapper.preferredFilename = videoFile.url.lastPathComponent 158 | folderWrapper.addFileWrapper(fileWrapper) 159 | case .textFile(let textFile): 160 | let fileWrapper = try textFile.fileWrapper(configuration: configuration) 161 | fileWrapper.preferredFilename = URL(fileURLWithPath: textFile.url).lastPathComponent 162 | folderWrapper.addFileWrapper(fileWrapper) 163 | case .directory(let directory): 164 | let directoryWrapper = try directory.fileWrapper(configuration: configuration) 165 | directoryWrapper.preferredFilename = directory.url.lastPathComponent 166 | folderWrapper.addFileWrapper(directoryWrapper) 167 | } 168 | } 169 | 170 | return folderWrapper 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /AnySense/Managers/USBManager.swift: -------------------------------------------------------------------------------- 1 | // 2 | // USBManager.swift 3 | // Anysense 4 | // 5 | // Created by Raunaq Bhirangi on 1/13/25. 6 | // 7 | 8 | import Network 9 | import UIKit 10 | import Compression 11 | 12 | 13 | struct PeerTalkHeader { 14 | var a: UInt32 15 | var b: UInt32 16 | var c: UInt32 17 | var body_size: UInt32 18 | } 19 | 20 | struct Record3DHeader { 21 | var rgbWidth: UInt32 22 | var rgbHeight: UInt32 23 | var depthWidth: UInt32 24 | var depthHeight: UInt32 25 | var confidenceWidth: UInt32 26 | var confidenceHeight: UInt32 27 | var rgbSize: UInt32 28 | var depthSize: UInt32 29 | var confidenceMapSize: UInt32 30 | var miscSize: UInt32 31 | var deviceType: UInt32 32 | } 33 | 34 | struct IntrinsicMatrixCoeffs { 35 | var fx: Float 36 | var fy: Float 37 | var tx: Float 38 | var ty: Float 39 | } 40 | 41 | struct CameraPose { 42 | // Quaternion coefficients 43 | var qx: Float 44 | var qy: Float 45 | var qz: Float 46 | var qw: Float 47 | 48 | var tx: Float 49 | var ty: Float 50 | var tz: Float 51 | } 52 | 53 | class USBManager { 54 | private var listener: NWListener? 55 | private var activeConnection: NWConnection? 56 | private var intrinsicMat = IntrinsicMatrixCoeffs(fx:714.178, fy: 714.178, tx: 359.1699, ty:482.075) 57 | private var ptHeader = PeerTalkHeader(a:1, b:1, c:1, body_size: 0) 58 | func connect() { 59 | do { 60 | listener = try NWListener(using: .tcp, on: 1337) // Port 5000 matches libusbmuxd example 61 | listener?.stateUpdateHandler = { state in 62 | switch state { 63 | case .ready: 64 | print("Server ready and listening on port 1337") 65 | case .failed(let error): 66 | print("Listener failed with error: \(error)") 67 | default: 68 | break 69 | } 70 | } 71 | 72 | listener?.newConnectionHandler = { [weak self] connection in 73 | print("Connection received") 74 | self?.handleConnection(connection: connection) 75 | 76 | // self?.sendData(connection: connection, message: "Hello from iPhone!") 77 | } 78 | 79 | listener?.start(queue: .main) 80 | } catch { 81 | print("Failed to start listener: \(error)") 82 | } 83 | } 84 | 85 | func disconnect() { 86 | // Cancel the listener if it exists 87 | if let listener = listener { 88 | listener.cancel() 89 | print("Listener cancelled") 90 | } 91 | listener = nil 92 | 93 | // Cancel the active connection if it exists 94 | if let connection = activeConnection { 95 | connection.cancel() 96 | print("Connection cancelled") 97 | } 98 | activeConnection = nil 99 | } 100 | 101 | private func handleConnection(connection: NWConnection) { 102 | self.activeConnection = connection 103 | connection.start(queue: .global()) 104 | } 105 | 106 | func sendData( 107 | record3dHeaderData: Data, 108 | intrinsicMatData: Data, 109 | poseData: Data, 110 | rgbImageData: Data, 111 | compressedDepthData: Data? = nil, 112 | compressedConfData: Data? = nil 113 | ) { 114 | guard let activeConnection = activeConnection else { 115 | print("No active connection. Cannot send data.") 116 | return 117 | } 118 | var messageBody = record3dHeaderData + intrinsicMatData + poseData + rgbImageData 119 | if let depthData = compressedDepthData { 120 | messageBody += depthData 121 | } 122 | if let depthConfData = compressedConfData { 123 | messageBody += depthConfData 124 | } 125 | 126 | self.ptHeader.body_size = UInt32(messageBody.count).bigEndian 127 | let ptHeaderData = Data(bytes: &self.ptHeader, count:MemoryLayout.size) 128 | 129 | let completeMessage = ptHeaderData + messageBody 130 | print("Sending data of size: \(completeMessage.count)") 131 | activeConnection.send(content:completeMessage, completion: .contentProcessed {error in 132 | if let error = error { 133 | print("Failed to send data: \(error)") 134 | } else { 135 | print("Image data sent successfully") 136 | } 137 | }) 138 | } 139 | 140 | func sendData(connection: NWConnection, message: String) { 141 | let data = message.data(using: .utf8)! 142 | connection.send(content: data, completion: .contentProcessed { error in 143 | if let error = error { 144 | print("Failed to send data: \(error)") 145 | } else { 146 | print("Data sent successfully") 147 | } 148 | }) 149 | } 150 | 151 | func compressData(from pixelBuffer: CVPixelBuffer, isDepth: Bool) -> Data? { 152 | // Extract depth data 153 | guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) else { 154 | print("Failed to access depth buffer base address") 155 | return nil 156 | } 157 | 158 | let width = CVPixelBufferGetWidth(pixelBuffer) 159 | let height = CVPixelBufferGetHeight(pixelBuffer) 160 | 161 | // Determine the element size based on the type of data 162 | let elementSize = isDepth ? MemoryLayout.size : MemoryLayout.size 163 | let dataSize = width * height * elementSize 164 | 165 | // Extract the raw data 166 | let data = Data(bytes: baseAddress, count: dataSize) 167 | 168 | // Allocate an output buffer for compressed data 169 | let compressedBuffer = UnsafeMutablePointer.allocate(capacity: dataSize) 170 | defer { compressedBuffer.deallocate() } 171 | 172 | let compressedSize = compression_encode_buffer( 173 | compressedBuffer, 174 | dataSize, 175 | [UInt8](data), 176 | data.count, 177 | nil, 178 | COMPRESSION_LZFSE 179 | ) 180 | 181 | guard compressedSize > 0 else { 182 | print("Failed to compress depth map") 183 | return nil 184 | } 185 | 186 | // Return compressed depth data 187 | return Data(bytes: compressedBuffer, count: compressedSize) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /AnySense/Managers/BluetoothManager.swift: -------------------------------------------------------------------------------- 1 | // 2 | // BluetoothManager.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/6/8. 6 | // 7 | 8 | import SwiftUI 9 | import CoreBluetooth 10 | 11 | class BluetoothManager : NSObject, ObservableObject{ 12 | private var centralManager: CBCentralManager? 13 | private var matchedPeripheral: CBPeripheral! 14 | private var rxCharacteristic: CBCharacteristic! 15 | private var displayLink: CADisplayLink? 16 | private var BTFileHandle: FileHandle? 17 | @Published var ifConnected: Bool = false 18 | @Published var discoveredPeripherals: [UUID: CBPeripheral] = [:] 19 | 20 | override init() { 21 | super.init() 22 | self.centralManager = CBCentralManager(delegate: self, queue: .main) 23 | } 24 | 25 | } 26 | 27 | extension BluetoothManager: CBCentralManagerDelegate{ 28 | func centralManagerDidUpdateState(_ central: CBCentralManager) { 29 | switch central.state { 30 | case .poweredOff: 31 | print("Is Powered Off.") 32 | case .poweredOn: 33 | print("Is Powered On.") 34 | self.scan() 35 | case .unsupported: 36 | print("Is Unsupported.") 37 | case .unauthorized: 38 | print("Is Unauthorized.") 39 | case .unknown: 40 | print("Unknown") 41 | case .resetting: 42 | print("Resetting") 43 | @unknown default: 44 | print("Error") 45 | } 46 | } 47 | func scan() -> Void{ 48 | centralManager?.scanForPeripherals(withServices: nil) 49 | } 50 | func disconnectFromDevice () { 51 | if let peripheral = matchedPeripheral { 52 | centralManager?.cancelPeripheralConnection(peripheral) 53 | matchedPeripheral = nil 54 | ifConnected = false 55 | } 56 | /* 57 | if matchedPeripheral != nil { 58 | centralManager?.cancelPeripheralConnection(matchedPeripheral!) 59 | } 60 | ifConnected = false 61 | */ 62 | } 63 | 64 | func centralManager(_ central: CBCentralManager, didDiscover peripheral: CBPeripheral, advertisementData: [String: Any], rssi RSSI: NSNumber){ 65 | guard peripheral.name != nil else { return } 66 | let peripheralUUID = peripheral.identifier 67 | 68 | if discoveredPeripherals[peripheralUUID] == nil { 69 | discoveredPeripherals[peripheralUUID] = peripheral 70 | } 71 | 72 | } 73 | 74 | func connectToPeripheral(withUUID uuid: UUID, completion: @escaping (Result) -> Void) { 75 | guard let central = centralManager else { 76 | completion(.failure(NSError(domain: "Bluetooth", code: 1, userInfo: [NSLocalizedDescriptionKey: "Central Manager is nil"]))) 77 | return 78 | } 79 | 80 | // Retrieve known peripherals (helps avoid stale references) 81 | let knownPeripherals = central.retrievePeripherals(withIdentifiers: [uuid]) 82 | 83 | guard let peripheral = knownPeripherals.first ?? discoveredPeripherals[uuid] else { 84 | completion(.failure(NSError(domain: "Bluetooth", code: 2, userInfo: [NSLocalizedDescriptionKey: "Peripheral not found"]))) 85 | return 86 | } 87 | 88 | // Disconnect if already connected to another peripheral 89 | if let currentPeripheral = matchedPeripheral, currentPeripheral.identifier != peripheral.identifier { 90 | print("🔄 Disconnecting previous peripheral: \(currentPeripheral.name ?? "Unknown")") 91 | central.cancelPeripheralConnection(currentPeripheral) 92 | } 93 | 94 | matchedPeripheral = peripheral 95 | peripheral.delegate = self 96 | 97 | central.connect(peripheral, options: nil) 98 | } 99 | 100 | func connectToPeripheral(peripheral: CBPeripheral){ 101 | if matchedPeripheral != nil && matchedPeripheral != peripheral { 102 | centralManager?.cancelPeripheralConnection(matchedPeripheral!) 103 | } 104 | matchedPeripheral = peripheral 105 | peripheral.delegate = self 106 | centralManager?.connect(peripheral, options: nil) 107 | /* 108 | centralManager?.connect(peripheral, options: nil) 109 | ifConnected = true 110 | */ 111 | } 112 | 113 | func centralManager(_ central: CBCentralManager, didConnect peripheral: CBPeripheral) { 114 | //matchedPeripheral.discoverServices(nil) 115 | ifConnected = true 116 | matchedPeripheral.discoverServices(nil) 117 | } 118 | 119 | func centralManager(_ central: CBCentralManager, didDisconnectPeripheral peripheral: CBPeripheral, error: Error?) { 120 | ifConnected = false 121 | matchedPeripheral = nil 122 | } 123 | } 124 | 125 | extension BluetoothManager: CBPeripheralDelegate{ 126 | func peripheral(_ peripheral: CBPeripheral, didDiscoverServices error: Error?) { 127 | print("*******************************************************") 128 | 129 | if ((error) != nil) { 130 | print("Error discovering services: \(error!.localizedDescription)") 131 | return 132 | } 133 | guard let services = peripheral.services else { 134 | return 135 | } 136 | //We need to discover the all characteristic 137 | for service in services { 138 | peripheral.discoverCharacteristics(nil, for: service) 139 | } 140 | print("Discovered Services: \(services)") 141 | } 142 | 143 | func peripheral(_ peripheral: CBPeripheral, didDiscoverCharacteristicsFor service: CBService, error: Error?) { 144 | guard let characteristics = service.characteristics else { 145 | return 146 | } 147 | 148 | print("Found \(characteristics.count) characteristics.") 149 | // NOTE: We will simply take the first Rx characteristic and use it for reading 150 | for characteristic in characteristics { 151 | if characteristic.properties.contains(.notify) || characteristic.properties.contains(.indicate) { 152 | print("This characteristic is Rx (Receiving data)") 153 | rxCharacteristic = characteristic 154 | peripheral.setNotifyValue(true, for: rxCharacteristic!) 155 | peripheral.readValue(for: characteristic) 156 | print("RX Characteristic: \(rxCharacteristic.uuid)") 157 | break 158 | } 159 | if characteristic.properties.contains(.write) || characteristic.properties.contains(.writeWithoutResponse) { 160 | print("This characteristic is Tx (Transmitting data)") 161 | // TODO: Code for handling Tx characteristics goes here 162 | } 163 | } 164 | } 165 | 166 | func peripheral(_ peripheral: CBPeripheral, didUpdateValueFor characteristic: CBCharacteristic, error: Error?) { 167 | if let error = error { 168 | print("Error updating characteristic: \(error.localizedDescription)") 169 | return 170 | } 171 | } 172 | 173 | } 174 | 175 | extension BluetoothManager: CBPeripheralManagerDelegate { 176 | 177 | func peripheralManagerDidUpdateState(_ peripheral: CBPeripheralManager) { 178 | switch peripheral.state { 179 | case .poweredOn: 180 | print("Peripheral Is Powered On.") 181 | case .unsupported: 182 | print("Peripheral Is Unsupported.") 183 | case .unauthorized: 184 | print("Peripheral Is Unauthorized.") 185 | case .unknown: 186 | print("Peripheral Unknown") 187 | case .resetting: 188 | print("Peripheral Resetting") 189 | case .poweredOff: 190 | print("Peripheral Is Powered Off.") 191 | @unknown default: 192 | print("Error") 193 | } 194 | } 195 | 196 | func startRecording(targetURL: URL, fps: Double) { 197 | do { 198 | self.BTFileHandle = try FileHandle(forWritingTo: targetURL) 199 | // defer {try? BTDataFileHandle.close()} 200 | try self.BTFileHandle?.seekToEnd() 201 | 202 | } catch { 203 | print("Error opening BTFileHandle") 204 | } 205 | 206 | displayLink = CADisplayLink(target: self, selector: #selector(recordSingleData)) 207 | displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: Float(fps), maximum: Float(fps), preferred: Float(fps)) 208 | displayLink?.add(to: .main, forMode: .common) 209 | } 210 | 211 | func stopRecording() { 212 | displayLink?.invalidate() 213 | displayLink = nil 214 | do { 215 | try BTFileHandle?.close() 216 | } catch { 217 | print("Error closing pose file") 218 | } 219 | } 220 | 221 | @objc private func recordSingleData(link: CADisplayLink){ 222 | if(ifConnected == true){ 223 | guard let characteristic = rxCharacteristic else { return 224 | } 225 | characteristicPeripheralUpdate(characteristic: characteristic) 226 | } 227 | } 228 | 229 | private func characteristicPeripheralUpdate(characteristic: CBCharacteristic){ 230 | let currentTimer = Date() 231 | var dataReadTimeStamp = Int64(currentTimer.timeIntervalSince1970 * 1000) 232 | let timeStampData = Data(bytes: &dataReadTimeStamp, count: MemoryLayout.size) 233 | 234 | let crlfData = Data([0x0D, 0x0A]) 235 | 236 | guard let characteristicValue = characteristic.value else {return} 237 | let writeData = timeStampData + characteristicValue + crlfData 238 | self.BTFileHandle!.write(writeData) 239 | } 240 | } 241 | 242 | -------------------------------------------------------------------------------- /Pods/Pods.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 56; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | 33E56350CAF6F80EE987985A9E16E05B /* Foundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 384DDA2CB25005BD6479B5987C619DD4 /* Foundation.framework */; }; 11 | 62FEBBC60AA0D71A405A3013FF153FD4 /* Pods-AnySense-dummy.m in Sources */ = {isa = PBXBuildFile; fileRef = 333BDE6E2297C5ACC8E31B51574F6FDC /* Pods-AnySense-dummy.m */; }; 12 | D381B9E02EE617FC3FC7A1F569773C31 /* Pods-AnySense-umbrella.h in Headers */ = {isa = PBXBuildFile; fileRef = 0EA841B3A0232FE50A1207F939029A78 /* Pods-AnySense-umbrella.h */; settings = {ATTRIBUTES = (Public, ); }; }; 13 | /* End PBXBuildFile section */ 14 | 15 | /* Begin PBXFileReference section */ 16 | 064BDC5C6927F775715E2C5298C2C526 /* Pods-AnySense-acknowledgements.markdown */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text; path = "Pods-AnySense-acknowledgements.markdown"; sourceTree = ""; }; 17 | 0EA841B3A0232FE50A1207F939029A78 /* Pods-AnySense-umbrella.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "Pods-AnySense-umbrella.h"; sourceTree = ""; }; 18 | 3135BE0BB6AE2E1BA299E89862FABFE0 /* Pods-AnySense.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; path = "Pods-AnySense.release.xcconfig"; sourceTree = ""; }; 19 | 333BDE6E2297C5ACC8E31B51574F6FDC /* Pods-AnySense-dummy.m */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.objc; path = "Pods-AnySense-dummy.m"; sourceTree = ""; }; 20 | 384DDA2CB25005BD6479B5987C619DD4 /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS18.0.sdk/System/Library/Frameworks/Foundation.framework; sourceTree = DEVELOPER_DIR; }; 21 | 4DFA3AB0C7B8B351DDA4D567C167FD42 /* Pods-AnySense-Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = "Pods-AnySense-Info.plist"; sourceTree = ""; }; 22 | 4EC2A966549D2D2B0496915E172191C6 /* Pods-AnySense-acknowledgements.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = "Pods-AnySense-acknowledgements.plist"; sourceTree = ""; }; 23 | 7ACCCC1C8B82DC6ECFCD195B8AE81848 /* Pods-AnySense */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; name = "Pods-AnySense"; path = Pods_AnySense.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 24 | 9D940727FF8FB9C785EB98E56350EF41 /* Podfile */ = {isa = PBXFileReference; explicitFileType = text.script.ruby; includeInIndex = 1; indentWidth = 2; lastKnownFileType = text; name = Podfile; path = ../Podfile; sourceTree = SOURCE_ROOT; tabWidth = 2; xcLanguageSpecificationIdentifier = xcode.lang.ruby; }; 25 | B604FBFF515AF55ED140F9537EF28586 /* Pods-AnySense.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; path = "Pods-AnySense.debug.xcconfig"; sourceTree = ""; }; 26 | E6CC51D43990F60B08E25E6206D7A6ED /* Pods-AnySense.modulemap */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.module; path = "Pods-AnySense.modulemap"; sourceTree = ""; }; 27 | /* End PBXFileReference section */ 28 | 29 | /* Begin PBXFrameworksBuildPhase section */ 30 | D5B3D28C921CBBACE831D83AA07F9827 /* Frameworks */ = { 31 | isa = PBXFrameworksBuildPhase; 32 | buildActionMask = 2147483647; 33 | files = ( 34 | 33E56350CAF6F80EE987985A9E16E05B /* Foundation.framework in Frameworks */, 35 | ); 36 | runOnlyForDeploymentPostprocessing = 0; 37 | }; 38 | /* End PBXFrameworksBuildPhase section */ 39 | 40 | /* Begin PBXGroup section */ 41 | 05EE42109B310AF6C0B1D04B88C8AD78 /* Pods-AnySense */ = { 42 | isa = PBXGroup; 43 | children = ( 44 | E6CC51D43990F60B08E25E6206D7A6ED /* Pods-AnySense.modulemap */, 45 | 064BDC5C6927F775715E2C5298C2C526 /* Pods-AnySense-acknowledgements.markdown */, 46 | 4EC2A966549D2D2B0496915E172191C6 /* Pods-AnySense-acknowledgements.plist */, 47 | 333BDE6E2297C5ACC8E31B51574F6FDC /* Pods-AnySense-dummy.m */, 48 | 4DFA3AB0C7B8B351DDA4D567C167FD42 /* Pods-AnySense-Info.plist */, 49 | 0EA841B3A0232FE50A1207F939029A78 /* Pods-AnySense-umbrella.h */, 50 | B604FBFF515AF55ED140F9537EF28586 /* Pods-AnySense.debug.xcconfig */, 51 | 3135BE0BB6AE2E1BA299E89862FABFE0 /* Pods-AnySense.release.xcconfig */, 52 | ); 53 | name = "Pods-AnySense"; 54 | path = "Target Support Files/Pods-AnySense"; 55 | sourceTree = ""; 56 | }; 57 | A8C2A32CD7D8E80980F6A54453DFBE45 /* Products */ = { 58 | isa = PBXGroup; 59 | children = ( 60 | 7ACCCC1C8B82DC6ECFCD195B8AE81848 /* Pods-AnySense */, 61 | ); 62 | name = Products; 63 | sourceTree = ""; 64 | }; 65 | CF1408CF629C7361332E53B88F7BD30C = { 66 | isa = PBXGroup; 67 | children = ( 68 | 9D940727FF8FB9C785EB98E56350EF41 /* Podfile */, 69 | D210D550F4EA176C3123ED886F8F87F5 /* Frameworks */, 70 | A8C2A32CD7D8E80980F6A54453DFBE45 /* Products */, 71 | E990ED27F0AB04DEDAF553E8E342BB79 /* Targets Support Files */, 72 | ); 73 | sourceTree = ""; 74 | }; 75 | D210D550F4EA176C3123ED886F8F87F5 /* Frameworks */ = { 76 | isa = PBXGroup; 77 | children = ( 78 | E4801F62A6B08CD9B5410329F1A18FDE /* iOS */, 79 | ); 80 | name = Frameworks; 81 | sourceTree = ""; 82 | }; 83 | E4801F62A6B08CD9B5410329F1A18FDE /* iOS */ = { 84 | isa = PBXGroup; 85 | children = ( 86 | 384DDA2CB25005BD6479B5987C619DD4 /* Foundation.framework */, 87 | ); 88 | name = iOS; 89 | sourceTree = ""; 90 | }; 91 | E990ED27F0AB04DEDAF553E8E342BB79 /* Targets Support Files */ = { 92 | isa = PBXGroup; 93 | children = ( 94 | 05EE42109B310AF6C0B1D04B88C8AD78 /* Pods-AnySense */, 95 | ); 96 | name = "Targets Support Files"; 97 | sourceTree = ""; 98 | }; 99 | /* End PBXGroup section */ 100 | 101 | /* Begin PBXHeadersBuildPhase section */ 102 | AD88BAEFCE81AC04BA66C82EF2C9D479 /* Headers */ = { 103 | isa = PBXHeadersBuildPhase; 104 | buildActionMask = 2147483647; 105 | files = ( 106 | D381B9E02EE617FC3FC7A1F569773C31 /* Pods-AnySense-umbrella.h in Headers */, 107 | ); 108 | runOnlyForDeploymentPostprocessing = 0; 109 | }; 110 | /* End PBXHeadersBuildPhase section */ 111 | 112 | /* Begin PBXNativeTarget section */ 113 | DA81DFEAE09A3338DD7A8DA44D3C0375 /* Pods-AnySense */ = { 114 | isa = PBXNativeTarget; 115 | buildConfigurationList = 48A90E48A26785554C63D73DE4113350 /* Build configuration list for PBXNativeTarget "Pods-AnySense" */; 116 | buildPhases = ( 117 | AD88BAEFCE81AC04BA66C82EF2C9D479 /* Headers */, 118 | 124264BC1CD6F6E50125FE42C9B91C88 /* Sources */, 119 | D5B3D28C921CBBACE831D83AA07F9827 /* Frameworks */, 120 | 06014D7D0EE41087AD1B8BC969A9D330 /* Resources */, 121 | ); 122 | buildRules = ( 123 | ); 124 | dependencies = ( 125 | ); 126 | name = "Pods-AnySense"; 127 | productName = Pods_AnySense; 128 | productReference = 7ACCCC1C8B82DC6ECFCD195B8AE81848 /* Pods-AnySense */; 129 | productType = "com.apple.product-type.framework"; 130 | }; 131 | /* End PBXNativeTarget section */ 132 | 133 | /* Begin PBXProject section */ 134 | BFDFE7DC352907FC980B868725387E98 /* Project object */ = { 135 | isa = PBXProject; 136 | attributes = { 137 | LastSwiftUpdateCheck = 1600; 138 | LastUpgradeCheck = 1600; 139 | }; 140 | buildConfigurationList = 4821239608C13582E20E6DA73FD5F1F9 /* Build configuration list for PBXProject "Pods" */; 141 | compatibilityVersion = "Xcode 14.0"; 142 | developmentRegion = en; 143 | hasScannedForEncodings = 0; 144 | knownRegions = ( 145 | Base, 146 | en, 147 | ); 148 | mainGroup = CF1408CF629C7361332E53B88F7BD30C; 149 | minimizedProjectReferenceProxies = 0; 150 | preferredProjectObjectVersion = 77; 151 | productRefGroup = A8C2A32CD7D8E80980F6A54453DFBE45 /* Products */; 152 | projectDirPath = ""; 153 | projectRoot = ""; 154 | targets = ( 155 | DA81DFEAE09A3338DD7A8DA44D3C0375 /* Pods-AnySense */, 156 | ); 157 | }; 158 | /* End PBXProject section */ 159 | 160 | /* Begin PBXResourcesBuildPhase section */ 161 | 06014D7D0EE41087AD1B8BC969A9D330 /* Resources */ = { 162 | isa = PBXResourcesBuildPhase; 163 | buildActionMask = 2147483647; 164 | files = ( 165 | ); 166 | runOnlyForDeploymentPostprocessing = 0; 167 | }; 168 | /* End PBXResourcesBuildPhase section */ 169 | 170 | /* Begin PBXSourcesBuildPhase section */ 171 | 124264BC1CD6F6E50125FE42C9B91C88 /* Sources */ = { 172 | isa = PBXSourcesBuildPhase; 173 | buildActionMask = 2147483647; 174 | files = ( 175 | 62FEBBC60AA0D71A405A3013FF153FD4 /* Pods-AnySense-dummy.m in Sources */, 176 | ); 177 | runOnlyForDeploymentPostprocessing = 0; 178 | }; 179 | /* End PBXSourcesBuildPhase section */ 180 | 181 | /* Begin XCBuildConfiguration section */ 182 | 6A5C026ED62BBFE3380CD257EEFAEB20 /* Debug */ = { 183 | isa = XCBuildConfiguration; 184 | buildSettings = { 185 | ALWAYS_SEARCH_USER_PATHS = NO; 186 | CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES; 187 | CLANG_ANALYZER_NONNULL = YES; 188 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 189 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 190 | CLANG_CXX_LIBRARY = "libc++"; 191 | CLANG_ENABLE_MODULES = YES; 192 | CLANG_ENABLE_OBJC_ARC = YES; 193 | CLANG_ENABLE_OBJC_WEAK = YES; 194 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 195 | CLANG_WARN_BOOL_CONVERSION = YES; 196 | CLANG_WARN_COMMA = YES; 197 | CLANG_WARN_CONSTANT_CONVERSION = YES; 198 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 199 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 200 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 201 | CLANG_WARN_EMPTY_BODY = YES; 202 | CLANG_WARN_ENUM_CONVERSION = YES; 203 | CLANG_WARN_INFINITE_RECURSION = YES; 204 | CLANG_WARN_INT_CONVERSION = YES; 205 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 206 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 207 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 208 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 209 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 210 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 211 | CLANG_WARN_STRICT_PROTOTYPES = YES; 212 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 213 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 214 | CLANG_WARN_UNREACHABLE_CODE = YES; 215 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 216 | COPY_PHASE_STRIP = NO; 217 | DEBUG_INFORMATION_FORMAT = dwarf; 218 | ENABLE_STRICT_OBJC_MSGSEND = YES; 219 | ENABLE_TESTABILITY = YES; 220 | GCC_C_LANGUAGE_STANDARD = gnu11; 221 | GCC_DYNAMIC_NO_PIC = NO; 222 | GCC_NO_COMMON_BLOCKS = YES; 223 | GCC_OPTIMIZATION_LEVEL = 0; 224 | GCC_PREPROCESSOR_DEFINITIONS = ( 225 | "POD_CONFIGURATION_DEBUG=1", 226 | "DEBUG=1", 227 | "$(inherited)", 228 | ); 229 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 230 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 231 | GCC_WARN_UNDECLARED_SELECTOR = YES; 232 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 233 | GCC_WARN_UNUSED_FUNCTION = YES; 234 | GCC_WARN_UNUSED_VARIABLE = YES; 235 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 236 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 237 | MTL_FAST_MATH = YES; 238 | ONLY_ACTIVE_ARCH = YES; 239 | PRODUCT_NAME = "$(TARGET_NAME)"; 240 | STRIP_INSTALLED_PRODUCT = NO; 241 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; 242 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 243 | SWIFT_VERSION = 5.0; 244 | SYMROOT = "${SRCROOT}/../build"; 245 | }; 246 | name = Debug; 247 | }; 248 | 86B9D7A6BCB5F187E489FF28FCF2E840 /* Release */ = { 249 | isa = XCBuildConfiguration; 250 | baseConfigurationReference = 3135BE0BB6AE2E1BA299E89862FABFE0 /* Pods-AnySense.release.xcconfig */; 251 | buildSettings = { 252 | ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; 253 | CLANG_ENABLE_OBJC_WEAK = NO; 254 | "CODE_SIGN_IDENTITY[sdk=appletvos*]" = ""; 255 | "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = ""; 256 | "CODE_SIGN_IDENTITY[sdk=watchos*]" = ""; 257 | CURRENT_PROJECT_VERSION = 1; 258 | DEFINES_MODULE = YES; 259 | DYLIB_COMPATIBILITY_VERSION = 1; 260 | DYLIB_CURRENT_VERSION = 1; 261 | DYLIB_INSTALL_NAME_BASE = "@rpath"; 262 | ENABLE_MODULE_VERIFIER = NO; 263 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 264 | INFOPLIST_FILE = "Target Support Files/Pods-AnySense/Pods-AnySense-Info.plist"; 265 | INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; 266 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 267 | LD_RUNPATH_SEARCH_PATHS = ( 268 | "$(inherited)", 269 | "@executable_path/Frameworks", 270 | "@loader_path/Frameworks", 271 | ); 272 | MACH_O_TYPE = staticlib; 273 | MODULEMAP_FILE = "Target Support Files/Pods-AnySense/Pods-AnySense.modulemap"; 274 | OTHER_LDFLAGS = ""; 275 | OTHER_LIBTOOLFLAGS = ""; 276 | PODS_ROOT = "$(SRCROOT)"; 277 | PRODUCT_BUNDLE_IDENTIFIER = "org.cocoapods.${PRODUCT_NAME:rfc1034identifier}"; 278 | PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; 279 | SDKROOT = iphoneos; 280 | SKIP_INSTALL = YES; 281 | TARGETED_DEVICE_FAMILY = "1,2"; 282 | VALIDATE_PRODUCT = YES; 283 | VERSIONING_SYSTEM = "apple-generic"; 284 | VERSION_INFO_PREFIX = ""; 285 | }; 286 | name = Release; 287 | }; 288 | 9110DF914744DF86C5D0C8B83C0F707D /* Debug */ = { 289 | isa = XCBuildConfiguration; 290 | baseConfigurationReference = B604FBFF515AF55ED140F9537EF28586 /* Pods-AnySense.debug.xcconfig */; 291 | buildSettings = { 292 | ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = NO; 293 | CLANG_ENABLE_OBJC_WEAK = NO; 294 | "CODE_SIGN_IDENTITY[sdk=appletvos*]" = ""; 295 | "CODE_SIGN_IDENTITY[sdk=iphoneos*]" = ""; 296 | "CODE_SIGN_IDENTITY[sdk=watchos*]" = ""; 297 | CURRENT_PROJECT_VERSION = 1; 298 | DEFINES_MODULE = YES; 299 | DYLIB_COMPATIBILITY_VERSION = 1; 300 | DYLIB_CURRENT_VERSION = 1; 301 | DYLIB_INSTALL_NAME_BASE = "@rpath"; 302 | ENABLE_MODULE_VERIFIER = NO; 303 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 304 | INFOPLIST_FILE = "Target Support Files/Pods-AnySense/Pods-AnySense-Info.plist"; 305 | INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; 306 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 307 | LD_RUNPATH_SEARCH_PATHS = ( 308 | "$(inherited)", 309 | "@executable_path/Frameworks", 310 | "@loader_path/Frameworks", 311 | ); 312 | MACH_O_TYPE = staticlib; 313 | MODULEMAP_FILE = "Target Support Files/Pods-AnySense/Pods-AnySense.modulemap"; 314 | OTHER_LDFLAGS = ""; 315 | OTHER_LIBTOOLFLAGS = ""; 316 | PODS_ROOT = "$(SRCROOT)"; 317 | PRODUCT_BUNDLE_IDENTIFIER = "org.cocoapods.${PRODUCT_NAME:rfc1034identifier}"; 318 | PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; 319 | SDKROOT = iphoneos; 320 | SKIP_INSTALL = YES; 321 | TARGETED_DEVICE_FAMILY = "1,2"; 322 | VERSIONING_SYSTEM = "apple-generic"; 323 | VERSION_INFO_PREFIX = ""; 324 | }; 325 | name = Debug; 326 | }; 327 | E873CA97BD124B21839AF13C9BE1CD18 /* Release */ = { 328 | isa = XCBuildConfiguration; 329 | buildSettings = { 330 | ALWAYS_SEARCH_USER_PATHS = NO; 331 | CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES; 332 | CLANG_ANALYZER_NONNULL = YES; 333 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 334 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 335 | CLANG_CXX_LIBRARY = "libc++"; 336 | CLANG_ENABLE_MODULES = YES; 337 | CLANG_ENABLE_OBJC_ARC = YES; 338 | CLANG_ENABLE_OBJC_WEAK = YES; 339 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 340 | CLANG_WARN_BOOL_CONVERSION = YES; 341 | CLANG_WARN_COMMA = YES; 342 | CLANG_WARN_CONSTANT_CONVERSION = YES; 343 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 344 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 345 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 346 | CLANG_WARN_EMPTY_BODY = YES; 347 | CLANG_WARN_ENUM_CONVERSION = YES; 348 | CLANG_WARN_INFINITE_RECURSION = YES; 349 | CLANG_WARN_INT_CONVERSION = YES; 350 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 351 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 352 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 353 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 354 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 355 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 356 | CLANG_WARN_STRICT_PROTOTYPES = YES; 357 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 358 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 359 | CLANG_WARN_UNREACHABLE_CODE = YES; 360 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 361 | COPY_PHASE_STRIP = NO; 362 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 363 | ENABLE_NS_ASSERTIONS = NO; 364 | ENABLE_STRICT_OBJC_MSGSEND = YES; 365 | GCC_C_LANGUAGE_STANDARD = gnu11; 366 | GCC_NO_COMMON_BLOCKS = YES; 367 | GCC_PREPROCESSOR_DEFINITIONS = ( 368 | "POD_CONFIGURATION_RELEASE=1", 369 | "$(inherited)", 370 | ); 371 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 372 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 373 | GCC_WARN_UNDECLARED_SELECTOR = YES; 374 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 375 | GCC_WARN_UNUSED_FUNCTION = YES; 376 | GCC_WARN_UNUSED_VARIABLE = YES; 377 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 378 | MTL_ENABLE_DEBUG_INFO = NO; 379 | MTL_FAST_MATH = YES; 380 | PRODUCT_NAME = "$(TARGET_NAME)"; 381 | STRIP_INSTALLED_PRODUCT = NO; 382 | SWIFT_COMPILATION_MODE = wholemodule; 383 | SWIFT_OPTIMIZATION_LEVEL = "-O"; 384 | SWIFT_VERSION = 5.0; 385 | SYMROOT = "${SRCROOT}/../build"; 386 | }; 387 | name = Release; 388 | }; 389 | /* End XCBuildConfiguration section */ 390 | 391 | /* Begin XCConfigurationList section */ 392 | 4821239608C13582E20E6DA73FD5F1F9 /* Build configuration list for PBXProject "Pods" */ = { 393 | isa = XCConfigurationList; 394 | buildConfigurations = ( 395 | 6A5C026ED62BBFE3380CD257EEFAEB20 /* Debug */, 396 | E873CA97BD124B21839AF13C9BE1CD18 /* Release */, 397 | ); 398 | defaultConfigurationIsVisible = 0; 399 | defaultConfigurationName = Release; 400 | }; 401 | 48A90E48A26785554C63D73DE4113350 /* Build configuration list for PBXNativeTarget "Pods-AnySense" */ = { 402 | isa = XCConfigurationList; 403 | buildConfigurations = ( 404 | 9110DF914744DF86C5D0C8B83C0F707D /* Debug */, 405 | 86B9D7A6BCB5F187E489FF28FCF2E840 /* Release */, 406 | ); 407 | defaultConfigurationIsVisible = 0; 408 | defaultConfigurationName = Release; 409 | }; 410 | /* End XCConfigurationList section */ 411 | }; 412 | rootObject = BFDFE7DC352907FC980B868725387E98 /* Project object */; 413 | } 414 | -------------------------------------------------------------------------------- /AnySense/Views/readView.swift: -------------------------------------------------------------------------------- 1 | // 2 | // readView.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/5/27. 6 | // 7 | 8 | import SwiftUI 9 | import UIKit 10 | import CoreBluetooth 11 | import BackgroundTasks 12 | import UserNotifications 13 | import Foundation 14 | import AVFoundation 15 | 16 | enum ActiveAlert { 17 | case first, second 18 | } 19 | 20 | struct ReadView : View{ 21 | @EnvironmentObject var appStatus : AppInformation 22 | @EnvironmentObject var bluetoothManager: BluetoothManager 23 | @ObservedObject var arViewModel: ARViewModel 24 | @State private var isReading = false 25 | @State var showingAlert : Bool = false 26 | @Environment(\.scenePhase) private var phase 27 | @State private var fileSetNames: RecordingFiles? 28 | @State var showingExporter = false 29 | @State var openFlash = true 30 | @State private var activeAlert: ActiveAlert = .first 31 | @State private var isRecordedOnce: Bool = false 32 | var body : some View{ 33 | let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask) 34 | GeometryReader { geometry in 35 | let screenWidth = geometry.size.width 36 | let screenHeight = geometry.size.height 37 | let arViewHeight = min(screenWidth * 1.33, 0.75 * screenHeight) 38 | let arViewWidth = min(arViewHeight / 1.33, screenWidth) 39 | let arViewPadding = 0.2 * arViewHeight 40 | let buttonSize: CGFloat = min(screenWidth * 0.3, 100) 41 | // let buttonPadding: CGFloat = 42 | let btBarHeight: CGFloat = 25.0 43 | let gridSize = appStatus.gridProjectionTrigger.rawValue 44 | 45 | ZStack { 46 | // Apply the custom background color 47 | Color.customizedBackground 48 | .ignoresSafeArea() 49 | ZStack{ 50 | ARViewContainer(session: arViewModel.session) 51 | .edgesIgnoringSafeArea(.all) 52 | .frame(width: arViewWidth, height: arViewHeight) 53 | // .clipShape(RoundedRectangle(cornerRadius: 30, style: .continuous)) 54 | .padding(.bottom, arViewPadding) 55 | .opacity(appStatus.rgbdVideoStreaming == .off ? 1 : 0) 56 | .allowsHitTesting(appStatus.rgbdVideoStreaming == .off) // Disable interaction in streaming mode 57 | if appStatus.rgbdVideoStreaming == .off { 58 | Text(appStatus.ifBluetoothConnected ? "bluetooth device connected" : "bluetooth device disconnected") 59 | .font(.footnote) 60 | .foregroundColor(Color.white) 61 | .frame(width: screenWidth, height: btBarHeight) 62 | .background(appStatus.ifBluetoothConnected ? .green : .red) 63 | .padding(.bottom, arViewPadding + arViewHeight + btBarHeight) 64 | .ignoresSafeArea() 65 | if appStatus.gridProjectionTrigger.rawValue > 0 { 66 | VStack { 67 | Path { path in 68 | for col in 1.. [FileElement] { 382 | guard let fileSetNames = fileSetNames else { 383 | print("❌ Error: Insufficient paths or fileSetNames elements") 384 | return [] 385 | } 386 | 387 | let rgbFile = FileElement.videoFile(VideoFile(url:fileSetNames.rgbFileName)) 388 | let depthFile = FileElement.videoFile(VideoFile(url:fileSetNames.depthFileName)) 389 | let poseFile = FileElement.textFile(TextFile(url:fileSetNames.poseFile.path)) 390 | // let rgbImageFolder = FileElement.directory(SubLevelDirectory(url:fileSetNames.rgbImagesDirectory)) 391 | let depthImageFolder = FileElement.directory(SubLevelDirectory(url: fileSetNames.depthImagesDirectory)) 392 | 393 | return [rgbFile, depthFile, poseFile, depthImageFolder] 394 | } 395 | 396 | func deleteRecordedData(url: [URL], targetDirect: String){ 397 | do { 398 | let urlToDelete = url[0].appendingPathComponent(targetDirect) 399 | try FileManager.default.removeItem(at: urlToDelete) 400 | print("Successfully deleted file!") 401 | } catch { 402 | print("Error deleting file: \(error)") 403 | } 404 | } 405 | 406 | 407 | 408 | } 409 | 410 | 411 | 412 | 413 | #Preview { 414 | ReadView(arViewModel: ARViewModel()) 415 | .environmentObject(AppInformation()) 416 | } 417 | 418 | -------------------------------------------------------------------------------- /AnySense.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 56; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | B9104F622BFDE63C000D4DDD /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = B9104F612BFDE63C000D4DDD /* Assets.xcassets */; }; 11 | B9944A342CFE983300232FBB /* MainPage.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9944A332CFE982E00232FBB /* MainPage.swift */; }; 12 | B997CCD32D4E081300F62B49 /* dataStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = B997CCD22D4E081300F62B49 /* dataStorage.swift */; }; 13 | B997CCD52D4E082900F62B49 /* AnySenseApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = B997CCD42D4E082900F62B49 /* AnySenseApp.swift */; }; 14 | B9E4B22D2D62A9BA0032E877 /* BluetoothManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */; }; 15 | B9E4B22E2D62A9BA0032E877 /* WebRTCManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */; }; 16 | B9E4B22F2D62A9BA0032E877 /* ARViewContainer.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */; }; 17 | B9E4B2302D62A9BA0032E877 /* USBManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B22A2D62A9BA0032E877 /* USBManager.swift */; }; 18 | B9E4B2362D62A9C40032E877 /* peripheralView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2332D62A9C40032E877 /* peripheralView.swift */; }; 19 | B9E4B2372D62A9C40032E877 /* accountView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2312D62A9C40032E877 /* accountView.swift */; }; 20 | B9E4B2382D62A9C40032E877 /* readView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2342D62A9C40032E877 /* readView.swift */; }; 21 | B9E4B2392D62A9C40032E877 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9E4B2322D62A9C40032E877 /* ContentView.swift */; }; 22 | B9E4D2B32D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = B9E4D2B22D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard */; }; 23 | B9E4D2B42D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = B9E4D2B22D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard */; }; 24 | B9E4D2B52D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = B9E4D2B22D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard */; }; 25 | /* End PBXBuildFile section */ 26 | 27 | /* Begin PBXContainerItemProxy section */ 28 | B9104F6B2BFDE63C000D4DDD /* PBXContainerItemProxy */ = { 29 | isa = PBXContainerItemProxy; 30 | containerPortal = B9104F522BFDE63A000D4DDD /* Project object */; 31 | proxyType = 1; 32 | remoteGlobalIDString = B9104F592BFDE63A000D4DDD; 33 | remoteInfo = USBInterfaceApp; 34 | }; 35 | B9104F752BFDE63C000D4DDD /* PBXContainerItemProxy */ = { 36 | isa = PBXContainerItemProxy; 37 | containerPortal = B9104F522BFDE63A000D4DDD /* Project object */; 38 | proxyType = 1; 39 | remoteGlobalIDString = B9104F592BFDE63A000D4DDD; 40 | remoteInfo = USBInterfaceApp; 41 | }; 42 | /* End PBXContainerItemProxy section */ 43 | 44 | /* Begin PBXFileReference section */ 45 | B9104F5A2BFDE63A000D4DDD /* AnySense.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = AnySense.app; sourceTree = BUILT_PRODUCTS_DIR; }; 46 | B9104F612BFDE63C000D4DDD /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 47 | B9104F6A2BFDE63C000D4DDD /* AnySenseTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = AnySenseTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 48 | B9104F742BFDE63C000D4DDD /* AnySenseUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = AnySenseUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 49 | B98D69632C04911F00648D95 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = ""; }; 50 | B9944A332CFE982E00232FBB /* MainPage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainPage.swift; sourceTree = ""; }; 51 | B997CCD22D4E081300F62B49 /* dataStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = dataStorage.swift; sourceTree = ""; }; 52 | B997CCD42D4E082900F62B49 /* AnySenseApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AnySenseApp.swift; sourceTree = ""; }; 53 | B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ARViewContainer.swift; sourceTree = ""; }; 54 | B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BluetoothManager.swift; sourceTree = ""; }; 55 | B9E4B22A2D62A9BA0032E877 /* USBManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = USBManager.swift; sourceTree = ""; }; 56 | B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebRTCManager.swift; sourceTree = ""; }; 57 | B9E4B2312D62A9C40032E877 /* accountView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = accountView.swift; sourceTree = ""; }; 58 | B9E4B2322D62A9C40032E877 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; 59 | B9E4B2332D62A9C40032E877 /* peripheralView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = peripheralView.swift; sourceTree = ""; }; 60 | B9E4B2342D62A9C40032E877 /* readView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = readView.swift; sourceTree = ""; }; 61 | B9E4D2B22D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = AnySenseLaunchScreen.storyboard; sourceTree = ""; }; 62 | BA11D17F77459219187F7A30 /* Pods-AnySense.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AnySense.release.xcconfig"; path = "Target Support Files/Pods-AnySense/Pods-AnySense.release.xcconfig"; sourceTree = ""; }; 63 | E98E38A27D08BC9D5BE3E2E6 /* Pods-AnySense.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-AnySense.debug.xcconfig"; path = "Target Support Files/Pods-AnySense/Pods-AnySense.debug.xcconfig"; sourceTree = ""; }; 64 | /* End PBXFileReference section */ 65 | 66 | /* Begin PBXFrameworksBuildPhase section */ 67 | B9104F572BFDE63A000D4DDD /* Frameworks */ = { 68 | isa = PBXFrameworksBuildPhase; 69 | buildActionMask = 2147483647; 70 | files = ( 71 | ); 72 | runOnlyForDeploymentPostprocessing = 0; 73 | }; 74 | B9104F672BFDE63C000D4DDD /* Frameworks */ = { 75 | isa = PBXFrameworksBuildPhase; 76 | buildActionMask = 2147483647; 77 | files = ( 78 | ); 79 | runOnlyForDeploymentPostprocessing = 0; 80 | }; 81 | B9104F712BFDE63C000D4DDD /* Frameworks */ = { 82 | isa = PBXFrameworksBuildPhase; 83 | buildActionMask = 2147483647; 84 | files = ( 85 | ); 86 | runOnlyForDeploymentPostprocessing = 0; 87 | }; 88 | /* End PBXFrameworksBuildPhase section */ 89 | 90 | /* Begin PBXGroup section */ 91 | B9104F512BFDE63A000D4DDD = { 92 | isa = PBXGroup; 93 | children = ( 94 | B9104F5C2BFDE63A000D4DDD /* AnySense */, 95 | B9104F5B2BFDE63A000D4DDD /* Products */, 96 | C3E474F763EDD5292D1EE42D /* Pods */, 97 | ); 98 | sourceTree = ""; 99 | }; 100 | B9104F5B2BFDE63A000D4DDD /* Products */ = { 101 | isa = PBXGroup; 102 | children = ( 103 | B9104F5A2BFDE63A000D4DDD /* AnySense.app */, 104 | B9104F6A2BFDE63C000D4DDD /* AnySenseTests.xctest */, 105 | B9104F742BFDE63C000D4DDD /* AnySenseUITests.xctest */, 106 | ); 107 | name = Products; 108 | sourceTree = ""; 109 | }; 110 | B9104F5C2BFDE63A000D4DDD /* AnySense */ = { 111 | isa = PBXGroup; 112 | children = ( 113 | B9944A332CFE982E00232FBB /* MainPage.swift */, 114 | B997CCD22D4E081300F62B49 /* dataStorage.swift */, 115 | B9104F612BFDE63C000D4DDD /* Assets.xcassets */, 116 | B997CCD42D4E082900F62B49 /* AnySenseApp.swift */, 117 | B98D69632C04911F00648D95 /* Info.plist */, 118 | B9E4B22C2D62A9BA0032E877 /* Managers */, 119 | B9E4B2352D62A9C40032E877 /* Views */, 120 | B9E4D2B22D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard */, 121 | ); 122 | path = AnySense; 123 | sourceTree = ""; 124 | }; 125 | B9E4B22C2D62A9BA0032E877 /* Managers */ = { 126 | isa = PBXGroup; 127 | children = ( 128 | B9E4B2282D62A9BA0032E877 /* ARViewContainer.swift */, 129 | B9E4B2292D62A9BA0032E877 /* BluetoothManager.swift */, 130 | B9E4B22A2D62A9BA0032E877 /* USBManager.swift */, 131 | B9E4B22B2D62A9BA0032E877 /* WebRTCManager.swift */, 132 | ); 133 | path = Managers; 134 | sourceTree = ""; 135 | }; 136 | B9E4B2352D62A9C40032E877 /* Views */ = { 137 | isa = PBXGroup; 138 | children = ( 139 | B9E4B2312D62A9C40032E877 /* accountView.swift */, 140 | B9E4B2322D62A9C40032E877 /* ContentView.swift */, 141 | B9E4B2332D62A9C40032E877 /* peripheralView.swift */, 142 | B9E4B2342D62A9C40032E877 /* readView.swift */, 143 | ); 144 | path = Views; 145 | sourceTree = ""; 146 | }; 147 | C3E474F763EDD5292D1EE42D /* Pods */ = { 148 | isa = PBXGroup; 149 | children = ( 150 | E98E38A27D08BC9D5BE3E2E6 /* Pods-AnySense.debug.xcconfig */, 151 | BA11D17F77459219187F7A30 /* Pods-AnySense.release.xcconfig */, 152 | ); 153 | path = Pods; 154 | sourceTree = ""; 155 | }; 156 | /* End PBXGroup section */ 157 | 158 | /* Begin PBXNativeTarget section */ 159 | B9104F592BFDE63A000D4DDD /* AnySense */ = { 160 | isa = PBXNativeTarget; 161 | buildConfigurationList = B9104F7E2BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySense" */; 162 | buildPhases = ( 163 | A42CDBF7C43F93BDD9CC0D8E /* [CP] Check Pods Manifest.lock */, 164 | B9104F562BFDE63A000D4DDD /* Sources */, 165 | B9104F572BFDE63A000D4DDD /* Frameworks */, 166 | B9104F582BFDE63A000D4DDD /* Resources */, 167 | ); 168 | buildRules = ( 169 | ); 170 | dependencies = ( 171 | ); 172 | name = AnySense; 173 | productName = USBInterfaceApp; 174 | productReference = B9104F5A2BFDE63A000D4DDD /* AnySense.app */; 175 | productType = "com.apple.product-type.application"; 176 | }; 177 | B9104F692BFDE63C000D4DDD /* AnySenseTests */ = { 178 | isa = PBXNativeTarget; 179 | buildConfigurationList = B9104F812BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySenseTests" */; 180 | buildPhases = ( 181 | B9104F662BFDE63C000D4DDD /* Sources */, 182 | B9104F672BFDE63C000D4DDD /* Frameworks */, 183 | B9104F682BFDE63C000D4DDD /* Resources */, 184 | ); 185 | buildRules = ( 186 | ); 187 | dependencies = ( 188 | B9104F6C2BFDE63C000D4DDD /* PBXTargetDependency */, 189 | ); 190 | name = AnySenseTests; 191 | productName = USBInterfaceAppTests; 192 | productReference = B9104F6A2BFDE63C000D4DDD /* AnySenseTests.xctest */; 193 | productType = "com.apple.product-type.bundle.unit-test"; 194 | }; 195 | B9104F732BFDE63C000D4DDD /* AnySenseUITests */ = { 196 | isa = PBXNativeTarget; 197 | buildConfigurationList = B9104F842BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySenseUITests" */; 198 | buildPhases = ( 199 | B9104F702BFDE63C000D4DDD /* Sources */, 200 | B9104F712BFDE63C000D4DDD /* Frameworks */, 201 | B9104F722BFDE63C000D4DDD /* Resources */, 202 | ); 203 | buildRules = ( 204 | ); 205 | dependencies = ( 206 | B9104F762BFDE63C000D4DDD /* PBXTargetDependency */, 207 | ); 208 | name = AnySenseUITests; 209 | productName = USBInterfaceAppUITests; 210 | productReference = B9104F742BFDE63C000D4DDD /* AnySenseUITests.xctest */; 211 | productType = "com.apple.product-type.bundle.ui-testing"; 212 | }; 213 | /* End PBXNativeTarget section */ 214 | 215 | /* Begin PBXProject section */ 216 | B9104F522BFDE63A000D4DDD /* Project object */ = { 217 | isa = PBXProject; 218 | attributes = { 219 | BuildIndependentTargetsInParallel = 1; 220 | KnownAssetTags = ( 221 | New, 222 | ); 223 | LastSwiftUpdateCheck = 1530; 224 | LastUpgradeCheck = 1530; 225 | TargetAttributes = { 226 | B9104F592BFDE63A000D4DDD = { 227 | CreatedOnToolsVersion = 15.3; 228 | }; 229 | B9104F692BFDE63C000D4DDD = { 230 | CreatedOnToolsVersion = 15.3; 231 | TestTargetID = B9104F592BFDE63A000D4DDD; 232 | }; 233 | B9104F732BFDE63C000D4DDD = { 234 | CreatedOnToolsVersion = 15.3; 235 | TestTargetID = B9104F592BFDE63A000D4DDD; 236 | }; 237 | }; 238 | }; 239 | buildConfigurationList = B9104F552BFDE63A000D4DDD /* Build configuration list for PBXProject "AnySense" */; 240 | compatibilityVersion = "Xcode 14.0"; 241 | developmentRegion = en; 242 | hasScannedForEncodings = 0; 243 | knownRegions = ( 244 | en, 245 | Base, 246 | ); 247 | mainGroup = B9104F512BFDE63A000D4DDD; 248 | packageReferences = ( 249 | B98D69592C0486D500648D95 /* XCRemoteSwiftPackageReference "PythonKit" */, 250 | B98D69602C048AF100648D95 /* XCRemoteSwiftPackageReference "numpy" */, 251 | B905B27B2C88D706006F994E /* XCRemoteSwiftPackageReference "WebRTC-iOS-SDK" */, 252 | ); 253 | productRefGroup = B9104F5B2BFDE63A000D4DDD /* Products */; 254 | projectDirPath = ""; 255 | projectRoot = ""; 256 | targets = ( 257 | B9104F592BFDE63A000D4DDD /* AnySense */, 258 | B9104F692BFDE63C000D4DDD /* AnySenseTests */, 259 | B9104F732BFDE63C000D4DDD /* AnySenseUITests */, 260 | ); 261 | }; 262 | /* End PBXProject section */ 263 | 264 | /* Begin PBXResourcesBuildPhase section */ 265 | B9104F582BFDE63A000D4DDD /* Resources */ = { 266 | isa = PBXResourcesBuildPhase; 267 | buildActionMask = 2147483647; 268 | files = ( 269 | B9104F622BFDE63C000D4DDD /* Assets.xcassets in Resources */, 270 | B9E4D2B42D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */, 271 | ); 272 | runOnlyForDeploymentPostprocessing = 0; 273 | }; 274 | B9104F682BFDE63C000D4DDD /* Resources */ = { 275 | isa = PBXResourcesBuildPhase; 276 | buildActionMask = 2147483647; 277 | files = ( 278 | B9E4D2B52D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */, 279 | ); 280 | runOnlyForDeploymentPostprocessing = 0; 281 | }; 282 | B9104F722BFDE63C000D4DDD /* Resources */ = { 283 | isa = PBXResourcesBuildPhase; 284 | buildActionMask = 2147483647; 285 | files = ( 286 | B9E4D2B32D6965B90044F2D4 /* AnySenseLaunchScreen.storyboard in Resources */, 287 | ); 288 | runOnlyForDeploymentPostprocessing = 0; 289 | }; 290 | /* End PBXResourcesBuildPhase section */ 291 | 292 | /* Begin PBXShellScriptBuildPhase section */ 293 | A42CDBF7C43F93BDD9CC0D8E /* [CP] Check Pods Manifest.lock */ = { 294 | isa = PBXShellScriptBuildPhase; 295 | buildActionMask = 2147483647; 296 | files = ( 297 | ); 298 | inputFileListPaths = ( 299 | ); 300 | inputPaths = ( 301 | "${PODS_PODFILE_DIR_PATH}/Podfile.lock", 302 | "${PODS_ROOT}/Manifest.lock", 303 | ); 304 | name = "[CP] Check Pods Manifest.lock"; 305 | outputFileListPaths = ( 306 | ); 307 | outputPaths = ( 308 | "$(DERIVED_FILE_DIR)/Pods-AnySense-checkManifestLockResult.txt", 309 | ); 310 | runOnlyForDeploymentPostprocessing = 0; 311 | shellPath = /bin/sh; 312 | shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; 313 | showEnvVarsInLog = 0; 314 | }; 315 | /* End PBXShellScriptBuildPhase section */ 316 | 317 | /* Begin PBXSourcesBuildPhase section */ 318 | B9104F562BFDE63A000D4DDD /* Sources */ = { 319 | isa = PBXSourcesBuildPhase; 320 | buildActionMask = 2147483647; 321 | files = ( 322 | B9E4B2362D62A9C40032E877 /* peripheralView.swift in Sources */, 323 | B9E4B2372D62A9C40032E877 /* accountView.swift in Sources */, 324 | B9E4B2382D62A9C40032E877 /* readView.swift in Sources */, 325 | B9E4B2392D62A9C40032E877 /* ContentView.swift in Sources */, 326 | B997CCD52D4E082900F62B49 /* AnySenseApp.swift in Sources */, 327 | B997CCD32D4E081300F62B49 /* dataStorage.swift in Sources */, 328 | B9E4B22D2D62A9BA0032E877 /* BluetoothManager.swift in Sources */, 329 | B9E4B22E2D62A9BA0032E877 /* WebRTCManager.swift in Sources */, 330 | B9E4B22F2D62A9BA0032E877 /* ARViewContainer.swift in Sources */, 331 | B9E4B2302D62A9BA0032E877 /* USBManager.swift in Sources */, 332 | B9944A342CFE983300232FBB /* MainPage.swift in Sources */, 333 | ); 334 | runOnlyForDeploymentPostprocessing = 0; 335 | }; 336 | B9104F662BFDE63C000D4DDD /* Sources */ = { 337 | isa = PBXSourcesBuildPhase; 338 | buildActionMask = 2147483647; 339 | files = ( 340 | ); 341 | runOnlyForDeploymentPostprocessing = 0; 342 | }; 343 | B9104F702BFDE63C000D4DDD /* Sources */ = { 344 | isa = PBXSourcesBuildPhase; 345 | buildActionMask = 2147483647; 346 | files = ( 347 | ); 348 | runOnlyForDeploymentPostprocessing = 0; 349 | }; 350 | /* End PBXSourcesBuildPhase section */ 351 | 352 | /* Begin PBXTargetDependency section */ 353 | B9104F6C2BFDE63C000D4DDD /* PBXTargetDependency */ = { 354 | isa = PBXTargetDependency; 355 | target = B9104F592BFDE63A000D4DDD /* AnySense */; 356 | targetProxy = B9104F6B2BFDE63C000D4DDD /* PBXContainerItemProxy */; 357 | }; 358 | B9104F762BFDE63C000D4DDD /* PBXTargetDependency */ = { 359 | isa = PBXTargetDependency; 360 | target = B9104F592BFDE63A000D4DDD /* AnySense */; 361 | targetProxy = B9104F752BFDE63C000D4DDD /* PBXContainerItemProxy */; 362 | }; 363 | /* End PBXTargetDependency section */ 364 | 365 | /* Begin XCBuildConfiguration section */ 366 | B9104F7C2BFDE63C000D4DDD /* Debug */ = { 367 | isa = XCBuildConfiguration; 368 | buildSettings = { 369 | ALWAYS_SEARCH_USER_PATHS = NO; 370 | ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; 371 | CLANG_ANALYZER_NONNULL = YES; 372 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 373 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; 374 | CLANG_ENABLE_MODULES = YES; 375 | CLANG_ENABLE_OBJC_ARC = YES; 376 | CLANG_ENABLE_OBJC_WEAK = YES; 377 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 378 | CLANG_WARN_BOOL_CONVERSION = YES; 379 | CLANG_WARN_COMMA = YES; 380 | CLANG_WARN_CONSTANT_CONVERSION = YES; 381 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 382 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 383 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 384 | CLANG_WARN_EMPTY_BODY = YES; 385 | CLANG_WARN_ENUM_CONVERSION = YES; 386 | CLANG_WARN_INFINITE_RECURSION = YES; 387 | CLANG_WARN_INT_CONVERSION = YES; 388 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 389 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 390 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 391 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 392 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 393 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 394 | CLANG_WARN_STRICT_PROTOTYPES = YES; 395 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 396 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 397 | CLANG_WARN_UNREACHABLE_CODE = YES; 398 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 399 | COPY_PHASE_STRIP = NO; 400 | DEBUG_INFORMATION_FORMAT = dwarf; 401 | ENABLE_STRICT_OBJC_MSGSEND = YES; 402 | ENABLE_TESTABILITY = YES; 403 | ENABLE_USER_SCRIPT_SANDBOXING = YES; 404 | GCC_C_LANGUAGE_STANDARD = gnu17; 405 | GCC_DYNAMIC_NO_PIC = NO; 406 | GCC_NO_COMMON_BLOCKS = YES; 407 | GCC_OPTIMIZATION_LEVEL = 0; 408 | GCC_PREPROCESSOR_DEFINITIONS = ( 409 | "DEBUG=1", 410 | "$(inherited)", 411 | ); 412 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 413 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 414 | GCC_WARN_UNDECLARED_SELECTOR = YES; 415 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 416 | GCC_WARN_UNUSED_FUNCTION = YES; 417 | GCC_WARN_UNUSED_VARIABLE = YES; 418 | INFOPLIST_KEY_NSBluetoothWhileInUseUsageDescription = ""; 419 | IPHONEOS_DEPLOYMENT_TARGET = 17.4; 420 | LOCALIZATION_PREFERS_STRING_CATALOGS = YES; 421 | MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; 422 | MTL_FAST_MATH = YES; 423 | ONLY_ACTIVE_ARCH = YES; 424 | SDKROOT = iphoneos; 425 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; 426 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 427 | }; 428 | name = Debug; 429 | }; 430 | B9104F7D2BFDE63C000D4DDD /* Release */ = { 431 | isa = XCBuildConfiguration; 432 | buildSettings = { 433 | ALWAYS_SEARCH_USER_PATHS = NO; 434 | ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; 435 | CLANG_ANALYZER_NONNULL = YES; 436 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 437 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; 438 | CLANG_ENABLE_MODULES = YES; 439 | CLANG_ENABLE_OBJC_ARC = YES; 440 | CLANG_ENABLE_OBJC_WEAK = YES; 441 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 442 | CLANG_WARN_BOOL_CONVERSION = YES; 443 | CLANG_WARN_COMMA = YES; 444 | CLANG_WARN_CONSTANT_CONVERSION = YES; 445 | CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; 446 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 447 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 448 | CLANG_WARN_EMPTY_BODY = YES; 449 | CLANG_WARN_ENUM_CONVERSION = YES; 450 | CLANG_WARN_INFINITE_RECURSION = YES; 451 | CLANG_WARN_INT_CONVERSION = YES; 452 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 453 | CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; 454 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 455 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 456 | CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; 457 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 458 | CLANG_WARN_STRICT_PROTOTYPES = YES; 459 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 460 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 461 | CLANG_WARN_UNREACHABLE_CODE = YES; 462 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 463 | COPY_PHASE_STRIP = NO; 464 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 465 | ENABLE_NS_ASSERTIONS = NO; 466 | ENABLE_STRICT_OBJC_MSGSEND = YES; 467 | ENABLE_USER_SCRIPT_SANDBOXING = YES; 468 | GCC_C_LANGUAGE_STANDARD = gnu17; 469 | GCC_NO_COMMON_BLOCKS = YES; 470 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 471 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 472 | GCC_WARN_UNDECLARED_SELECTOR = YES; 473 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 474 | GCC_WARN_UNUSED_FUNCTION = YES; 475 | GCC_WARN_UNUSED_VARIABLE = YES; 476 | INFOPLIST_KEY_NSBluetoothWhileInUseUsageDescription = ""; 477 | IPHONEOS_DEPLOYMENT_TARGET = 17.4; 478 | LOCALIZATION_PREFERS_STRING_CATALOGS = YES; 479 | MTL_ENABLE_DEBUG_INFO = NO; 480 | MTL_FAST_MATH = YES; 481 | SDKROOT = iphoneos; 482 | SWIFT_COMPILATION_MODE = wholemodule; 483 | VALIDATE_PRODUCT = YES; 484 | }; 485 | name = Release; 486 | }; 487 | B9104F7F2BFDE63C000D4DDD /* Debug */ = { 488 | isa = XCBuildConfiguration; 489 | baseConfigurationReference = E98E38A27D08BC9D5BE3E2E6 /* Pods-AnySense.debug.xcconfig */; 490 | buildSettings = { 491 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 492 | ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; 493 | CODE_SIGN_IDENTITY = "Apple Development"; 494 | CODE_SIGN_STYLE = Automatic; 495 | CURRENT_PROJECT_VERSION = 4; 496 | DEVELOPMENT_ASSET_PATHS = ""; 497 | DEVELOPMENT_TEAM = 88NB9U5CK6; 498 | ENABLE_PREVIEWS = YES; 499 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 500 | EXCLUDED_ARCHS = ""; 501 | GENERATE_INFOPLIST_FILE = YES; 502 | INFOPLIST_FILE = AnySense/Info.plist; 503 | INFOPLIST_KEY_CFBundleDisplayName = AnySense; 504 | INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; 505 | INFOPLIST_KEY_LSSupportsOpeningDocumentsInPlace = YES; 506 | INFOPLIST_KEY_NSBluetoothAlwaysUsageDescription = "This app can connect to Bluetooth devices to record streaming data from auxiliary sensors."; 507 | INFOPLIST_KEY_NSBluetoothPeripheralUsageDescription = "This app can connect to Bluetooth devices to record streaming data from auxiliary sensors."; 508 | INFOPLIST_KEY_NSBluetoothWhileInUseUsageDescription = ""; 509 | INFOPLIST_KEY_NSCameraUsageDescription = "This app needs access to the camera for RGB-D data collection and streaming."; 510 | INFOPLIST_KEY_NSMicrophoneUsageDescription = "This app uses microphone data for audio data collection and streaming."; 511 | INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; 512 | INFOPLIST_KEY_UILaunchStoryboardName = AnySenseLaunchScreen.storyboard; 513 | INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = UIInterfaceOrientationPortrait; 514 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 515 | LD_RUNPATH_SEARCH_PATHS = ( 516 | "$(inherited)", 517 | "@executable_path/Frameworks", 518 | ); 519 | MARKETING_VERSION = 1.0; 520 | PRODUCT_BUNDLE_IDENTIFIER = GRAIL.AnySense; 521 | PRODUCT_NAME = "$(TARGET_NAME)"; 522 | PROVISIONING_PROFILE_SPECIFIER = ""; 523 | SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; 524 | SUPPORTS_MACCATALYST = NO; 525 | SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = NO; 526 | SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO; 527 | SWIFT_EMIT_LOC_STRINGS = YES; 528 | SWIFT_VERSION = 5.0; 529 | TARGETED_DEVICE_FAMILY = 1; 530 | }; 531 | name = Debug; 532 | }; 533 | B9104F802BFDE63C000D4DDD /* Release */ = { 534 | isa = XCBuildConfiguration; 535 | baseConfigurationReference = BA11D17F77459219187F7A30 /* Pods-AnySense.release.xcconfig */; 536 | buildSettings = { 537 | ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; 538 | ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; 539 | CODE_SIGN_IDENTITY = "Apple Development"; 540 | CODE_SIGN_STYLE = Automatic; 541 | CURRENT_PROJECT_VERSION = 4; 542 | DEVELOPMENT_ASSET_PATHS = ""; 543 | DEVELOPMENT_TEAM = 88NB9U5CK6; 544 | ENABLE_PREVIEWS = YES; 545 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 546 | GENERATE_INFOPLIST_FILE = YES; 547 | INFOPLIST_FILE = AnySense/Info.plist; 548 | INFOPLIST_KEY_CFBundleDisplayName = AnySense; 549 | INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; 550 | INFOPLIST_KEY_LSSupportsOpeningDocumentsInPlace = YES; 551 | INFOPLIST_KEY_NSBluetoothAlwaysUsageDescription = "This app can connect to Bluetooth devices to record streaming data from auxiliary sensors."; 552 | INFOPLIST_KEY_NSBluetoothPeripheralUsageDescription = "This app can connect to Bluetooth devices to record streaming data from auxiliary sensors."; 553 | INFOPLIST_KEY_NSBluetoothWhileInUseUsageDescription = ""; 554 | INFOPLIST_KEY_NSCameraUsageDescription = "This app needs access to the camera for RGB-D data collection and streaming."; 555 | INFOPLIST_KEY_NSMicrophoneUsageDescription = "This app uses microphone data for audio data collection and streaming."; 556 | INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; 557 | INFOPLIST_KEY_UILaunchStoryboardName = AnySenseLaunchScreen.storyboard; 558 | INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = UIInterfaceOrientationPortrait; 559 | IPHONEOS_DEPLOYMENT_TARGET = 17.0; 560 | LD_RUNPATH_SEARCH_PATHS = ( 561 | "$(inherited)", 562 | "@executable_path/Frameworks", 563 | ); 564 | MARKETING_VERSION = 1.0; 565 | PRODUCT_BUNDLE_IDENTIFIER = GRAIL.AnySense; 566 | PRODUCT_NAME = "$(TARGET_NAME)"; 567 | PROVISIONING_PROFILE_SPECIFIER = ""; 568 | SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; 569 | SUPPORTS_MACCATALYST = NO; 570 | SUPPORTS_MAC_DESIGNED_FOR_IPHONE_IPAD = NO; 571 | SUPPORTS_XR_DESIGNED_FOR_IPHONE_IPAD = NO; 572 | SWIFT_EMIT_LOC_STRINGS = YES; 573 | SWIFT_VERSION = 5.0; 574 | TARGETED_DEVICE_FAMILY = 1; 575 | }; 576 | name = Release; 577 | }; 578 | B9104F822BFDE63C000D4DDD /* Debug */ = { 579 | isa = XCBuildConfiguration; 580 | buildSettings = { 581 | BUNDLE_LOADER = "$(TEST_HOST)"; 582 | CODE_SIGN_STYLE = Automatic; 583 | CURRENT_PROJECT_VERSION = 1; 584 | DEVELOPMENT_TEAM = 9U23FMDF45; 585 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 586 | GENERATE_INFOPLIST_FILE = YES; 587 | IPHONEOS_DEPLOYMENT_TARGET = 17.4; 588 | MARKETING_VERSION = 1.0; 589 | PRODUCT_BUNDLE_IDENTIFIER = Raunaq.USBInterfaceAppTests; 590 | PRODUCT_NAME = "$(TARGET_NAME)"; 591 | SWIFT_EMIT_LOC_STRINGS = NO; 592 | SWIFT_VERSION = 5.0; 593 | TARGETED_DEVICE_FAMILY = "1,2"; 594 | TEST_HOST = "$(BUILT_PRODUCTS_DIR)/AnySense.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/AnySense"; 595 | }; 596 | name = Debug; 597 | }; 598 | B9104F832BFDE63C000D4DDD /* Release */ = { 599 | isa = XCBuildConfiguration; 600 | buildSettings = { 601 | BUNDLE_LOADER = "$(TEST_HOST)"; 602 | CODE_SIGN_STYLE = Automatic; 603 | CURRENT_PROJECT_VERSION = 1; 604 | DEVELOPMENT_TEAM = 9U23FMDF45; 605 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 606 | GENERATE_INFOPLIST_FILE = YES; 607 | IPHONEOS_DEPLOYMENT_TARGET = 17.4; 608 | MARKETING_VERSION = 1.0; 609 | PRODUCT_BUNDLE_IDENTIFIER = Raunaq.USBInterfaceAppTests; 610 | PRODUCT_NAME = "$(TARGET_NAME)"; 611 | SWIFT_EMIT_LOC_STRINGS = NO; 612 | SWIFT_VERSION = 5.0; 613 | TARGETED_DEVICE_FAMILY = "1,2"; 614 | TEST_HOST = "$(BUILT_PRODUCTS_DIR)/AnySense.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/AnySense"; 615 | }; 616 | name = Release; 617 | }; 618 | B9104F852BFDE63C000D4DDD /* Debug */ = { 619 | isa = XCBuildConfiguration; 620 | buildSettings = { 621 | CODE_SIGN_STYLE = Automatic; 622 | CURRENT_PROJECT_VERSION = 1; 623 | DEVELOPMENT_TEAM = 9U23FMDF45; 624 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 625 | GENERATE_INFOPLIST_FILE = YES; 626 | MARKETING_VERSION = 1.0; 627 | PRODUCT_BUNDLE_IDENTIFIER = Raunaq.USBInterfaceAppUITests; 628 | PRODUCT_NAME = "$(TARGET_NAME)"; 629 | SWIFT_EMIT_LOC_STRINGS = NO; 630 | SWIFT_VERSION = 5.0; 631 | TARGETED_DEVICE_FAMILY = "1,2"; 632 | TEST_TARGET_NAME = USBInterfaceApp; 633 | }; 634 | name = Debug; 635 | }; 636 | B9104F862BFDE63C000D4DDD /* Release */ = { 637 | isa = XCBuildConfiguration; 638 | buildSettings = { 639 | CODE_SIGN_STYLE = Automatic; 640 | CURRENT_PROJECT_VERSION = 1; 641 | DEVELOPMENT_TEAM = 9U23FMDF45; 642 | ENABLE_USER_SCRIPT_SANDBOXING = NO; 643 | GENERATE_INFOPLIST_FILE = YES; 644 | MARKETING_VERSION = 1.0; 645 | PRODUCT_BUNDLE_IDENTIFIER = Raunaq.USBInterfaceAppUITests; 646 | PRODUCT_NAME = "$(TARGET_NAME)"; 647 | SWIFT_EMIT_LOC_STRINGS = NO; 648 | SWIFT_VERSION = 5.0; 649 | TARGETED_DEVICE_FAMILY = "1,2"; 650 | TEST_TARGET_NAME = USBInterfaceApp; 651 | }; 652 | name = Release; 653 | }; 654 | /* End XCBuildConfiguration section */ 655 | 656 | /* Begin XCConfigurationList section */ 657 | B9104F552BFDE63A000D4DDD /* Build configuration list for PBXProject "AnySense" */ = { 658 | isa = XCConfigurationList; 659 | buildConfigurations = ( 660 | B9104F7C2BFDE63C000D4DDD /* Debug */, 661 | B9104F7D2BFDE63C000D4DDD /* Release */, 662 | ); 663 | defaultConfigurationIsVisible = 0; 664 | defaultConfigurationName = Release; 665 | }; 666 | B9104F7E2BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySense" */ = { 667 | isa = XCConfigurationList; 668 | buildConfigurations = ( 669 | B9104F7F2BFDE63C000D4DDD /* Debug */, 670 | B9104F802BFDE63C000D4DDD /* Release */, 671 | ); 672 | defaultConfigurationIsVisible = 0; 673 | defaultConfigurationName = Release; 674 | }; 675 | B9104F812BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySenseTests" */ = { 676 | isa = XCConfigurationList; 677 | buildConfigurations = ( 678 | B9104F822BFDE63C000D4DDD /* Debug */, 679 | B9104F832BFDE63C000D4DDD /* Release */, 680 | ); 681 | defaultConfigurationIsVisible = 0; 682 | defaultConfigurationName = Release; 683 | }; 684 | B9104F842BFDE63C000D4DDD /* Build configuration list for PBXNativeTarget "AnySenseUITests" */ = { 685 | isa = XCConfigurationList; 686 | buildConfigurations = ( 687 | B9104F852BFDE63C000D4DDD /* Debug */, 688 | B9104F862BFDE63C000D4DDD /* Release */, 689 | ); 690 | defaultConfigurationIsVisible = 0; 691 | defaultConfigurationName = Release; 692 | }; 693 | /* End XCConfigurationList section */ 694 | 695 | /* Begin XCRemoteSwiftPackageReference section */ 696 | B905B27B2C88D706006F994E /* XCRemoteSwiftPackageReference "WebRTC-iOS-SDK" */ = { 697 | isa = XCRemoteSwiftPackageReference; 698 | repositoryURL = "https://github.com/ant-media/WebRTC-iOS-SDK"; 699 | requirement = { 700 | branch = master; 701 | kind = branch; 702 | }; 703 | }; 704 | B98D69592C0486D500648D95 /* XCRemoteSwiftPackageReference "PythonKit" */ = { 705 | isa = XCRemoteSwiftPackageReference; 706 | repositoryURL = "https://github.com/pvieito/PythonKit"; 707 | requirement = { 708 | branch = master; 709 | kind = branch; 710 | }; 711 | }; 712 | B98D69602C048AF100648D95 /* XCRemoteSwiftPackageReference "numpy" */ = { 713 | isa = XCRemoteSwiftPackageReference; 714 | repositoryURL = "https://github.com/numpy/numpy"; 715 | requirement = { 716 | branch = main; 717 | kind = branch; 718 | }; 719 | }; 720 | /* End XCRemoteSwiftPackageReference section */ 721 | }; 722 | rootObject = B9104F522BFDE63A000D4DDD /* Project object */; 723 | } 724 | -------------------------------------------------------------------------------- /AnySense/Managers/ARViewContainer.swift: -------------------------------------------------------------------------------- 1 | // 2 | // ARViewContainer.swift 3 | // Anysense 4 | // 5 | // Created by Michael on 2024/7/25. 6 | // 7 | 8 | import SwiftUI 9 | import ARKit 10 | import RealityKit 11 | import Foundation 12 | import AVFoundation 13 | import Network 14 | import CoreMedia 15 | import CoreImage 16 | import UIKit 17 | import CoreImage.CIFilterBuiltins 18 | //import WebRTC 19 | 20 | struct RecordingFiles { 21 | let rgbFileName: URL 22 | let depthFileName: URL 23 | let timestamp: String 24 | let rgbImagesDirectory: URL 25 | let depthImagesDirectory: URL 26 | let poseFile: URL 27 | let generalDataDirectory: String 28 | let tactileFile: URL 29 | } 30 | 31 | func createFile(fileURL: URL) throws { 32 | let success = FileManager.default.createFile(atPath: fileURL.path, contents: nil, attributes: nil) 33 | if !success { 34 | throw NSError(domain: "FileCreationError", code: 1, userInfo: [NSLocalizedDescriptionKey: "Failed to create file at \(fileURL.path)"]) 35 | } 36 | } 37 | 38 | struct ARViewContainer: UIViewRepresentable { 39 | var session: ARSession 40 | typealias UIViewType = ARView 41 | 42 | func makeUIView(context: Context) -> ARView { 43 | // Initialize the ARView 44 | let arView = ARView(frame: .zero, cameraMode: .ar, automaticallyConfigureSession: false) 45 | arView.session = session 46 | arView.environment.sceneUnderstanding.options = [] // No extra scene understanding 47 | return arView 48 | } 49 | func updateUIView(_ uiView: ARView, context: Context) { 50 | if uiView.session !== session { 51 | uiView.session = session 52 | } 53 | } 54 | } 55 | 56 | class DepthStatus: ObservableObject { 57 | @Published var isDepthAvailable: Bool = true 58 | @Published var showAlert: Bool = false 59 | 60 | public func setUnavailable() { 61 | isDepthAvailable = false 62 | showAlert = true 63 | } 64 | } 65 | 66 | class ARViewModel: ObservableObject{ 67 | @Published var isOpen : Bool = false 68 | @Published var depthStatus = DepthStatus() 69 | var session = ARSession() 70 | var audioSession = AVCaptureSession() 71 | var audioCaptureDelegate: AudioCaptureDelegate? 72 | 73 | public var userFPS: Double? 74 | public var isColorMapOpened = false 75 | private var usbManager = USBManager() 76 | 77 | private var orientation: UIInterfaceOrientation = .portrait 78 | 79 | // Control the destination of rgb and depth video file 80 | private var assetWriter: AVAssetWriter? 81 | private var videoInput: AVAssetWriterInput? 82 | private var audioInput: AVAssetWriterInput? 83 | private var pixelBufferAdapter: AVAssetWriterInputPixelBufferAdaptor? 84 | private var depthAssetWriter: AVAssetWriter? 85 | private var depthVideoInput: AVAssetWriterInput? 86 | private var depthPixelBufferAdapter: AVAssetWriterInputPixelBufferAdaptor? 87 | private var viewPortSize = CGSize(width: 720, height: 960) 88 | private var depthViewPortSize = CGSize(width: 192, height: 256) 89 | 90 | private var combinedRGBTransform: CGAffineTransform? 91 | private var combinedDepthTransform: CGAffineTransform? 92 | 93 | private var rgbOutputPixelBufferUSB: CVPixelBuffer? 94 | private var depthOutputPixelBufferUSB: CVPixelBuffer? 95 | private var depthConfidenceOutputPixelBufferUSB: CVPixelBuffer? 96 | 97 | private var poseFileHandle: FileHandle? 98 | 99 | // Control the destination of rgb images directory and depth images directory 100 | private var rgbDirect: URL = URL(fileURLWithPath: "") 101 | private var depthDirect: URL = URL(fileURLWithPath: "") 102 | // Control the destination of pose data text file 103 | private var poseURL: URL = URL(fileURLWithPath: "") 104 | private var generalURL: URL = URL(fileURLWithPath: "") 105 | private var globalPoseFileName: String = "" 106 | 107 | private var depthRetryCount = 0 108 | private var maxDepthRetries = 50 109 | 110 | private var startTime: CMTime? 111 | private let ciContext: CIContext 112 | 113 | private var displayLink: CADisplayLink? 114 | private var lastTimestamp: CFTimeInterval = 0 115 | 116 | private var streamConnection: NWConnection? 117 | 118 | private var rgbAttributes: [String: Any] = [:] 119 | private var depthAttributes: [String: Any] = [:] 120 | private var depthConfAttributes: [String: Any] = [:] 121 | private var audioOutputSettings: [String: Any] = [:] 122 | 123 | init() { 124 | self.rgbAttributes = [ 125 | kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32ARGB), 126 | kCVPixelBufferWidthKey as String: Int(viewPortSize.width), 127 | kCVPixelBufferHeightKey as String: Int(viewPortSize.height) 128 | ] 129 | self.depthAttributes = [ 130 | kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_OneComponent32Float, 131 | kCVPixelBufferWidthKey as String: Int(depthViewPortSize.width), 132 | kCVPixelBufferHeightKey as String: Int(depthViewPortSize.height) 133 | ] 134 | self.depthConfAttributes = [ 135 | kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_OneComponent8, 136 | kCVPixelBufferWidthKey as String: Int(depthViewPortSize.width), 137 | kCVPixelBufferHeightKey as String: Int(depthViewPortSize.height) 138 | ] 139 | self.audioOutputSettings = [ 140 | AVFormatIDKey: kAudioFormatMPEG4AAC, 141 | AVNumberOfChannelsKey: 2, 142 | AVSampleRateKey: 44100.0, 143 | AVEncoderBitRateKey: 128000 144 | ] 145 | 146 | self.ciContext = CIContext() 147 | } 148 | 149 | 150 | 151 | private func setupAudioSession() { 152 | guard let audioDevice = AVCaptureDevice.default(for: .audio), 153 | let audioDeviceInput = try? AVCaptureDeviceInput(device: audioDevice) else { 154 | print("Unable to access microphone.") 155 | return 156 | } 157 | audioSession.addInput(audioDeviceInput) 158 | 159 | let audioOutput = AVCaptureAudioDataOutput() 160 | if audioSession.canAddOutput(audioOutput) { 161 | audioSession.addOutput(audioOutput) 162 | } 163 | } 164 | 165 | private func setupTransforms() { 166 | DispatchQueue.global(qos: .userInitiated).async { 167 | while self.depthRetryCount < self.maxDepthRetries { 168 | guard let currentFrame = self.session.currentFrame else { 169 | usleep(10000) 170 | continue 171 | } 172 | let flipTransform = (self.orientation.isPortrait) 173 | ? CGAffineTransform(scaleX: -1, y: -1).translatedBy(x: -1, y: -1) 174 | : .identity 175 | 176 | if self.combinedRGBTransform == nil { 177 | self.initializeRGBTransform(frame: currentFrame, flipTransform: flipTransform) 178 | } 179 | 180 | if !self.depthStatus.isDepthAvailable { break } 181 | 182 | if self.combinedDepthTransform == nil { 183 | if self.initializeDepthTransform(frame: currentFrame, flipTransform: flipTransform) { 184 | break 185 | } 186 | } 187 | 188 | self.depthRetryCount += 1 189 | usleep(10000) 190 | } 191 | 192 | if self.combinedDepthTransform == nil { 193 | print("Depth initialization failed after \(self.maxDepthRetries) attempts.") 194 | } 195 | } 196 | } 197 | 198 | private func initializeRGBTransform(frame: ARFrame, flipTransform: CGAffineTransform) { 199 | let rgbPixelBuffer = frame.capturedImage 200 | let rgbSize = CGSize(width: CVPixelBufferGetWidth(rgbPixelBuffer), height: CVPixelBufferGetHeight(rgbPixelBuffer)) 201 | let normalizeTransform = CGAffineTransform(scaleX: 1.0/rgbSize.width, y: 1.0/rgbSize.height) 202 | let displayTransform = frame.displayTransform(for: self.orientation, viewportSize: self.viewPortSize) 203 | let toViewPortTransform = CGAffineTransform(scaleX: self.viewPortSize.width, y: self.viewPortSize.height) 204 | 205 | self.combinedRGBTransform = normalizeTransform 206 | .concatenating(flipTransform) 207 | .concatenating(displayTransform) 208 | .concatenating(toViewPortTransform) 209 | } 210 | 211 | private func initializeDepthTransform(frame: ARFrame, flipTransform: CGAffineTransform) -> Bool { 212 | guard let depthPixelBuffer = frame.sceneDepth?.depthMap else { 213 | print("Depth map unavailable. Retrying (\(self.depthRetryCount)/\(self.maxDepthRetries))") 214 | return false 215 | } 216 | let depthSize = CGSize(width: CVPixelBufferGetWidth(depthPixelBuffer), height: CVPixelBufferGetHeight(depthPixelBuffer)) 217 | let normalizeTransform = CGAffineTransform(scaleX: 1.0 / depthSize.width, y: 1.0 / depthSize.height) 218 | 219 | let depthDisplayTransform = frame.displayTransform(for: self.orientation, viewportSize: self.depthViewPortSize) 220 | let toDepthViewPortTransform = CGAffineTransform(scaleX: self.depthViewPortSize.width, y: self.depthViewPortSize.height) 221 | 222 | self.combinedDepthTransform = normalizeTransform 223 | .concatenating(flipTransform) 224 | .concatenating(depthDisplayTransform) 225 | .concatenating(toDepthViewPortTransform) 226 | 227 | return true 228 | } 229 | 230 | func setupARSession() { 231 | self.startARSession() 232 | 233 | setupAudioSession() 234 | 235 | setupTransforms() 236 | 237 | print("Finished setting up ARViewModel.") 238 | } 239 | 240 | func startARSession() { 241 | let status = AVCaptureDevice.authorizationStatus(for: .video) 242 | guard status == .authorized else { 243 | print("Camera permissions not granted.") 244 | return 245 | } 246 | // Create and configure the AR session configuration 247 | let configuration = ARWorldTrackingConfiguration() 248 | 249 | // Loop through available video formats and select the wide-angle camera format 250 | for videoFormat in ARWorldTrackingConfiguration.supportedVideoFormats { 251 | if videoFormat.captureDeviceType == .builtInWideAngleCamera { 252 | print("Wide-angle camera selected: \(videoFormat)") 253 | configuration.videoFormat = videoFormat 254 | break 255 | } else { 256 | print("Unsupported video format: \(videoFormat.captureDeviceType)") 257 | } 258 | } 259 | 260 | // Set the session configuration properties 261 | if ARWorldTrackingConfiguration.supportsFrameSemantics(.sceneDepth) { 262 | configuration.frameSemantics.insert(.sceneDepth) 263 | } else { 264 | depthStatus.setUnavailable() 265 | } 266 | configuration.planeDetection = [] 267 | configuration.environmentTexturing = .none // No environment texturing 268 | configuration.sceneReconstruction = [] // No scene reconstruction 269 | configuration.isAutoFocusEnabled = false 270 | 271 | // Run the session with the configuration 272 | session.run(configuration, options: [.resetTracking, .removeExistingAnchors]) 273 | print("Starting session") 274 | isOpen = true 275 | } 276 | 277 | func pauseARSession(){ 278 | session.pause() 279 | isOpen = false 280 | } 281 | 282 | func killARSession() { 283 | session.pause() // Pause before releasing resources 284 | session = ARSession() // Replace with a new ARSession 285 | isOpen = false 286 | print("ARSession killed and reset.") 287 | } 288 | 289 | func startUSBStreaming() { 290 | displayLink = CADisplayLink(target: self, selector: #selector(sendFrameUSB)) 291 | displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: Float(self.userFPS!), maximum: Float(self.userFPS!), preferred: Float(self.userFPS!)) 292 | displayLink?.add(to: .main, forMode: .common) 293 | } 294 | 295 | func stopUSBStreaming() { 296 | displayLink?.invalidate() 297 | displayLink = nil 298 | } 299 | 300 | func setupUSBStreaming() { 301 | var rgbBuffer: CVPixelBuffer? 302 | 303 | let status = CVPixelBufferCreate( 304 | kCFAllocatorDefault, 305 | Int(viewPortSize.width), 306 | Int(viewPortSize.height), 307 | kCVPixelFormatType_32ARGB, 308 | rgbAttributes as CFDictionary, 309 | &rgbBuffer 310 | ) 311 | guard status == kCVReturnSuccess else { 312 | print("Failed to create CVPixelBuffer") 313 | return 314 | } 315 | self.rgbOutputPixelBufferUSB = rgbBuffer 316 | 317 | if self.depthStatus.isDepthAvailable { 318 | var depthBuffer: CVPixelBuffer? 319 | var depthConfidenceBuffer: CVPixelBuffer? 320 | 321 | let depthStatus = CVPixelBufferCreate( 322 | kCFAllocatorDefault, 323 | Int(depthViewPortSize.width), 324 | Int(depthViewPortSize.height), 325 | kCVPixelFormatType_DepthFloat32, 326 | depthAttributes as CFDictionary, 327 | &depthBuffer 328 | ) 329 | 330 | guard depthStatus == kCVReturnSuccess else { 331 | print("Failed to create CVPixelBuffer") 332 | return 333 | } 334 | self.depthOutputPixelBufferUSB = depthBuffer 335 | 336 | let depthConfidenceStatus = CVPixelBufferCreate( 337 | kCFAllocatorDefault, 338 | Int(depthViewPortSize.width), 339 | Int(depthViewPortSize.height), 340 | kCVPixelFormatType_OneComponent8, 341 | depthConfAttributes as CFDictionary, 342 | &depthConfidenceBuffer 343 | ) 344 | guard depthConfidenceStatus == kCVReturnSuccess else { 345 | print("Failed to create CVPixelBuffer") 346 | return 347 | } 348 | self.depthConfidenceOutputPixelBufferUSB = depthConfidenceBuffer 349 | } 350 | 351 | print("Made all USB Buffers") 352 | usbManager.connect() 353 | } 354 | 355 | func killUSBStreaming() { 356 | self.usbManager.disconnect() 357 | 358 | self.rgbOutputPixelBufferUSB = nil 359 | self.depthOutputPixelBufferUSB = nil 360 | self.depthConfidenceOutputPixelBufferUSB = nil 361 | } 362 | 363 | // func startWiFiStreaming(host: String, port: UInt16) { 364 | // Set up the network connection 365 | // // Start WebRTC connection 366 | // webRTCManager.setupConnection() 367 | // } 368 | 369 | // func stopWiFiStreaming() { 370 | // displayLink?.invalidate() 371 | // displayLink = nil 372 | // streamConnection?.cancel() 373 | // streamConnection = nil 374 | // } 375 | 376 | @objc private func sendFrame(link: CADisplayLink) { 377 | streamVideoFrameUSB() 378 | } 379 | 380 | @objc private func sendFrameUSB(link: CADisplayLink) { 381 | streamVideoFrameUSB() 382 | } 383 | 384 | private func processDepthStreamData(depthPixelBuffer: CVPixelBuffer, outputBuffer: CVPixelBuffer, isDepth: Bool) -> Data? { 385 | CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly) 386 | CVPixelBufferLockBaseAddress(outputBuffer, []) 387 | 388 | let depthCiImage = CIImage(cvPixelBuffer: depthPixelBuffer) 389 | let depthTransformedImage = depthCiImage.transformed(by: self.combinedDepthTransform!) 390 | self.ciContext.render(depthTransformedImage, to: outputBuffer) 391 | 392 | let compressedData = self.usbManager.compressData(from: outputBuffer, isDepth: isDepth) 393 | 394 | CVPixelBufferUnlockBaseAddress(outputBuffer, []) 395 | CVPixelBufferUnlockBaseAddress(depthPixelBuffer, .readOnly) 396 | 397 | return compressedData 398 | } 399 | 400 | func streamVideoFrameUSB() { 401 | guard let currentFrame = session.currentFrame else {return} 402 | 403 | let rgbPixelBuffer = currentFrame.capturedImage 404 | // TODO: Check if we need to change this at all 405 | var depthPixelBuffer: CVPixelBuffer? = nil 406 | var depthConfidencePixelBuffer: CVPixelBuffer? = nil 407 | if self.depthStatus.isDepthAvailable { 408 | guard let depthBuffer = currentFrame.sceneDepth?.depthMap else { return } 409 | depthPixelBuffer = depthBuffer 410 | guard let depthConfidenceBuffer = currentFrame.sceneDepth?.confidenceMap else { return } 411 | depthConfidencePixelBuffer = depthConfidenceBuffer 412 | } 413 | 414 | 415 | let cameraIntrinsics = currentFrame.camera.intrinsics 416 | var intrinsicCoeffs = IntrinsicMatrixCoeffs( 417 | fx: cameraIntrinsics.columns.0.x, 418 | fy: cameraIntrinsics.columns.1.y, 419 | tx: cameraIntrinsics.columns.2.x, 420 | ty: cameraIntrinsics.columns.2.y 421 | ) 422 | let cameraTransform = currentFrame.camera.transform 423 | 424 | // Transform the orientation matrix to unit quaternion 425 | let quaternion = simd_quaternion(cameraTransform) 426 | var camera_pose = CameraPose( 427 | qx: quaternion.vector.x, 428 | qy: quaternion.vector.y, 429 | qz: quaternion.vector.z, 430 | qw: quaternion.vector.w, 431 | tx: cameraTransform.columns.3.x, 432 | ty: cameraTransform.columns.3.y, 433 | tz: cameraTransform.columns.3.z 434 | ) 435 | var record3dHeader = Record3DHeader( 436 | rgbWidth: UInt32(self.viewPortSize.width), 437 | rgbHeight: UInt32(self.viewPortSize.height), 438 | depthWidth: UInt32(self.depthViewPortSize.width), 439 | depthHeight: UInt32(self.depthViewPortSize.height), 440 | confidenceWidth: UInt32(self.depthViewPortSize.width), 441 | confidenceHeight: UInt32(self.depthViewPortSize.height), 442 | rgbSize: 0, 443 | depthSize: 0, 444 | confidenceMapSize: 0, 445 | miscSize: 0, 446 | deviceType: 1 447 | ) 448 | 449 | DispatchQueue.global(qos: .userInitiated).async { 450 | CVPixelBufferLockBaseAddress(rgbPixelBuffer, .readOnly) 451 | CVPixelBufferLockBaseAddress(self.rgbOutputPixelBufferUSB!, []) 452 | 453 | let rgbCiImage = CIImage(cvPixelBuffer: rgbPixelBuffer) 454 | let rgbTransformedImage = rgbCiImage.transformed(by: self.combinedRGBTransform!) 455 | 456 | guard let rgbCgImage = self.ciContext.createCGImage(rgbTransformedImage, from: rgbTransformedImage.extent) else{ 457 | return 458 | } 459 | let rgbImageData = UIImage(cgImage: rgbCgImage).jpegData(compressionQuality: 0.5) 460 | 461 | record3dHeader.rgbSize = UInt32(rgbImageData!.count) 462 | 463 | CVPixelBufferUnlockBaseAddress(self.rgbOutputPixelBufferUSB!, []) 464 | CVPixelBufferUnlockBaseAddress(rgbPixelBuffer, .readOnly) 465 | 466 | var compressedDepthData: Data? = nil 467 | var compressedDepthConfData: Data? = nil 468 | 469 | if self.depthStatus.isDepthAvailable { 470 | compressedDepthData = self.processDepthStreamData(depthPixelBuffer: depthPixelBuffer!, outputBuffer: self.depthOutputPixelBufferUSB!, isDepth: true) 471 | compressedDepthConfData = self.processDepthStreamData(depthPixelBuffer: depthConfidencePixelBuffer!, outputBuffer: self.depthConfidenceOutputPixelBufferUSB!, isDepth: false) 472 | 473 | record3dHeader.depthSize = UInt32(compressedDepthData?.count ?? 0) 474 | record3dHeader.confidenceMapSize = UInt32(compressedDepthConfData?.count ?? 0) 475 | } 476 | self.usbManager.sendData( 477 | record3dHeaderData: Data(bytes: &record3dHeader, count: MemoryLayout.size), 478 | intrinsicMatData: Data(bytes: &intrinsicCoeffs, count: MemoryLayout.size), 479 | poseData: Data(bytes: &camera_pose, count: MemoryLayout.size), 480 | rgbImageData: rgbImageData!, 481 | compressedDepthData: compressedDepthData, 482 | compressedConfData: compressedDepthConfData 483 | ) 484 | 485 | 486 | } 487 | 488 | } 489 | 490 | @objc private func updateFrame(link: CADisplayLink) { 491 | guard lastTimestamp > 0 else { 492 | // Initialize timestamp on the first call 493 | lastTimestamp = link.timestamp 494 | return 495 | } 496 | captureVideoFrame() 497 | } 498 | 499 | func startRecording() -> RecordingFiles { 500 | let saveFileNames = setupRecording() 501 | 502 | assetWriter?.startWriting() 503 | startTime = CMTimeMake(value: Int64(CACurrentMediaTime() * 1000), timescale: 1000) 504 | assetWriter?.startSession(atSourceTime: startTime!) 505 | 506 | DispatchQueue.global(qos: .background).async { 507 | self.audioSession.startRunning() 508 | } 509 | if self.depthStatus.isDepthAvailable { 510 | depthAssetWriter?.startWriting() 511 | depthAssetWriter?.startSession(atSourceTime: startTime!) 512 | } 513 | 514 | displayLink = CADisplayLink(target: self, selector: #selector(updateFrame)) 515 | displayLink?.preferredFrameRateRange = CAFrameRateRange(minimum: Float(self.userFPS!), maximum: Float(self.userFPS!), preferred: Float(self.userFPS!)) 516 | displayLink?.add(to: .main, forMode: .common) 517 | 518 | return saveFileNames! 519 | 520 | } 521 | 522 | 523 | func stopRecording(){ 524 | displayLink?.invalidate() 525 | displayLink = nil 526 | audioSession.stopRunning() 527 | audioInput?.markAsFinished() 528 | videoInput?.markAsFinished() 529 | 530 | audioCaptureDelegate = nil 531 | 532 | assetWriter?.finishWriting { 533 | self.assetWriter = nil 534 | print("RGB Video recording finished.") 535 | } 536 | 537 | depthVideoInput?.markAsFinished() 538 | depthAssetWriter?.finishWriting { 539 | self.depthAssetWriter = nil 540 | print("Depth Video recording finished.") 541 | } 542 | 543 | do { 544 | try poseFileHandle?.close() 545 | } catch { 546 | print("Error closing pose file") 547 | } 548 | } 549 | 550 | func setupRecording() -> RecordingFiles? { 551 | // Determine all the destinated file saving URL or this recording by its start time 552 | let dateFormatter = DateFormatter() 553 | dateFormatter.dateFormat = "yyyy-MM-dd-HH_mm_ss" 554 | let timestamp = dateFormatter.string(from: Date()) 555 | 556 | let fileNames = [ 557 | "RGB": "RGB_\(timestamp).mp4", 558 | "Depth": "Depth_\(timestamp).mp4", 559 | "Pose": "AR_Pose_\(timestamp).txt", 560 | "Tactile": "Tactile_\(timestamp).bin", 561 | "RGBImages": "RGB_Images_\(timestamp)", 562 | "DepthImages": isColorMapOpened ? "Depth_Colored_Images_\(timestamp)" : "Depth_Images_\(timestamp)" 563 | ] 564 | 565 | guard let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else { 566 | print("Failed to get document directory") 567 | return nil 568 | } 569 | 570 | let generalDataDirectory = documentsURL.appendingPathComponent(timestamp) 571 | let rgbVideoURL = generalDataDirectory.appendingPathComponent(fileNames["RGB"]!) 572 | let depthVideoURL = generalDataDirectory.appendingPathComponent(fileNames["Depth"]!) 573 | let poseTextURL = generalDataDirectory.appendingPathComponent(fileNames["Pose"]!) 574 | let tactileFileURL = generalDataDirectory.appendingPathComponent(fileNames["Tactile"]!) 575 | let rgbImagesDirectory = generalDataDirectory.appendingPathComponent(fileNames["RGBImages"]!) 576 | let depthImagesDirectory = generalDataDirectory.appendingPathComponent(fileNames["DepthImages"]!) 577 | 578 | do { 579 | try FileManager.default.createDirectory(at: generalDataDirectory, withIntermediateDirectories: true) 580 | if self.depthStatus.isDepthAvailable { 581 | try FileManager.default.createDirectory(at: depthImagesDirectory, withIntermediateDirectories: true) 582 | } 583 | try createFile(fileURL: poseTextURL) 584 | } catch { 585 | print("Error creating directories") 586 | } 587 | 588 | self.rgbDirect = rgbImagesDirectory 589 | self.depthDirect = depthImagesDirectory 590 | self.poseURL = poseTextURL 591 | self.generalURL = generalDataDirectory 592 | 593 | do { 594 | // Determine which video file url the assetWriter will write into 595 | 596 | // RGB 597 | self.assetWriter = try AVAssetWriter(outputURL: rgbVideoURL, fileType: .mp4) 598 | 599 | let videoSettings: [String: Any] = [ 600 | AVVideoCodecKey: AVVideoCodecType.h264, 601 | AVVideoWidthKey: viewPortSize.width, 602 | AVVideoHeightKey: viewPortSize.height, 603 | AVVideoScalingModeKey: AVVideoScalingModeResizeAspectFill, 604 | /* 605 | AVVideoCompressionPropertiesKey: [ 606 | AVVideoAverageBitRateKey: 2000000, 607 | AVVideoMaxKeyFrameIntervalKey: 30 608 | ] 609 | */ 610 | ] 611 | 612 | self.videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings) 613 | self.videoInput?.expectsMediaDataInRealTime = true 614 | self.assetWriter?.add(videoInput!) 615 | 616 | self.audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioOutputSettings) 617 | self.audioInput?.expectsMediaDataInRealTime = true 618 | self.assetWriter?.add(audioInput!) 619 | 620 | self.pixelBufferAdapter = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoInput!, sourcePixelBufferAttributes: rgbAttributes) 621 | 622 | // Update the audio delegate with the new audioWriterInput 623 | self.audioCaptureDelegate = AudioCaptureDelegate(writerInput: audioInput!) 624 | 625 | // Attach the new delegate to the existing AVCaptureAudioDataOutput 626 | if let audioOutput = self.audioSession.outputs.first(where: { $0 is AVCaptureAudioDataOutput }) as? AVCaptureAudioDataOutput { 627 | let audioQueue = DispatchQueue(label: "AudioProcessingQueue") 628 | audioOutput.setSampleBufferDelegate(self.audioCaptureDelegate, queue: audioQueue) 629 | } 630 | 631 | if self.depthStatus.isDepthAvailable { 632 | setupDepthRecording(depthVideoURL: depthVideoURL) 633 | } 634 | 635 | self.poseFileHandle = try FileHandle(forWritingTo: poseTextURL) 636 | try poseFileHandle?.seekToEnd() 637 | } catch { 638 | print("Failed to setup recording: \(error)") 639 | } 640 | 641 | return RecordingFiles( 642 | rgbFileName: rgbVideoURL, 643 | depthFileName: depthVideoURL, 644 | timestamp: timestamp, 645 | rgbImagesDirectory: rgbImagesDirectory, 646 | depthImagesDirectory: depthImagesDirectory, 647 | poseFile: poseTextURL, 648 | generalDataDirectory: timestamp, 649 | tactileFile: tactileFileURL 650 | ) 651 | } 652 | 653 | private func setupDepthRecording(depthVideoURL: URL) { 654 | do { 655 | depthAssetWriter = try AVAssetWriter(outputURL: depthVideoURL, fileType: .mp4) 656 | let depthVideoSettings: [String: Any] = [ 657 | AVVideoCodecKey: AVVideoCodecType.h264, 658 | AVVideoWidthKey: Int(depthViewPortSize.width), 659 | AVVideoHeightKey: Int(depthViewPortSize.height), 660 | AVVideoScalingModeKey: AVVideoScalingModeResizeAspectFill, 661 | ] 662 | depthVideoInput = AVAssetWriterInput(mediaType: .video, outputSettings: depthVideoSettings) 663 | depthVideoInput?.expectsMediaDataInRealTime = true 664 | depthAssetWriter?.add(depthVideoInput!) 665 | 666 | let recordingDepthAttributes: [String: Any] = [ 667 | kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_OneComponent8, 668 | kCVPixelBufferWidthKey as String: Int(self.depthViewPortSize.width), 669 | kCVPixelBufferHeightKey as String: Int(self.depthViewPortSize.height) 670 | ] 671 | 672 | depthPixelBufferAdapter = AVAssetWriterInputPixelBufferAdaptor( 673 | assetWriterInput: depthVideoInput!, 674 | sourcePixelBufferAttributes: recordingDepthAttributes 675 | ) 676 | } catch { 677 | print("Failed to setup depth recording: \(error)") 678 | } 679 | } 680 | 681 | private func processRGBCaptureData(rgbPixelBuffer: CVPixelBuffer, cropRect: CGRect, currentTime: CMTime) -> Bool { 682 | guard let videoInput = self.videoInput, videoInput.isReadyForMoreMediaData else { return false } 683 | guard let outputPixelBufferPool = self.pixelBufferAdapter?.pixelBufferPool else { return false } 684 | 685 | var outputPixelBuffer: CVPixelBuffer? 686 | let status = CVPixelBufferPoolCreatePixelBuffer(nil, outputPixelBufferPool, &outputPixelBuffer) 687 | guard status == kCVReturnSuccess, let outputBuffer = outputPixelBuffer else { return false } 688 | 689 | CVPixelBufferLockBaseAddress(rgbPixelBuffer, .readOnly) 690 | CVPixelBufferLockBaseAddress(outputBuffer, []) 691 | 692 | let ciImage = CIImage(cvPixelBuffer: rgbPixelBuffer) 693 | let transformedImage = ciImage.transformed(by: self.combinedRGBTransform!) //.cropped(to: cropRect) 694 | self.ciContext.render(transformedImage, to: outputBuffer, bounds: cropRect, colorSpace: CGColorSpaceCreateDeviceRGB()) 695 | 696 | guard let pixelBufferAdapter = self.pixelBufferAdapter else { 697 | print("Failed to append RGB pixel buffer: Pixel buffer adapter is nil.") 698 | return false 699 | } 700 | 701 | if !pixelBufferAdapter.append(outputBuffer, withPresentationTime: currentTime) { 702 | let isReady = pixelBufferAdapter.assetWriterInput.isReadyForMoreMediaData 703 | let writerError = self.assetWriter?.error?.localizedDescription ?? "Unknown asset writer error." 704 | print("Failed to append RGB pixel buffer. Adapter state: \(isReady), Time: \(currentTime), Error: \(writerError)") 705 | return false 706 | } 707 | CVPixelBufferUnlockBaseAddress(outputBuffer, []) 708 | CVPixelBufferUnlockBaseAddress(rgbPixelBuffer, .readOnly) 709 | return true 710 | } 711 | 712 | private func processDepthCaptureData(depthPixelBuffer: CVPixelBuffer?, cropRect: CGRect, currentTime: CMTime) -> Bool { 713 | guard let depthVideoInput = self.depthVideoInput, depthVideoInput.isReadyForMoreMediaData else { return false } 714 | guard let depthPixelBuffer = depthPixelBuffer else { return false } 715 | guard let pixelBufferPool = self.depthPixelBufferAdapter?.pixelBufferPool else { 716 | print("Depth pixel buffer pool is nil.") 717 | return false 718 | } 719 | 720 | var outputPixelBuffer: CVPixelBuffer? 721 | let status = CVPixelBufferPoolCreatePixelBuffer(nil, pixelBufferPool, &outputPixelBuffer) 722 | guard status == kCVReturnSuccess, let depthOutputBuffer = outputPixelBuffer else { 723 | print("Unable to create output pixel buffer for depth.") 724 | return false 725 | } 726 | 727 | CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly) 728 | CVPixelBufferLockBaseAddress(depthOutputBuffer, []) 729 | 730 | self.saveBinaryDepthData(depthPixelBuffer: depthPixelBuffer) 731 | 732 | let ciImage = CIImage(cvPixelBuffer: depthPixelBuffer) 733 | let processedDepthImage = self.applyDepthFilters(ciImage: ciImage) 734 | 735 | self.ciContext.render( 736 | processedDepthImage, 737 | to: depthOutputBuffer, 738 | bounds: cropRect, 739 | colorSpace: self.isColorMapOpened ? CGColorSpaceCreateDeviceRGB() : CGColorSpaceCreateDeviceGray() 740 | ) 741 | 742 | guard let depthPixelBufferAdapter = self.depthPixelBufferAdapter else { 743 | print("Failed to append depth pixel buffer: Pixel buffer adapter is nil.") 744 | return false 745 | } 746 | 747 | if !depthPixelBufferAdapter.append(depthOutputBuffer, withPresentationTime: currentTime) { 748 | let isReady = depthPixelBufferAdapter.assetWriterInput.isReadyForMoreMediaData 749 | let writerError = self.depthAssetWriter?.error?.localizedDescription ?? "Unknown asset writer error." 750 | print("❌ Failed to append RGB pixel buffer. Adapter state: \(isReady), Time: \(currentTime), Error: \(writerError)") 751 | return false 752 | } 753 | CVPixelBufferUnlockBaseAddress(depthOutputBuffer, []) 754 | CVPixelBufferUnlockBaseAddress(depthPixelBuffer, .readOnly) 755 | return true 756 | } 757 | 758 | private func processPoseData(frame: ARFrame) { 759 | let cameraTransform = frame.camera.transform 760 | // Transform the orientation matrix to unit quaternion 761 | let quaternion = simd_quaternion(cameraTransform) 762 | 763 | let timestamp = Int64(Date().timeIntervalSince1970 * 1000) 764 | 765 | let poseValues: [Float] = [ 766 | quaternion.vector.x, quaternion.vector.y, quaternion.vector.z, quaternion.vector.w, 767 | cameraTransform.columns.3.x, cameraTransform.columns.3.y, cameraTransform.columns.3.z 768 | ] 769 | let poseString = "\"<\(timestamp)>\" ," + poseValues.map { String($0) }.joined(separator: ",") + "\n" 770 | 771 | do { 772 | if let data = poseString.data(using: .utf8) { 773 | try self.poseFileHandle?.write(contentsOf: data) 774 | } 775 | } catch { 776 | print("❌ Error writing pose data: \(error)") 777 | } 778 | } 779 | 780 | private func applyDepthFilters(ciImage: CIImage) -> CIImage { 781 | var filteredImage = ciImage 782 | 783 | let depthFilter = CIFilter(name: "CIColorControls")! 784 | depthFilter.setValue(ciImage, forKey: kCIInputImageKey) 785 | depthFilter.setValue(2.0, forKey: kCIInputSaturationKey) // Keep saturation 786 | depthFilter.setValue(0.0, forKey: kCIInputBrightnessKey) // Adjust brightness 787 | depthFilter.setValue(3.0, forKey: kCIInputContrastKey) // Increase contrast for clarity 788 | 789 | if let outputImage = depthFilter.outputImage { 790 | filteredImage = outputImage 791 | } else { 792 | print("❌ Failed to apply color controls filter to depth image.") 793 | } 794 | 795 | if(self.isColorMapOpened){ 796 | let falseColorFilter = CIFilter.falseColor() 797 | falseColorFilter.color0 = CIColor(red: 1, green: 1, blue: 0) 798 | falseColorFilter.color1 = CIColor(red: 0, green: 0, blue: 1) 799 | falseColorFilter.inputImage = filteredImage 800 | if let outputImage = falseColorFilter.outputImage { 801 | filteredImage = outputImage 802 | } else { 803 | print("❌ Failed to apply false color filter to depth image.") 804 | } 805 | } 806 | return filteredImage.transformed(by: self.combinedDepthTransform!) //.cropped(to: cropRect) 807 | } 808 | 809 | private func saveBinaryDepthData(depthPixelBuffer: CVPixelBuffer) { 810 | // Save metric depth data as binary file 811 | let width = CVPixelBufferGetWidth(depthPixelBuffer) 812 | let height = CVPixelBufferGetHeight(depthPixelBuffer) 813 | 814 | guard let baseAddress = CVPixelBufferGetBaseAddress(depthPixelBuffer) else { 815 | CVPixelBufferUnlockBaseAddress(depthPixelBuffer, .readOnly) 816 | return 817 | } 818 | 819 | let floatBuffer = baseAddress.assumingMemoryBound(to: Float32.self) 820 | let dataSize = width * height * MemoryLayout.size 821 | let data = Data(bytes: floatBuffer, count: dataSize) 822 | 823 | // Save binary data to a file 824 | let fileURL = self.depthDirect.appendingPathComponent("\(Int64(Date().timeIntervalSince1970*1000)).bin") 825 | do { 826 | try data.write(to: fileURL) 827 | } catch { 828 | print("Error saving binary file: \(error)") 829 | } 830 | } 831 | 832 | func captureVideoFrame() { 833 | 834 | guard let currentFrame = session.currentFrame else {return} 835 | 836 | var imgSuccessFlag = true 837 | 838 | let currentTime = CMTimeMake(value: Int64(CACurrentMediaTime() * 1000), timescale: 1000) 839 | 840 | let rgbPixelBuffer = currentFrame.capturedImage 841 | var depthPixelBuffer: CVPixelBuffer? 842 | 843 | if self.depthStatus.isDepthAvailable { 844 | guard let depthBuffer = currentFrame.sceneDepth?.depthMap else { return } 845 | depthPixelBuffer = depthBuffer 846 | } 847 | 848 | let cropRect = CGRect( 849 | x: 0, y: 0, width: self.viewPortSize.width, height: self.viewPortSize.height 850 | ) 851 | let depthCropRect = CGRect( 852 | x: 0, y: 0, width: self.depthViewPortSize.width, height: self.depthViewPortSize.height 853 | ) 854 | 855 | DispatchQueue.global(qos: .userInitiated).async { 856 | let rgbSuccess = self.processRGBCaptureData(rgbPixelBuffer: rgbPixelBuffer, cropRect: cropRect, currentTime: currentTime) 857 | imgSuccessFlag = imgSuccessFlag && rgbSuccess 858 | if self.depthStatus.isDepthAvailable && imgSuccessFlag { 859 | let depthSuccess = self.processDepthCaptureData(depthPixelBuffer: depthPixelBuffer, cropRect: depthCropRect, currentTime: currentTime) 860 | imgSuccessFlag = imgSuccessFlag && depthSuccess 861 | } 862 | if imgSuccessFlag { 863 | self.processPoseData(frame: currentFrame) 864 | } 865 | } 866 | } 867 | 868 | func getDocumentsDirect() -> URL{ 869 | let paths = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask) 870 | print(paths[0].path) 871 | return paths[0] 872 | } 873 | 874 | } 875 | 876 | class AudioCaptureDelegate: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate { 877 | private let writerInput: AVAssetWriterInput? 878 | 879 | init(writerInput: AVAssetWriterInput) { 880 | self.writerInput = writerInput 881 | } 882 | 883 | func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) { 884 | // Append audio sample buffer to the writer input 885 | guard writerInput?.isReadyForMoreMediaData == true else { 886 | print("Not ready") 887 | return 888 | } 889 | writerInput?.append(sampleBuffer) 890 | } 891 | } 892 | --------------------------------------------------------------------------------