├── .clang-format ├── screenshot.sh ├── vvv.sh ├── inject.sh ├── README.md └── visionos_stereo_screenshots.m /.clang-format: -------------------------------------------------------------------------------- 1 | BasedOnStyle: Google 2 | -------------------------------------------------------------------------------- /screenshot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec xcrun simctl spawn booted launchctl kill USR1 user/$UID/com.apple.backboardd 3 | -------------------------------------------------------------------------------- /vvv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | rsync -a . mini2:~/Documents/stereoscreenshots 4 | ssh mini2 "cd ~/Documents/stereoscreenshots && bash build.sh" 5 | -------------------------------------------------------------------------------- /inject.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | xcrun simctl spawn booted launchctl debug user/$UID/com.apple.backboardd --environment DYLD_INSERT_LIBRARIES=$PWD/libvisionos_stereo_screenshots.dylib 4 | xcrun simctl spawn booted launchctl kill TERM user/$UID/com.apple.backboardd 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Take stereoscopic (3D) screenshots in the visionOS simulator. 2 | 3 | ![example screenshot](https://github.com/zhuowei/VisionOSStereoScreenshots/assets/704768/c9945210-eaf8-4a59-90da-5a0787b25598) 4 | 5 | An example screenshot from the visionOS simulator in side-by-side stereo. 6 | 7 | Tested on macOS 14 beta 2 / Xcode 15 beta 2. 8 | 9 | ## Setup 10 | 11 | ### Non-Metal Immersive apps 12 | 13 | ``` 14 | ./build.sh 15 | ./inject.sh 16 | # this resprings the simulator 17 | ``` 18 | 19 | ### Metal Immersive (CompositorService) apps 20 | 21 | TODO 22 | 23 | ## Usage 24 | 25 | ### Non-Metal Immersive apps 26 | 27 | ``` 28 | ./screenshot.sh 29 | ``` 30 | 31 | Screenshots are saved in `/tmp/visionos_stereo_screenshot_{time}.png`. 32 | 33 | ### How it works 34 | 35 | This hooks CompositorService to give backboardd an extra right eye view to render. 36 | -------------------------------------------------------------------------------- /visionos_stereo_screenshots.m: -------------------------------------------------------------------------------- 1 | @import CompositorServices; 2 | @import Darwin; 3 | @import ObjectiveC; 4 | @import UniformTypeIdentifiers; 5 | 6 | #define DYLD_INTERPOSE(_replacement, _replacee) \ 7 | __attribute__((used)) static struct { \ 8 | const void* replacement; \ 9 | const void* replacee; \ 10 | } _interpose_##_replacee __attribute__((section("__DATA,__interpose,interposing"))) = { \ 11 | (const void*)(unsigned long)&_replacement, (const void*)(unsigned long)&_replacee} 12 | 13 | // 6.5cm translation on x 14 | static simd_float4x4 gRightEyeMatrix = {.columns = { 15 | {1, 0, 0, 0}, 16 | {0, 1, 0, 0}, 17 | {0, 0, 1, 0}, 18 | {0.065, 0, 0, 1}, 19 | }}; 20 | 21 | // cp_drawable_get_view 22 | struct cp_view { 23 | simd_float4x4 transform; // 0x0 24 | char unknown[0x110 - 0x40]; // 0x40 25 | }; 26 | static_assert(sizeof(struct cp_view) == 0x110, "cp_view size is wrong"); 27 | 28 | // cp_view_texture_map_get_texture_index 29 | struct cp_view_texture_map { 30 | size_t texture_index; // 0x0 31 | size_t slice_index; // 0x8 32 | MTLViewport viewport; // 0x10 33 | }; 34 | 35 | static const int kTakeScreenshotStatusIdle = 0; 36 | static const int kTakeScreenshotStatusScreenshotNextFrame = 1; 37 | static const int kTakeScreenshotStatusScreenshotInProgress = 2; 38 | 39 | // TODO(zhuowei): do I need locking for this? 40 | static int gTakeScreenshotStatus = kTakeScreenshotStatusIdle; 41 | 42 | // TODO(zhuowei): multiple screenshots in flight 43 | static cp_drawable_t gHookedDrawable; 44 | 45 | static id gHookedExtraTexture = nil; 46 | static id gHookedExtraDepthTexture = nil; 47 | static id gHookedExtraScrapTexture = nil; 48 | static id gHookedExtraScrapDepthTexture = nil; 49 | static id gHookedRealTexture = nil; 50 | 51 | static void DumpScreenshot(void); 52 | 53 | static id MakeOurTextureBasedOnTheirTexture(id device, 54 | id originalTexture) { 55 | MTLTextureDescriptor* descriptor = 56 | [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:originalTexture.pixelFormat 57 | width:originalTexture.width 58 | height:originalTexture.height 59 | mipmapped:false]; 60 | descriptor.storageMode = originalTexture.storageMode; 61 | return [device newTextureWithDescriptor:descriptor]; 62 | } 63 | 64 | static cp_drawable_t hook_cp_frame_query_drawable(cp_frame_t frame) { 65 | cp_drawable_t retval = cp_frame_query_drawable(frame); 66 | gHookedDrawable = nil; 67 | if (!gHookedExtraTexture) { 68 | // only make this once 69 | id metalDevice = MTLCreateSystemDefaultDevice(); 70 | id originalTexture = cp_drawable_get_color_texture(retval, 0); 71 | id originalDepthTexture = cp_drawable_get_depth_texture(retval, 0); 72 | gHookedExtraTexture = MakeOurTextureBasedOnTheirTexture(metalDevice, originalTexture); 73 | gHookedExtraDepthTexture = MakeOurTextureBasedOnTheirTexture(metalDevice, originalDepthTexture); 74 | gHookedExtraScrapTexture = MakeOurTextureBasedOnTheirTexture(metalDevice, originalTexture); 75 | gHookedExtraScrapDepthTexture = 76 | MakeOurTextureBasedOnTheirTexture(metalDevice, originalDepthTexture); 77 | } 78 | if (gTakeScreenshotStatus == kTakeScreenshotStatusScreenshotNextFrame) { 79 | gTakeScreenshotStatus = kTakeScreenshotStatusScreenshotInProgress; 80 | gHookedDrawable = retval; 81 | gHookedRealTexture = cp_drawable_get_color_texture(retval, 0); 82 | NSLog(@"visionos_stereo_screenshots starting screenshot!"); 83 | } 84 | cp_view_t leftView = cp_drawable_get_view(retval, 0); 85 | cp_view_t rightView = cp_drawable_get_view(retval, 1); 86 | memcpy(rightView, leftView, sizeof(*leftView)); 87 | rightView->transform = gRightEyeMatrix; 88 | cp_view_get_view_texture_map(rightView)->texture_index = 1; 89 | return retval; 90 | } 91 | 92 | DYLD_INTERPOSE(hook_cp_frame_query_drawable, cp_frame_query_drawable); 93 | 94 | static void hook_cp_drawable_encode_present(cp_drawable_t drawable, 95 | id command_buffer) { 96 | if (gHookedDrawable == drawable) { 97 | [command_buffer addCompletedHandler:^(id buffer) { 98 | DumpScreenshot(); 99 | }]; 100 | } 101 | return cp_drawable_encode_present(drawable, command_buffer); 102 | } 103 | 104 | DYLD_INTERPOSE(hook_cp_drawable_encode_present, cp_drawable_encode_present); 105 | 106 | static size_t hook_cp_drawable_get_view_count(cp_drawable_t drawable) { return 2; } 107 | 108 | DYLD_INTERPOSE(hook_cp_drawable_get_view_count, cp_drawable_get_view_count); 109 | 110 | static size_t hook_cp_drawable_get_texture_count(cp_drawable_t drawable) { return 2; } 111 | 112 | DYLD_INTERPOSE(hook_cp_drawable_get_texture_count, cp_drawable_get_texture_count); 113 | 114 | static id hook_cp_drawable_get_color_texture(cp_drawable_t drawable, size_t index) { 115 | if (index == 1) { 116 | return drawable == gHookedDrawable ? gHookedExtraTexture : gHookedExtraScrapTexture; 117 | } 118 | return cp_drawable_get_color_texture(drawable, 0); 119 | } 120 | 121 | DYLD_INTERPOSE(hook_cp_drawable_get_color_texture, cp_drawable_get_color_texture); 122 | 123 | static id hook_cp_drawable_get_depth_texture(cp_drawable_t drawable, size_t index) { 124 | if (index == 1) { 125 | return drawable == gHookedDrawable ? gHookedExtraDepthTexture : gHookedExtraScrapDepthTexture; 126 | } 127 | return cp_drawable_get_depth_texture(drawable, 0); 128 | } 129 | 130 | DYLD_INTERPOSE(hook_cp_drawable_get_depth_texture, cp_drawable_get_depth_texture); 131 | 132 | size_t cp_layer_properties_get_view_count(cp_layer_renderer_properties_t properties); 133 | 134 | static size_t hook_cp_layer_properties_get_view_count(cp_layer_renderer_properties_t properties) { 135 | return 2; 136 | } 137 | 138 | DYLD_INTERPOSE(hook_cp_layer_properties_get_view_count, cp_layer_properties_get_view_count); 139 | 140 | cp_layer_renderer_layout cp_layer_configuration_get_layout_private( 141 | cp_layer_renderer_configuration_t configuration); 142 | 143 | static cp_layer_renderer_layout hook_cp_layer_configuration_get_layout_private( 144 | cp_layer_renderer_configuration_t configuration) { 145 | return cp_layer_renderer_layout_dedicated; 146 | } 147 | 148 | DYLD_INTERPOSE(hook_cp_layer_configuration_get_layout_private, 149 | cp_layer_configuration_get_layout_private); 150 | 151 | static void DumpScreenshot() { 152 | NSLog(@"visionos_stereo_screenshot: DumpScreenshot"); 153 | gTakeScreenshotStatus = kTakeScreenshotStatusIdle; 154 | 155 | size_t textureDataSize = gHookedExtraTexture.width * gHookedExtraTexture.height * 4; 156 | NSMutableData* outputData = [NSMutableData dataWithLength:textureDataSize]; 157 | [gHookedRealTexture 158 | getBytes:outputData.mutableBytes 159 | bytesPerRow:gHookedRealTexture.width * 4 160 | bytesPerImage:textureDataSize 161 | fromRegion:MTLRegionMake2D(0, 0, gHookedRealTexture.width, gHookedRealTexture.height) 162 | mipmapLevel:0 163 | slice:0]; 164 | CGColorSpaceRef colorSpace = CGColorSpaceCreateWithName(kCGColorSpaceSRGB); 165 | CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)outputData); 166 | CGImageRef cgImage = CGImageCreate( 167 | gHookedRealTexture.width, gHookedRealTexture.height, /*bitsPerComponent=*/8, 168 | /*bitsPerPixel=*/32, /*bytesPerRow=*/gHookedRealTexture.width * 4, colorSpace, 169 | kCGImageByteOrder32Little | kCGImageAlphaPremultipliedFirst, provider, /*decode=*/nil, 170 | /*shouldInterpolate=*/false, /*intent=*/kCGRenderingIntentDefault); 171 | 172 | NSMutableData* outputData2 = [NSMutableData dataWithLength:textureDataSize]; 173 | [gHookedExtraTexture 174 | getBytes:outputData2.mutableBytes 175 | bytesPerRow:gHookedExtraTexture.width * 4 176 | bytesPerImage:textureDataSize 177 | fromRegion:MTLRegionMake2D(0, 0, gHookedExtraTexture.width, gHookedExtraTexture.height) 178 | mipmapLevel:0 179 | slice:0]; 180 | CGDataProviderRef provider2 = CGDataProviderCreateWithCFData((__bridge CFDataRef)outputData2); 181 | CGImageRef cgImage2 = CGImageCreate( 182 | gHookedExtraTexture.width, gHookedExtraTexture.height, /*bitsPerComponent=*/8, 183 | /*bitsPerPixel=*/32, /*bytesPerRow=*/gHookedExtraTexture.width * 4, colorSpace, 184 | kCGImageByteOrder32Little | kCGImageAlphaPremultipliedFirst, provider2, /*decode=*/nil, 185 | /*shouldInterpolate=*/false, /*intent=*/kCGRenderingIntentDefault); 186 | 187 | CGContextRef cgContext = CGBitmapContextCreate( 188 | nil, gHookedExtraTexture.width * 2, gHookedExtraTexture.height, /*bitsPerComponent=*/8, 189 | /*bytesPerRow=*/0, colorSpace, kCGImageByteOrder32Little | kCGImageAlphaPremultipliedFirst); 190 | CGContextDrawImage( 191 | cgContext, CGRectMake(0, 0, gHookedRealTexture.width, gHookedRealTexture.height), cgImage); 192 | CGContextDrawImage(cgContext, 193 | CGRectMake(gHookedRealTexture.width, 0, gHookedExtraTexture.width, 194 | gHookedExtraTexture.height), 195 | cgImage2); 196 | CGImageRef outputImage = CGBitmapContextCreateImage(cgContext); 197 | 198 | NSString* filePath = 199 | [NSString stringWithFormat:@"/tmp/visionos_stereo_screenshot_%ld.png", time(nil)]; 200 | ; 201 | 202 | CGImageDestinationRef destination = 203 | CGImageDestinationCreateWithURL((__bridge CFURLRef)[NSURL fileURLWithPath:filePath], 204 | (__bridge CFStringRef)UTTypePNG.identifier, 1, nil); 205 | CGImageDestinationAddImage(destination, outputImage, nil); 206 | bool success = CGImageDestinationFinalize(destination); 207 | 208 | if (success) { 209 | NSLog(@"visionos_stereo_screenshots wrote screenshot to %@", filePath); 210 | } else { 211 | NSLog(@"visionos_stereo_screenshots failed to write screenshot to %@", filePath); 212 | } 213 | 214 | CFRelease(destination); 215 | CFRelease(outputImage); 216 | CFRelease(cgContext); 217 | CFRelease(cgImage); 218 | CFRelease(cgImage2); 219 | CFRelease(colorSpace); 220 | CFRelease(provider); 221 | CFRelease(provider2); 222 | } 223 | 224 | __attribute__((constructor)) static void SetupSignalHandler() { 225 | NSLog(@"visionos_stereo_screenshots starting!"); 226 | static dispatch_queue_t signal_queue; 227 | static dispatch_source_t signal_source; 228 | signal_queue = dispatch_queue_create("com.worthdoingbadly.stereoscreenshots.signalqueue", 229 | DISPATCH_QUEUE_SERIAL); 230 | signal_source = 231 | dispatch_source_create(DISPATCH_SOURCE_TYPE_SIGNAL, SIGUSR1, /*mask=*/0, signal_queue); 232 | dispatch_source_set_event_handler(signal_source, ^{ 233 | if (gTakeScreenshotStatus == kTakeScreenshotStatusIdle) { 234 | gTakeScreenshotStatus = kTakeScreenshotStatusScreenshotNextFrame; 235 | NSLog(@"visionos_stereo_screenshots preparing to take screenshot!"); 236 | } 237 | }); 238 | signal(SIGUSR1, SIG_IGN); 239 | dispatch_activate(signal_source); 240 | } 241 | --------------------------------------------------------------------------------