Merge pull request #128 from filleduchaos/main

fix yielded presentation timestamp on macOS

Author: Brendonovich

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "[rust]": {
+        "editor.formatOnSave": true
+    }
+}

src/capturer/engine/mac/apple_sys.rs

@@ -1,3 +1,5 @@
+pub use screencapturekit_sys::os_types::base::*;
+
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct __CFDictionary {
@@ -35,6 +37,7 @@ extern "C" {
         theType: CFNumberType,
         valuePtr: *mut ::std::os::raw::c_void,
     ) -> Boolean;
+    pub fn CMTimeGetSeconds(time: CMTime) -> Float64;
     pub static SCStreamFrameInfoStatus: SCStreamFrameInfo;
 }
 pub const CFNumberType_kCFNumberSInt64Type: CFNumberType = 4;

src/capturer/engine/mac/mod.rs

@@ -14,10 +14,13 @@ use screencapturekit::{
 use screencapturekit_sys::os_types::base::{CMTime, CMTimeScale};
 use screencapturekit_sys::os_types::geometry::{CGPoint, CGRect, CGSize};
 
-use crate::{capturer::{Area, Options, Point, Size}, frame::BGRAFrame};
 use crate::frame::{Frame, FrameType};
 use crate::targets::Target;
 use crate::{capturer::Resolution, targets};
+use crate::{
+    capturer::{Area, Options, Point, Size},
+    frame::BGRAFrame,
+};
 
 mod apple_sys;
 mod pixelformat;
@@ -72,18 +75,18 @@ impl StreamOutput for Capturer {
                         // Quick hack - just send an empty frame, and the caller can figure out how to handle it
                         match self.output_type {
                             FrameType::BGRAFrame => {
-                                let display_time = sample.sys_ref.get_presentation_timestamp().value as u64;
+                                let display_time = pixelformat::get_pts_in_nanoseconds(&sample);
                                 let frame = BGRAFrame {
                                     display_time,
                                     width: 0,
                                     height: 0,
                                     data: vec![],
                                 };
                                 self.tx.send(Frame::BGRA(frame)).unwrap_or(());
-                            },
+                            }
                             _ => {}
                         }
-                    },
+                    }
                     _ => {}
                 }
             }

src/capturer/engine/mac/pixelformat.rs

@@ -17,6 +17,16 @@ use core_video_sys::{
     CVPixelBufferLockBaseAddress, CVPixelBufferRef, CVPixelBufferUnlockBaseAddress,
 };
 
+// Returns a frame's presentation timestamp in nanoseconds since an arbitrary start time.
+// This is typically yielded from a monotonic clock started on system boot.
+pub fn get_pts_in_nanoseconds(sample_buffer: &CMSampleBuffer) -> u64 {
+    let pts = sample_buffer.sys_ref.get_presentation_timestamp();
+
+    let seconds = unsafe { CMTimeGetSeconds(pts) };
+
+    (seconds * 1_000_000_000.).trunc() as u64
+}
+
 pub unsafe fn create_yuv_frame(sample_buffer: CMSampleBuffer) -> Option<YUVFrame> {
     // Check that the frame status is complete
     let buffer_ref = &(*sample_buffer.sys_ref);
@@ -47,8 +57,7 @@ pub unsafe fn create_yuv_frame(sample_buffer: CMSampleBuffer) -> Option<YUVFrame
         }
     }
 
-    //let epoch = CMSampleBufferGetPresentationTimeStamp(buffer_ref).epoch;
-    let epoch = sample_buffer.sys_ref.get_presentation_timestamp().value;
+    let display_time = get_pts_in_nanoseconds(&sample_buffer);
     let pixel_buffer = CMSampleBufferGetImageBuffer(buffer_ref) as CVPixelBufferRef;
 
     CVPixelBufferLockBaseAddress(pixel_buffer, 0);
@@ -78,7 +87,7 @@ pub unsafe fn create_yuv_frame(sample_buffer: CMSampleBuffer) -> Option<YUVFrame
     CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
 
     YUVFrame {
-        display_time: epoch as u64,
+        display_time,
         width: width as i32,
         height: height as i32,
         luminance_bytes,
@@ -90,8 +99,8 @@ pub unsafe fn create_yuv_frame(sample_buffer: CMSampleBuffer) -> Option<YUVFrame
 }
 
 pub unsafe fn create_bgr_frame(sample_buffer: CMSampleBuffer) -> Option<BGRFrame> {
+    let display_time = get_pts_in_nanoseconds(&sample_buffer);
     let buffer_ref = &(*sample_buffer.sys_ref);
-    let epoch = sample_buffer.sys_ref.get_presentation_timestamp().value;
     let pixel_buffer = CMSampleBufferGetImageBuffer(buffer_ref) as CVPixelBufferRef;
 
     CVPixelBufferLockBaseAddress(pixel_buffer, 0);
@@ -117,16 +126,16 @@ pub unsafe fn create_bgr_frame(sample_buffer: CMSampleBuffer) -> Option<BGRFrame
     CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
 
     Some(BGRFrame {
-        display_time: epoch as u64,
+        display_time,
         width: width as i32, // width does not give accurate results - https://stackoverflow.com/questions/19587185/cvpixelbuffergetbytesperrow-for-cvimagebufferref-returns-unexpected-wrong-valu
         height: height as i32,
         data: remove_alpha_channel(cropped_data),
     })
 }
 
 pub unsafe fn create_bgra_frame(sample_buffer: CMSampleBuffer) -> Option<BGRAFrame> {
+    let display_time = get_pts_in_nanoseconds(&sample_buffer);
     let buffer_ref = &(*sample_buffer.sys_ref);
-    let epoch = sample_buffer.sys_ref.get_presentation_timestamp().value;
     let pixel_buffer = CMSampleBufferGetImageBuffer(buffer_ref) as CVPixelBufferRef;
 
     CVPixelBufferLockBaseAddress(pixel_buffer, 0);
@@ -149,16 +158,16 @@ pub unsafe fn create_bgra_frame(sample_buffer: CMSampleBuffer) -> Option<BGRAFra
     CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
 
     Some(BGRAFrame {
-        display_time: epoch as u64,
+        display_time,
         width: width as i32, // width does not give accurate results - https://stackoverflow.com/questions/19587185/cvpixelbuffergetbytesperrow-for-cvimagebufferref-returns-unexpected-wrong-valu
         height: height as i32,
         data,
     })
 }
 
 pub unsafe fn create_rgb_frame(sample_buffer: CMSampleBuffer) -> Option<RGBFrame> {
+    let display_time = get_pts_in_nanoseconds(&sample_buffer);
     let buffer_ref = &(*sample_buffer.sys_ref);
-    let epoch = sample_buffer.sys_ref.get_presentation_timestamp().value;
     let pixel_buffer = CMSampleBufferGetImageBuffer(buffer_ref) as CVPixelBufferRef;
 
     CVPixelBufferLockBaseAddress(pixel_buffer, 0);
@@ -184,7 +193,7 @@ pub unsafe fn create_rgb_frame(sample_buffer: CMSampleBuffer) -> Option<RGBFrame
     CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
 
     Some(RGBFrame {
-        display_time: epoch as u64,
+        display_time,
         width: width as i32, // width does not give accurate results - https://stackoverflow.com/questions/19587185/cvpixelbuffergetbytesperrow-for-cvimagebufferref-returns-unexpected-wrong-valu
         height: height as i32,
         data: convert_bgra_to_rgb(cropped_data),

src/capturer/engine/mod.rs

@@ -12,6 +12,24 @@ mod win;
 #[cfg(target_os = "linux")]
 mod linux;
 
+pub fn get_output_frame_size(options: &Options) -> [u32; 2] {
+    #[cfg(target_os = "macos")]
+    {
+        mac::get_output_frame_size(options)
+    }
+
+    #[cfg(target_os = "windows")]
+    {
+        win::get_output_frame_size(options)
+    }
+
+    #[cfg(target_os = "linux")]
+    {
+        // TODO: How to calculate this on Linux?
+        return [0, 0];
+    }
+}
+
 pub struct Engine {
     options: Options,
     #[cfg(target_os = "macos")]
@@ -90,19 +108,6 @@ impl Engine {
     }
 
     pub fn get_output_frame_size(&mut self) -> [u32; 2] {
-        #[cfg(target_os = "macos")]
-        {
-            mac::get_output_frame_size(&self.options)
-        }
-
-        #[cfg(target_os = "windows")]
-        {
-            win::get_output_frame_size(&self.options)
-        }
-
-        #[cfg(target_os = "linux")]
-        {
-            return [0, 0];
-        }
+        get_output_frame_size(&self.options)
     }
 }

src/capturer/mod.rs

@@ -8,6 +8,8 @@ use crate::{
     targets::Target,
 };
 
+pub use engine::get_output_frame_size;
+
 #[derive(Debug, Clone, Copy, Default)]
 pub enum Resolution {
     _480p,