Auto merge of #303 - ceyusa:external-oes-texture, r=ferjm

Handling external OES textures

Fix #291

Author: bors-servo

Cargo.lock

@@ -598,7 +598,7 @@ impl GStreamerPlayer {
                         let frame = render
                             .lock()
                             .unwrap()
-                            .get_frame_from_sample(&sample)
+                            .get_frame_from_sample(sample)
                             .or_else(|_| Err(gst::FlowError::Error))?;
                         renderer.lock().unwrap().render(frame);
                         notify!(observer, PlayerEvent::FrameUpdated);

backends/gstreamer/player.rs

@@ -4,7 +4,7 @@
 //!
 //! Internally it uses GStreamer's *glsinkbin* element as *videosink*
 //! wrapping the *appsink* from the Player. And the shared frames are
-//! mapped as texuture IDs.
+//! mapped as texture IDs.
 
 #![cfg(any(
     target_os = "linux",
@@ -31,6 +31,7 @@ use sm_player::PlayerError;
 use std::sync::{Arc, Mutex};
 
 struct GStreamerBuffer {
+    is_external_oes: bool,
     frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
 }
 
@@ -39,7 +40,11 @@ impl Buffer for GStreamerBuffer {
         // packed formats are guaranteed to be in a single plane
         if self.frame.format() == gst_video::VideoFormat::Rgba {
             let tex_id = self.frame.get_texture_id(0).ok_or_else(|| ())?;
-            Ok(FrameData::Texture(tex_id))
+            Ok(if self.is_external_oes {
+                FrameData::OESTexture(tex_id)
+            } else {
+                FrameData::Texture(tex_id)
+            })
         } else {
             Err(())
         }
@@ -170,7 +175,7 @@ impl Render for RenderUnix {
         true
     }
 
-    fn build_frame(&self, buffer: gst::Buffer, info: gst_video::VideoInfo) -> Result<Frame, ()> {
+    fn build_frame(&self, sample: gst::Sample) -> Result<Frame, ()> {
         if self.gst_context.lock().unwrap().is_none() && self.gl_upload.lock().unwrap().is_some() {
             *self.gst_context.lock().unwrap() =
                 if let Some(glupload) = self.gl_upload.lock().unwrap().as_ref() {
@@ -183,13 +188,33 @@ impl Render for RenderUnix {
                 };
         }
 
+        let buffer = sample.get_buffer_owned().ok_or_else(|| ())?;
+        let caps = sample.get_caps().ok_or_else(|| ())?;
+
+        let is_external_oes = caps
+            .get_structure(0)
+            .and_then(|s| {
+                s.get::<&str>("texture-target").and_then(|target| {
+                    if target == "external-oes" {
+                        Some(s)
+                    } else {
+                        None
+                    }
+                })
+            })
+            .is_some();
+
+        let info = gst_video::VideoInfo::from_caps(caps).ok_or_else(|| ())?;
         let frame =
             gst_video::VideoFrame::from_buffer_readable_gl(buffer, &info).or_else(|_| Err(()))?;
 
         Frame::new(
             info.width() as i32,
             info.height() as i32,
-            Arc::new(GStreamerBuffer { frame }),
+            Arc::new(GStreamerBuffer {
+                is_external_oes,
+                frame,
+            }),
         )
     }
 
@@ -210,7 +235,7 @@ impl Render for RenderUnix {
         let caps = gst::Caps::builder("video/x-raw")
             .features(&[&gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY])
             .field("format", &gst_video::VideoFormat::Rgba.to_string())
-            .field("texture-target", &"2D")
+            .field("texture-target", &gst::List::new(&[&"2D", &"external-oes"]))
             .build();
         appsink
             .set_property("caps", &caps)

backends/gstreamer/render-unix/lib.rs

@@ -53,7 +53,7 @@ mod platform {
             false
         }
 
-        fn build_frame(&self, _: gst::Buffer, _: gst_video::VideoInfo) -> Result<Frame, ()> {
+        fn build_frame(&self, _: gst::Sample) -> Result<Frame, ()> {
             Err(())
         }
 
@@ -95,14 +95,14 @@ impl GStreamerRender {
         }
     }
 
-    pub fn get_frame_from_sample(&self, sample: &gst::Sample) -> Result<Frame, ()> {
-        let buffer = sample.get_buffer_owned().ok_or_else(|| ())?;
-        let caps = sample.get_caps().ok_or_else(|| ())?;
-        let info = gst_video::VideoInfo::from_caps(caps).ok_or_else(|| ())?;
-
+    pub fn get_frame_from_sample(&self, sample: gst::Sample) -> Result<Frame, ()> {
         if let Some(render) = self.render.as_ref() {
-            render.build_frame(buffer, info)
+            render.build_frame(sample)
         } else {
+            let buffer = sample.get_buffer_owned().ok_or_else(|| ())?;
+            let caps = sample.get_caps().ok_or_else(|| ())?;
+            let info = gst_video::VideoInfo::from_caps(caps).ok_or_else(|| ())?;
+
             let frame =
                 gst_video::VideoFrame::from_buffer_readable(buffer, &info).or_else(|_| Err(()))?;
 

backends/gstreamer/render.rs

@@ -24,20 +24,15 @@ pub trait Render {
 
     /// Returns the Player's `Frame` to be consumed by the API user.
     ///
-    /// The implementator of this method will map the `buffer`,
-    /// according the `info`, to the rendering appropiate
-    /// structure. In the case of OpenGL-based renders, the `Frame`,
-    /// instead of the raw data, will transfer the texture ID.
+    /// The implementator of this method will map the `sample`'s
+    /// buffer to the rendering appropiate structure. In the case of
+    /// OpenGL-based renders, the `Frame`, instead of the raw data,
+    /// will transfer the texture ID.
     ///
     /// # Arguments
     ///
-    /// * `buffer` -  the GStreamer buffer to map
-    /// * `info` - buffer's video information
-    fn build_frame(
-        &self,
-        buffer: gst::Buffer,
-        info: gst_video::VideoInfo,
-    ) -> Result<sm_player::frame::Frame, ()>;
+    /// * `sample` -  the GStreamer sample with the buffer to map
+    fn build_frame(&self, sample: gst::Sample) -> Result<sm_player::frame::Frame, ()>;
 
     /// Sets the proper *video-sink* to GStreamer's `pipeline`, this
     /// video sink is simply a decorator of the passed `appsink`.

backends/gstreamer/render/lib.rs

@@ -8,7 +8,6 @@ euclid = { version = "0.19.0", optional = true }
 failure = { version = "0.1", optional = true }
 failure_derive = { version = "0.1", optional = true }
 gleam = { version = "0.6.8", optional = true }
-hyper = { version = "0.12", optional = true }
 rand = { version = "0.5.0", optional = true }
 time = { version = "0.1.40", optional = true }
 serde = {version = "1.0", optional = true }
@@ -29,7 +28,7 @@ clap = { version = "2.33", default-features = false, optional = true }
 
 [features]
 default = []
-gui = [ "winit", "glutin", "clap", "webrender", "webrender_api" ]
+gui = [ "euclid", "clap", "failure", "failure_derive", "gleam", "glutin", "webrender", "webrender_api", "winit" ]
 player = [ "ipc-channel" ]
 noise = [ "rand" ]
 webrtc = [ "serde", "serde_derive", "serde_json" ]