media/
gstreamer_video.rs

1//! GStreamer video capture (M-MEDIA.6 / AUT-102) — CLI-pipe pattern.
2//!
3//! Spawns `gst-launch-1.0` with a pipeline that emits raw BGRA frames
4//! on stdout, then chunks the byte stream into [`VideoFrame`]s (the
5//! same type `decode` uses).
6//!
7//! # Pipeline
8//!
9//! ```text
10//! videotestsrc is-live=false
11//!   ! videoconvert
12//!   ! video/x-raw,format=BGRA,width=W,height=H,framerate=F/1
13//!   ! fdsink fd=1
14//! ```
15//!
16//! AUT-102 only asks for the `videotestsrc` path. M-MEDIA.16 (live
17//! webcam) will add an `autovideosrc` variant; M-MEDIA.17 (playback
18//! harness) will add a `filesrc ! decodebin` variant.
19
20use std::io::{ErrorKind, Read};
21use std::process::{Child, ChildStdout, Command, Stdio};
22
23use crate::clock::MediaTime;
24use crate::video::VideoFrame;
25
26/// Failure modes for the GStreamer video capture pipe.
27#[derive(Debug, thiserror::Error)]
28pub enum Error {
29    /// `gst-launch-1.0` could not be launched. The `PATH` snapshot in
30    /// the message makes CI diagnoses easier.
31    #[error("failed to spawn `gst-launch-1.0`: {source} (PATH={path})")]
32    Spawn {
33        /// The OS-level reason the spawn failed.
34        #[source]
35        source: std::io::Error,
36        /// `$PATH` at the moment of failure.
37        path: String,
38    },
39    /// Stdout was not piped.
40    #[error("child stdout was not piped")]
41    NoStdout,
42    /// I/O error while reading from the child's stdout.
43    #[error("read error: {0}")]
44    Io(#[from] std::io::Error),
45    /// Pipeline ended before the requested frame was read.
46    #[error("video pipeline ended after {frames_read} frames")]
47    EndOfStream {
48        /// Frames actually delivered before EOF.
49        frames_read: u64,
50    },
51    /// Dimensions or framerate would produce zero-byte frames.
52    #[error("invalid format: width={width} height={height} framerate={framerate} fps")]
53    InvalidFormat {
54        /// Requested width in pixels.
55        width: u32,
56        /// Requested height in pixels.
57        height: u32,
58        /// Requested framerate in fps.
59        framerate: f64,
60    },
61    /// The picker handed us a `camera_id` that no longer matches any
62    /// device on the host — typically the camera was unplugged
63    /// between `list_cameras()` and `from_camera()`. Callers should
64    /// re-enumerate and re-prompt (M-CAM.4 / AUT — see
65    /// `milestone-2-record-and-export.md`).
66    #[error("camera id `{id}` not present on this host (was the camera unplugged?)")]
67    CameraNotFound {
68        /// The id the caller passed in.
69        id: String,
70    },
71}
72
73/// Streaming video capture wrapping a `gst-launch-1.0` child process.
74pub struct GstreamerVideoCapture {
75    child: Child,
76    stdout: ChildStdout,
77    width: u32,
78    height: u32,
79    framerate: f64,
80    next_index: u64,
81    /// Pre-allocated scratch buffer for the raw bytes of one frame.
82    raw_buffer: Vec<u8>,
83}
84
85impl std::fmt::Debug for GstreamerVideoCapture {
86    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87        f.debug_struct("GstreamerVideoCapture")
88            .field("width", &self.width)
89            .field("height", &self.height)
90            .field("framerate", &self.framerate)
91            .field("frames_emitted", &self.next_index)
92            .finish_non_exhaustive()
93    }
94}
95
96impl GstreamerVideoCapture {
97    /// Build a capture from the **default OS camera** via gst's
98    /// `autovideosrc` (M-CAM.0 / AUT-254).
99    ///
100    /// On macOS `autovideosrc` routes to `avfvideosrc`; on Linux
101    /// `v4l2src`; on Windows `mfvideosrc`. Caller picks the output
102    /// dimensions + framerate — gst's `videoconvert` step resizes /
103    /// converts whatever the camera natively produces.
104    ///
105    /// ```admonish important
106    /// **macOS gotcha:** `avfvideosrc` requires
107    /// `NSCameraUsageDescription` in the bundled app's Info.plist.
108    /// Without it the gst pipeline fails with a misleading "device
109    /// busy" error AND the OS permission prompt never shows. See
110    /// `crates/app/tauri.conf.json` for the project's declaration.
111    /// ```
112    ///
113    /// # Errors
114    ///
115    /// - [`Error::InvalidFormat`] if any dimension is zero.
116    /// - [`Error::Spawn`] if `gst-launch-1.0` isn't on `PATH`.
117    /// - [`Error::NoStdout`] if the child's stdout pipe is missing
118    ///   (shouldn't happen — we request it explicitly).
119    ///
120    /// Cross-OS behaviour: on a host without a default camera the
121    /// pipeline spawns but `next_frame` returns
122    /// [`Error::EndOfStream`] as soon as the OS denies access /
123    /// reports no device. Integration tests should call
124    /// [`default_camera_available`] first and skip cleanly.
125    pub fn from_default_camera(width: u32, height: u32, framerate: u32) -> Result<Self, Error> {
126        if width == 0 || height == 0 || framerate == 0 {
127            return Err(Error::InvalidFormat {
128                width,
129                height,
130                framerate: f64::from(framerate),
131            });
132        }
133        let caps = format!(
134            "video/x-raw,format=BGRA,width={width},height={height},framerate={framerate}/1"
135        );
136        let mut cmd = Command::new("gst-launch-1.0");
137        cmd.args(["-q", "autovideosrc"])
138            .args(live_camera_tail_args(&caps))
139            .stdout(Stdio::piped())
140            .stderr(Stdio::null());
141        let mut child = cmd.spawn().map_err(|source| Error::Spawn {
142            source,
143            path: std::env::var("PATH").unwrap_or_else(|_| "<unset>".into()),
144        })?;
145        let stdout = child.stdout.take().ok_or(Error::NoStdout)?;
146        Ok(Self {
147            child,
148            stdout,
149            width,
150            height,
151            framerate: f64::from(framerate),
152            next_index: 0,
153            raw_buffer: Vec::new(),
154        })
155    }
156
157    /// Build a capture pinned to a *specific* camera resolved by its
158    /// stable id (M-CAM.4). Re-probes `list_cameras()` at call time to
159    /// turn `camera_id` into the OS-native source element + props
160    /// (`avfvideosrc device-index=N` on macOS, `mfvideosrc
161    /// device-path=...` on Windows, `v4l2src device=...` on Linux).
162    ///
163    /// If the camera is in the enumeration but the parser couldn't
164    /// extract a `gst_source` for it (unusual — would indicate a
165    /// gst-device-monitor output format the parser didn't recognise),
166    /// falls back to `autovideosrc` with a `tracing::warn` so the user
167    /// still sees *some* camera. The picker just won't be honored.
168    ///
169    /// # Errors
170    ///
171    /// - [`Error::InvalidFormat`] if any dimension is zero.
172    /// - [`Error::CameraNotFound`] if no enumerated camera matches
173    ///   `camera_id` (typical: the camera was unplugged between
174    ///   `list_cameras()` and `from_camera()`).
175    /// - [`Error::Spawn`] if `gst-launch-1.0` isn't on `PATH`.
176    /// - [`Error::NoStdout`] if the child's stdout pipe is missing.
177    pub fn from_camera(
178        camera_id: &str,
179        width: u32,
180        height: u32,
181        framerate: u32,
182    ) -> Result<Self, Error> {
183        if width == 0 || height == 0 || framerate == 0 {
184            return Err(Error::InvalidFormat {
185                width,
186                height,
187                framerate: f64::from(framerate),
188            });
189        }
190        let device = crate::camera::find_by_id(camera_id).ok_or_else(|| Error::CameraNotFound {
191            id: camera_id.to_string(),
192        })?;
193        let source_tokens: Vec<String> = if let Some(ref s) = device.gst_source {
194            s.split_whitespace().map(str::to_string).collect()
195        } else {
196            tracing::warn!(
197                camera_id = %camera_id,
198                label = %device.label,
199                "from_camera: device enumerated but `gst_source` was None — falling back to autovideosrc; \
200                 per-device routing will NOT pin to this physical camera"
201            );
202            vec!["autovideosrc".to_string()]
203        };
204        let caps = format!(
205            "video/x-raw,format=BGRA,width={width},height={height},framerate={framerate}/1"
206        );
207        let mut cmd = Command::new("gst-launch-1.0");
208        cmd.arg("-q");
209        for tok in &source_tokens {
210            cmd.arg(tok);
211        }
212        cmd.args(live_camera_tail_args(&caps))
213            .stdout(Stdio::piped())
214            .stderr(Stdio::null());
215        tracing::info!(
216            camera_id = %camera_id,
217            label = %device.label,
218            source = %source_tokens.join(" "),
219            "from_camera: spawning gst-launch with pinned source"
220        );
221        let mut child = cmd.spawn().map_err(|source| Error::Spawn {
222            source,
223            path: std::env::var("PATH").unwrap_or_else(|_| "<unset>".into()),
224        })?;
225        let stdout = child.stdout.take().ok_or(Error::NoStdout)?;
226        Ok(Self {
227            child,
228            stdout,
229            width,
230            height,
231            framerate: f64::from(framerate),
232            next_index: 0,
233            raw_buffer: Vec::new(),
234        })
235    }
236
237    /// Build a capture from `videotestsrc` at the given dimensions +
238    /// framerate. The default `videotestsrc` pattern is the SMPTE
239    /// colorbars — useful for visual smoke checks because every frame
240    /// is visually distinct (animated subpattern).
241    pub fn test_source(width: u32, height: u32, framerate: u32) -> Result<Self, Error> {
242        if width == 0 || height == 0 || framerate == 0 {
243            return Err(Error::InvalidFormat {
244                width,
245                height,
246                framerate: f64::from(framerate),
247            });
248        }
249        let caps = format!(
250            "video/x-raw,format=BGRA,width={width},height={height},framerate={framerate}/1"
251        );
252        let mut cmd = Command::new("gst-launch-1.0");
253        cmd.args([
254            "-q",
255            "videotestsrc",
256            "is-live=false",
257            "!",
258            "videoconvert",
259            "!",
260            &caps,
261            "!",
262            "fdsink",
263            "fd=1",
264        ])
265        .stdout(Stdio::piped())
266        .stderr(Stdio::null());
267        let mut child = cmd.spawn().map_err(|source| Error::Spawn {
268            source,
269            path: std::env::var("PATH").unwrap_or_else(|_| "<unset>".into()),
270        })?;
271        let stdout = child.stdout.take().ok_or(Error::NoStdout)?;
272        Ok(Self {
273            child,
274            stdout,
275            width,
276            height,
277            framerate: f64::from(framerate),
278            next_index: 0,
279            raw_buffer: Vec::new(),
280        })
281    }
282
283    /// Width × height the captured frames will carry.
284    #[must_use]
285    pub fn dimensions(&self) -> (u32, u32) {
286        (self.width, self.height)
287    }
288
289    /// Framerate the pipeline emits.
290    #[must_use]
291    pub fn framerate(&self) -> f64 {
292        self.framerate
293    }
294
295    /// Cumulative frames emitted across `next_frame` calls.
296    #[must_use]
297    pub fn frames_emitted(&self) -> u64 {
298        self.next_index
299    }
300
301    /// Read one BGRA frame. PTS is computed from the frame index and
302    /// the captured framerate.
303    ///
304    /// # Errors
305    ///
306    /// - [`Error::Io`] on read failures.
307    /// - [`Error::EndOfStream`] if the pipeline ends mid-frame.
308    pub fn next_frame(&mut self) -> Result<VideoFrame, Error> {
309        let need = usize::try_from(self.width)
310            .expect("width fits usize")
311            .checked_mul(usize::try_from(self.height).expect("height fits usize"))
312            .and_then(|n| n.checked_mul(4))
313            .ok_or(Error::InvalidFormat {
314                width: self.width,
315                height: self.height,
316                framerate: self.framerate,
317            })?;
318        if self.raw_buffer.len() < need {
319            self.raw_buffer.resize(need, 0);
320        }
321        let slice = &mut self.raw_buffer[..need];
322        let mut read = 0;
323        while read < need {
324            match self.stdout.read(&mut slice[read..]) {
325                Ok(0) => {
326                    return Err(Error::EndOfStream {
327                        frames_read: self.next_index,
328                    });
329                }
330                Ok(n) => read += n,
331                Err(e) if e.kind() == ErrorKind::Interrupted => {}
332                Err(e) => return Err(Error::Io(e)),
333            }
334        }
335        let frame = VideoFrame {
336            width: self.width,
337            height: self.height,
338            bgra: slice.to_vec(),
339            pts_seconds: MediaTime::from_frame(self.next_index, self.framerate).as_seconds(),
340            frame_index: self.next_index,
341        };
342        self.next_index = self.next_index.saturating_add(1);
343        Ok(frame)
344    }
345}
346
347fn live_camera_tail_args(caps: &str) -> [&str; 12] {
348    [
349        "!",
350        "videoconvert",
351        "!",
352        // M-QUAL.3 — center-crop the webcam's native 16:9 (or other)
353        // frame to 1:1 BEFORE scaling to the square preview caps, so
354        // the circular bubble shows an undistorted face. Without this
355        // `videoscale` squishes the full frame into the square (a
356        // horizontally-compressed face). `aspectratiocrop` is in
357        // gst-plugins-good, shipped with every `gstreamer` install.
358        "aspectratiocrop",
359        "aspect-ratio=1/1",
360        "!",
361        "videoscale",
362        "!",
363        caps,
364        "!",
365        "fdsink",
366        "fd=1",
367    ]
368}
369
370impl Drop for GstreamerVideoCapture {
371    fn drop(&mut self) {
372        let _ = self.child.kill();
373        let _ = self.child.wait();
374    }
375}
376
377/// Best-effort probe for "is there at least one video capture device
378/// the OS will hand us?" (M-CAM.0 / AUT-254).
379///
380/// Spawns `gst-device-monitor-1.0 Video/Source` with a short timeout
381/// and parses the output for at least one device line. Returns `false`
382/// if the binary isn't on `PATH`, returns `false` if no devices are
383/// listed — never panics. Integration tests use this to skip cleanly
384/// on a host without a webcam.
385#[must_use]
386pub fn default_camera_available() -> bool {
387    let output = Command::new("gst-device-monitor-1.0")
388        .args(["Video/Source"])
389        .stdout(Stdio::piped())
390        .stderr(Stdio::null())
391        .output();
392    match output {
393        Ok(out) if out.status.success() => {
394            let stdout = String::from_utf8_lossy(&out.stdout);
395            // The text format has `Device found:` lines, one per
396            // device. Any match = at least one camera.
397            stdout.contains("Device found:")
398        }
399        _ => false,
400    }
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406
407    #[test]
408    fn invalid_dimensions_rejected_at_construction() {
409        assert!(matches!(
410            GstreamerVideoCapture::test_source(0, 360, 30),
411            Err(Error::InvalidFormat { width: 0, .. })
412        ));
413        assert!(matches!(
414            GstreamerVideoCapture::test_source(640, 0, 30),
415            Err(Error::InvalidFormat { height: 0, .. })
416        ));
417        assert!(matches!(
418            GstreamerVideoCapture::test_source(640, 360, 0),
419            Err(Error::InvalidFormat { framerate, .. }) if framerate.abs() < 1e-9
420        ));
421    }
422
423    #[test]
424    fn capture_is_send() {
425        fn assert_send<T: Send>() {}
426        assert_send::<GstreamerVideoCapture>();
427    }
428
429    #[test]
430    fn live_camera_pipeline_crops_then_scales_before_square_caps() {
431        let caps = "video/x-raw,format=BGRA,width=720,height=720,framerate=30/1";
432        let args = live_camera_tail_args(caps);
433        let crop_pos = args
434            .iter()
435            .position(|arg| *arg == "aspectratiocrop")
436            .expect("live camera pipeline should center-crop to square (M-QUAL.3)");
437        let scale_pos = args
438            .iter()
439            .position(|arg| *arg == "videoscale")
440            .expect("live camera pipeline should include videoscale");
441        let caps_pos = args
442            .iter()
443            .position(|arg| arg.starts_with("video/x-raw"))
444            .expect("live camera pipeline should include raw caps");
445
446        // Order matters: crop the native frame to 1:1 first (undistorted
447        // face), THEN scale to the square preview caps. Cropping after
448        // the squish-scale would be too late.
449        assert!(
450            crop_pos < scale_pos && scale_pos < caps_pos,
451            "expected aspectratiocrop → videoscale → caps, got {args:?}"
452        );
453        // The crop must request a 1:1 aspect, else it's a no-op.
454        assert!(
455            args.contains(&"aspect-ratio=1/1"),
456            "aspectratiocrop must target 1/1: {args:?}"
457        );
458    }
459}
media/gstreamer_video.rs

media/
gstreamer_video.rs
