diff --git a/src/pipeline.vala b/src/pipeline.vala new file mode 100644 index 0000000000000000000000000000000000000000..a2dadaf4f71c7bbbc99c0f642467da3072bff2d8 --- /dev/null +++ b/src/pipeline.vala @@ -0,0 +1,687 @@ +/* + * This file is part of queso. + * + * Copyright © 2020 Canek Peláez Valdés + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * along with this program. If not, see . + * + * Author: + * Canek Peláez Valdés + */ + +namespace Queso { + + /** + * Class to handle the pipeline. + * + * Initial raw pipeline: + * + *{{{ + * ┌────────────────────┐ + * | video-src | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-filter | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-queue-pretee | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-convert-gtk | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-text-record | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-text-info | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-gtk-sink | + * └────────────────────┘ + * }}} + * + * Recording raw pipeline: + * + * {{{ + * ┌────────────────────┐ ┌────────────────────┐ + * | video-src | | audio-src | + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-filter | │ audio-filter │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-queue-pretee | │ audio-queue-rate │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ | + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-tee │ | audio-rate │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * ┌────────────┴───────────┐ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-queue-gtk │ │ video-queue-file │ │ audio-convert │ + * └─────────┬──────────┘ └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-convert-gtk | | video-convert-file | │ audio-encoder │ + * └─────────┬──────────┘ └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-text-record | │ video-h264-encoder │ │ audio-queue-mp4mux │ + * └─────────┬──────────┘ └─────────┬──────────┘ └─────────┬──────────┘ + * │ └────────────┬───────────┘ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-text-info | | file-mp4mux | + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-gtk-sink | | file-queue │ + * └────────────────────┘ └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | file-sink │ + * └────────────────────┘ + * }}} + * + * Initial H.264 pipeline: + * + *{{{ + * ┌────────────────────┐ + * | video-src | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-filter | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-h264-parse | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-queue-pretee | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-h264-decoder | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-convert-gtk | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-text-record | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-text-info | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-gtk-sink | + * └────────────────────┘ + * }}} + * + * Recording H.264 pipeline: + * + * {{{ + * ┌────────────────────┐ ┌────────────────────┐ + * | video-src | | audio-src | + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-filter | | audio-filter | + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-h264-parse | │ audio-queue-rate │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-queue-pretee | │ audio-rate │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ | + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-tee │ | audio-convert │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * ┌────────────┴───────────┐ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-queue-gtk │ │ video-queue-file │ │ audio-encoder │ + * └─────────┬──────────┘ └─────────┬──────────┘ └─────────┬──────────┘ + * │ │ │ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-h264-decoder | | video-convert-file | │ audio-queue-mp4mux │ + * └─────────┬──────────┘ └─────────┬──────────┘ └─────────┬──────────┘ + * │ └────────────┬───────────┘ + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-convert-gtk | │ file-queue │ + * └─────────┬──────────┘ └─────────┬──────────┘ + * │ | + * ┌─────────┴──────────┐ ┌─────────┴──────────┐ + * | video-text-record | | file-sink | + * └─────────┬──────────┘ └────────────────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-text-info | + * └─────────┬──────────┘ + * │ + * ┌─────────┴──────────┐ + * | video-gtk-sink | + * └────────────────────┘ + * }}} + * + */ + public abstract class Pipeline : GLib.Object { + + public const int DEFAULT_AUDIO_RATE = 44100; + + private Gtk.Widget _widget; + public unowned Gtk.Widget widget { + get { + video_gtk_sink.get("widget", out _widget); + return _widget; + } + } + + public bool saving { public get; private set; } + + /* The video and audio devices. */ + private Devices devices; + public VideoDevice videodev; + public VideoDevice new_videodev; + public AudioDevice audiodev; + public VideoSetup video_setup; + public AudioSetup audio_setup; + + /* Pipeline and elements. */ + protected Gst.Pipeline pipeline; + public Gst.Bus bus; + + /* Raw pipeline. */ + public Gst.Element video_src; + public Gst.Element video_filter; + public Gst.Element video_queue_pretee; + public Gst.Element video_convert_gtk; + public Gst.Element video_text_record; + public Gst.Element video_text_info; + public Gst.Element video_gtk_sink; + + /* H.264 elements. */ + public Gst.Element video_h264_parse; + public Gst.Element video_h264_decoder; + + /* Audio pipeline. */ + public Gst.Element audio_src; + public Gst.Element audio_filter; + public Gst.Element audio_queue_rate; + public Gst.Element audio_rate; + public Gst.Element audio_convert; + public Gst.Element audio_encoder; + public Gst.Element audio_queue_mp4mux; + + /* File sink branch. */ + public Gst.Element video_tee; + public Gst.Element video_queue_gtk; + public Gst.Element video_queue_file; + public Gst.Element video_convert_file; + public Gst.Element video_h264_encoder; + public Gst.Element file_mp4mux; + + /* File sink*/ + public Gst.Element file_queue; + public Gst.Element file_sink; + + /* Temporary filename . */ + private string temporary_fn; + + /* Timestamps. */ + private double start_saving_ts; + private double last_saving_ts; + + private bool show_info; + private bool saving_done; + + public Pipeline(Devices devices) { + this.devices = devices; + devices.device_changed.connect(device_changed); + videodev = devices.video_device; + audiodev = devices.audio_device; + video_setup = videodev.setups.first(); + audio_setup = audiodev.setups.first(); + + pipeline = new Gst.Pipeline("queso"); + + switch (video_setup.media_type) { + case VideoSetup.RAW: + set_raw_pipeline(); + break; + case VideoSetup.H264: + set_h264_pipeline(); + break; + } + + bus = pipeline.get_bus(); + + bus.add_watch(GLib.Priority.DEFAULT, pipeline_bus_watch); + } + + public void start_saving() { + start_saving_ts = last_saving_ts = Timestamp.get_timestamp(); + video_text_record.set("text", "•REC 00:00:00"); + video_text_record.set("halignment", 2); + video_text_record.set("valignment", 2); + GLib.Idle.add(update_counter); + saving_done = false; + var pad = video_queue_pretee.get_static_pad("src"); + pad.add_probe(Gst.PadProbeType.BLOCK | Gst.PadProbeType.BUFFER, + start_saving_block_probe); + } + + public void stop_saving() { + var pad = video_queue_pretee.get_static_pad("src"); + pad.add_probe(Gst.PadProbeType.BLOCK | Gst.PadProbeType.BUFFER, + stop_saving_block_probe); + video_text_record.set("text", ""); + } + + private void set_raw_pipeline() { + video_src = videodev.create_element("video-source"); + video_filter = Gst.ElementFactory.make("capsfilter", + "video-filter"); + var fr = video_setup.framerates.first(); + var caps = new Gst.Caps.empty_simple(video_setup.media_type); + caps.set_simple("framerate", typeof(Gst.Fraction), + fr.numerator, fr.denominator, + "width", typeof(int), video_setup.width, + "height", typeof(int), video_setup.height); + video_filter.set("caps", caps); + video_queue_pretee = Gst.ElementFactory.make("queue", + "video-queue-pretee"); + video_convert_gtk = Gst.ElementFactory.make("videoconvert", + "video-convert-gtk"); + video_text_record = Gst.ElementFactory.make("textoverlay", + "video-text-record"); + video_text_record.set("font-desc", "sans 14"); + video_text_record.set("halignment", 2); + video_text_record.set("valignment", 2); + video_text_info = Gst.ElementFactory.make("textoverlay", + "video-text-info"); + video_text_info.set("font-desc", "sans 10"); + video_text_info.set("halignment", 0); + video_text_info.set("valignment", 2); + video_text_info.set("line-alignment", 0); + video_gtk_sink = Gst.ElementFactory.make("gtksink", + "video-gtk-sink"); + + pipeline.add_many(video_src, + video_filter, + video_queue_pretee, + video_convert_gtk, + video_text_record, + video_text_info, + video_gtk_sink); + video_src.link_many(video_filter, + video_queue_pretee, + video_convert_gtk, + video_text_record, + video_text_info, + video_gtk_sink); + } + + private void set_h264_pipeline() { + } + + /* Pipeline bus watch. */ + private bool + pipeline_bus_watch(Gst.Bus bus, Gst.Message message) { + switch (message.type) { + case Gst.MessageType.ELEMENT: + unowned Gst.Structure structure = message.get_structure(); + if (!structure.has_name("GstBinForwarded")) + return GLib.Source.CONTINUE; + Gst.Message forward_message = null; + structure.get("message", typeof(Gst.Message), + out forward_message); + if (forward_message.type != Gst.MessageType.EOS) + return GLib.Source.CONTINUE; + if (saving_done) + return GLib.Source.CONTINUE; + + saving_done = true; + + video_tee.set_state(Gst.State.NULL); + video_queue_gtk.set_state(Gst.State.NULL); + video_queue_file.set_state(Gst.State.NULL); + video_convert_file.set_state(Gst.State.NULL); + video_h264_encoder.set_state(Gst.State.NULL); + file_mp4mux.set_state(Gst.State.NULL); + + audio_src.set_state(Gst.State.NULL); + audio_filter.set_state(Gst.State.NULL); + audio_queue_rate.set_state(Gst.State.NULL); + audio_rate.set_state(Gst.State.NULL); + audio_convert.set_state(Gst.State.NULL); + audio_encoder.set_state(Gst.State.NULL); + audio_queue_mp4mux.set_state(Gst.State.NULL); + + file_queue.set_state(Gst.State.NULL); + file_sink.set_state(Gst.State.NULL); + + pipeline.remove_many(video_tee, + video_queue_gtk, + video_queue_file, + video_convert_file, + video_h264_encoder, + audio_src, + audio_filter, + audio_queue_rate, + audio_rate, + audio_convert, + audio_encoder, + audio_queue_mp4mux, + file_mp4mux, + file_queue, + file_sink); + var src_pad = video_queue_pretee.get_static_pad("src"); + var sink_pad = video_convert_gtk.get_static_pad("sink"); + src_pad.link(sink_pad); + + video_tee = null; + video_queue_gtk = null; + video_queue_file = null; + video_convert_file = null; + video_h264_encoder = null; + + /* FIXME: This causes a warning: + gst_object_unref: + assertion '((GObject *) object)->ref_count > 0' failed + Find why. */ + audio_src = null; + + audio_filter = null; + audio_queue_rate = null; + audio_rate = null; + audio_convert = null; + audio_encoder = null; + audio_queue_mp4mux = null; + file_mp4mux = null; + file_queue = null; + file_sink = null; + + var destination_fn = get_destination(); + move_file(temporary_fn, destination_fn); + temporary_fn = destination_fn = null; + return GLib.Source.CONTINUE; + default: + return GLib.Source.CONTINUE; + } + } + + /* Saving block probe. */ + private Gst.PadProbeReturn + stop_saving_block_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + pad.remove_probe(info.id); + var src = video_queue_pretee.get_static_pad("src"); + var sink = video_tee.get_static_pad("sink"); + src.unlink(sink); + src = video_queue_gtk.get_static_pad("src"); + sink = video_convert_gtk.get_static_pad("sink"); + src.unlink(sink); + src = video_queue_pretee.get_static_pad("src"); + sink = video_convert_gtk.get_static_pad("sink"); + src.link(sink); + new Thread("eos-push-thread", push_eos_thread); + return Gst.PadProbeReturn.DROP; + } + + /* Change video source block probe. */ + private Gst.PadProbeReturn + start_saving_block_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + pad.remove_probe(info.id); + temporary_fn = random_filename(); + + var src_pad = video_queue_pretee.get_static_pad("src"); + var sink_pad = video_convert_gtk.get_static_pad("sink"); + + src_pad.unlink(sink_pad); + + audio_src = audiodev.create_element("audio-src"); + audio_filter = Gst.ElementFactory.make("capsfilter", + "audio-filter"); + int rate = get_audio_rate(); + int channels = audio_setup.max_rate > 1 ? 2 : 1; + var fmt = audio_setup.formats.first(); + var acaps = new Gst.Caps.empty_simple(audio_setup.media_type); + acaps.set_simple("format", typeof(string), fmt.format, + "rate", typeof(int), rate, + "channels", typeof(int), channels); + audio_filter.set("caps", acaps); + audio_queue_rate = Gst.ElementFactory.make("queue", + "audio-queue-rate"); + audio_queue_rate.set("leaky", 2); + audio_rate = Gst.ElementFactory.make("audiorate", "audio-rate"); + audio_convert = Gst.ElementFactory.make("audioconvert", + "audio-convert"); + audio_encoder = Gst.ElementFactory.make("avenc_aac", + "audio_encoder"); + audio_queue_mp4mux = Gst.ElementFactory.make("queue", + "audio_queue_mp4mux"); + audio_queue_mp4mux.set("max-size-time", 1 * Gst.SECOND); + audio_queue_mp4mux.set("max-size-bytes", 0); + audio_queue_mp4mux.set("max-size-buffers", 0); + + video_tee = Gst.ElementFactory.make("tee", "video-tee"); + video_queue_gtk = Gst.ElementFactory.make("queue", + "video-queue-gtk"); + video_queue_file = Gst.ElementFactory.make("queue", + "video-queue-file"); + video_convert_file = Gst.ElementFactory.make("videoconvert", + "video-convert-file"); + video_h264_encoder = Gst.ElementFactory.make("x264enc", + "video-h264-encoder"); + file_mp4mux = Gst.ElementFactory.make("mpegtsmux", + "file-mp4mux"); + file_queue = Gst.ElementFactory.make("queue", "file-queue"); + file_sink = Gst.ElementFactory.make("filesink", "file-sink"); + file_sink.set("async", false); + file_sink.set("location", temporary_fn); + + pipeline.add_many(video_tee, + audio_src, + audio_filter, + audio_queue_rate, + audio_rate, + audio_convert, + audio_encoder, + audio_queue_mp4mux, + video_queue_gtk, + video_queue_file, + video_convert_file, + video_h264_encoder, + file_mp4mux, + file_queue, + file_sink); + video_queue_pretee.link(video_tee); + video_tee.link_many(video_queue_gtk, + video_convert_gtk); + video_tee.link_many(video_queue_file, + video_convert_file, + video_h264_encoder, + file_mp4mux, + file_queue, + file_sink); + audio_src.link_many(audio_filter, + audio_queue_rate, + audio_rate, + audio_convert, + audio_encoder, + audio_queue_mp4mux, + file_mp4mux); + + new Thread("add-wait-keyframe-thread", + add_wait_keyframe_probe); + + return Gst.PadProbeReturn.DROP; + } + + private void device_changed(DeviceType device_type) { + switch (device_type) { + case VIDEO: + video_device_changed(); + break; + } + } + + private void video_device_changed() { + video_text_record.set("halignment", 1); + video_text_record.set("valignment", 1); + video_text_record.set("text", "Changing video source..."); + new_videodev = devices.video_device; + var pad = video_queue_pretee.get_static_pad("src"); + pad.add_probe(Gst.PadProbeType.BLOCK | + Gst.PadProbeType.BUFFER, + change_video_source_block_probe); + } + + /* Change video source block probe. */ + private Gst.PadProbeReturn + change_video_source_block_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + pad.remove_probe(info.id); + + video_src.set_state(Gst.State.NULL); + video_filter.set_state(Gst.State.NULL); + pipeline.remove_many(video_src, video_filter); + + videodev = new_videodev; + new_videodev = null; + video_setup = videodev.setups.first(); + video_src = videodev.create_element("video-src"); + video_filter = Gst.ElementFactory.make("capsfilter", + "video-filter"); + var fr = video_setup.framerates.first(); + var caps = new Gst.Caps.empty_simple(video_setup.media_type); + caps.set_simple("framerate", typeof(Gst.Fraction), + fr.numerator, fr.denominator, + "width", typeof(int), video_setup.width, + "height", typeof(int), video_setup.height); + video_filter.set("caps", caps); + pipeline.add_many(video_src, video_filter); + video_src.link_many(video_filter, video_queue_pretee); + video_src.set_state(Gst.State.PLAYING); + video_filter.set_state(Gst.State.PLAYING); + + new Thread("add-wait-keyframe-thread", + add_wait_keyframe_probe); + + return Gst.PadProbeReturn.DROP; + } + + private void* + add_wait_keyframe_probe() { + var pad = video_queue_pretee.get_static_pad("src"); + pad.add_probe(Gst.PadProbeType.BLOCK | Gst.PadProbeType.BUFFER, + wait_keyframe_probe); + return null; + } + + private Gst.PadProbeReturn + wait_keyframe_probe(Gst.Pad pad, Gst.PadProbeInfo info) { + Gst.Buffer buffer = (Gst.Buffer)info.data; + var flags = buffer.get_flags(); + if ((flags & Gst.BufferFlags.DELTA_UNIT) == 0) { + return Gst.PadProbeReturn.REMOVE; + } else { + return Gst.PadProbeReturn.DROP; + } + } + + /* Gets the final destination for the video. */ + private string get_destination() { + var now = new GLib.DateTime.now_local(); + string sts = "%04d-%02d-%02d %02d:%02d:%02d".printf( + now.get_year(), now.get_month(), now.get_day_of_month(), + now.get_hour(), now.get_minute(), now.get_second()); + var video_dir = GLib.Environment.get_user_special_dir( + GLib.UserDirectory.VIDEOS); + return string.join(GLib.Path.DIR_SEPARATOR_S, video_dir, + "Queso-%s.mp4".printf(sts)); + } + + /* Pushes the EOS event. */ + private void* + push_eos_thread() { + var pad = file_queue.get_static_pad("src"); + var peer = pad.get_peer(); + pipeline.message_forward = true; + peer.send_event(new Gst.Event.eos()); + return null; + } + + /* Moves a file. */ + private void move_file(string src_path, string dest_path) { + GLib.File src = GLib.File.new_for_path(src_path); + GLib.File dest = GLib.File.new_for_path(dest_path); + + try { + src.move(dest, GLib.FileCopyFlags.NONE, null); + } catch (GLib.Error e) { + GLib.warning("Could not move “%s” to “%s”: %s", + src_path, dest_path, e.message); + } + } + + /* Update counter idle callback. */ + private bool update_counter() { + if (!saving) { + video_text_record.set("text", ""); + return GLib.Source.REMOVE; + } + Gst.State state = Gst.State.NULL; + Gst.State pending; + pipeline.get_state(out state, out pending, 100); + if (state != Gst.State.PLAYING) + return GLib.Source.CONTINUE; + double ts = Timestamp.get_timestamp(); + if (ts - last_saving_ts < 1.0) + return GLib.Source.CONTINUE; + last_saving_ts = ts; + string sts = Timestamp.to_string(ts - start_saving_ts); + video_text_record.set("text", "•REC " + sts); + return GLib.Source.CONTINUE; + } + + /* Gets a random filename. */ + private string random_filename() { + int ts = (int)Timestamp.get_timestamp(); + return string.join(GLib.Path.DIR_SEPARATOR_S, + GLib.Environment.get_tmp_dir(), + "queso-%08X.mp4".printf(ts)); + } + + private int get_audio_rate() { + int rate = DEFAULT_AUDIO_RATE; + while (rate > audio_setup.max_rate) + rate /= 2; + return rate; + } + } +}