SM
Software Mansion•2mo ago
Fermuch

How to add frames to slow fps source?

I have a very slow source (MJPEG, about ~1 FPS) and I want to add an overlay on it, and store it as 30fps video. The overlay displays the current time (updated every 100ms). Here is my pipeline: 
defmodule MyPipeline do
  use Membrane.Pipeline

  alias Membrane.RawVideo

  @impl true
  def handle_init(_ctx, url) do
    spec =
      child(:source, %ZenFlow.VideoPipelines.Components.XavSource{source: url, is_device: false})
      |> child(:converter, %Membrane.FFmpeg.SWScale.PixelFormatConverter{format: :I420})
      # |> child(:framerate_converter, %Membrane.FramerateConverter{framerate: {30, 1}})

      # Add timestamp overlay
      |> child(:overlay_filter, %Membrane.OverlayFilter{
        initial_overlay: %Membrane.OverlayFilter.OverlayDescription{
          overlay: create_initial_timestamp(),
          x: :right,
          y: :top
        }
      })
      |> child(:timer, %ZenFlow.VideoPipelines.Components.TimestampOverlay{
        overlay_filter_name: :overlay_filter
      })


      # |> child(:realtimer, Membrane.Realtimer)
      |> child(Membrane.SDL.Player)

    {[spec: spec], %{}}
  end

  @impl true
  def handle_child_notification({:update_overlay, overlay_desc} = msg, _origin_element, _ctx, state) do
    {[notify_child: {:overlay_filter, msg}], state}
  end

  defp create_initial_timestamp do
    {:ok, vix_image} = Image.Text.text("00:00:00",
      font_size: 24,
      text_fill_color: :white,
      background_fill_color: :transparent,
      font: "Arial"
    )
    vix_image
  end
end
defmodule MyPipeline do
  use Membrane.Pipeline

  alias Membrane.RawVideo

  @impl true
  def handle_init(_ctx, url) do
    spec =
      child(:source, %ZenFlow.VideoPipelines.Components.XavSource{source: url, is_device: false})
      |> child(:converter, %Membrane.FFmpeg.SWScale.PixelFormatConverter{format: :I420})
      # |> child(:framerate_converter, %Membrane.FramerateConverter{framerate: {30, 1}})

      # Add timestamp overlay
      |> child(:overlay_filter, %Membrane.OverlayFilter{
        initial_overlay: %Membrane.OverlayFilter.OverlayDescription{
          overlay: create_initial_timestamp(),
          x: :right,
          y: :top
        }
      })
      |> child(:timer, %ZenFlow.VideoPipelines.Components.TimestampOverlay{
        overlay_filter_name: :overlay_filter
      })


      # |> child(:realtimer, Membrane.Realtimer)
      |> child(Membrane.SDL.Player)

    {[spec: spec], %{}}
  end

  @impl true
  def handle_child_notification({:update_overlay, overlay_desc} = msg, _origin_element, _ctx, state) do
    {[notify_child: {:overlay_filter, msg}], state}
  end

  defp create_initial_timestamp do
    {:ok, vix_image} = Image.Text.text("00:00:00",
      font_size: 24,
      text_fill_color: :white,
      background_fill_color: :transparent,
      font: "Arial"
    )
    vix_image
  end
end
I was expecting FramerateConverter to: - Recycle the same frame - Enable time counter to update every 100ms - Video shown at 30fps But if I enable FramerateConverter, it simply won't work anymore. Only one frame is shown, the first one, and then nothing gets updated ever. NOTE: Xav is non-blocking. It has another process reading frames and storing it in its buffer, then sending it with handle_demand (and using redemand if not enough frames exist)
2 Replies
Mateusz Front
Mateusz Front•2mo ago
Hi there, I suppose something may be wrong with the demand handling in the source, but I'm afraid I can't help without the source I tried to reproduce with the camera capture and without your custom elements, but it works fine: 
Mix.install([
  :membrane_camera_capture_plugin,
  :membrane_sdl_plugin,
  :membrane_ffmpeg_swscale_plugin,
  :membrane_framerate_converter_plugin,
  :membrane_overlay_plugin
])

Logger.configure(level: :info)

defmodule Run do
  import Membrane.ChildrenSpec

  def run do
    p = Membrane.RCPipeline.start_link!()

    Membrane.RCPipeline.exec_actions(p,
      spec:
        child(Membrane.CameraCapture)
        |> child(%Membrane.FFmpeg.SWScale.PixelFormatConverter{format: :I420})
        |> child(%Membrane.FramerateConverter{framerate: {1, 1}})
        |> child(%Membrane.FramerateConverter{framerate: {30, 1}})
        |> child(:overlay_filter, %Membrane.OverlayFilter{
          initial_overlay: %Membrane.OverlayFilter.OverlayDescription{
            overlay: create_initial_timestamp(),
            x: :right,
            y: :top
          }
        })
        |> child(Membrane.SDL.Player)
    )
  end

  defp create_initial_timestamp do
    {:ok, vix_image} =
      Image.Text.text("00:00:00",
        font_size: 24,
        text_fill_color: :white,
        background_fill_color: :transparent,
        font: "Arial"
      )

    vix_image
  end
end

Run.run()
Process.sleep(:infinity)
Mix.install([
  :membrane_camera_capture_plugin,
  :membrane_sdl_plugin,
  :membrane_ffmpeg_swscale_plugin,
  :membrane_framerate_converter_plugin,
  :membrane_overlay_plugin
])

Logger.configure(level: :info)

defmodule Run do
  import Membrane.ChildrenSpec

  def run do
    p = Membrane.RCPipeline.start_link!()

    Membrane.RCPipeline.exec_actions(p,
      spec:
        child(Membrane.CameraCapture)
        |> child(%Membrane.FFmpeg.SWScale.PixelFormatConverter{format: :I420})
        |> child(%Membrane.FramerateConverter{framerate: {1, 1}})
        |> child(%Membrane.FramerateConverter{framerate: {30, 1}})
        |> child(:overlay_filter, %Membrane.OverlayFilter{
          initial_overlay: %Membrane.OverlayFilter.OverlayDescription{
            overlay: create_initial_timestamp(),
            x: :right,
            y: :top
          }
        })
        |> child(Membrane.SDL.Player)
    )
  end

  defp create_initial_timestamp do
    {:ok, vix_image} =
      Image.Text.text("00:00:00",
        font_size: 24,
        text_fill_color: :white,
        background_fill_color: :transparent,
        font: "Arial"
      )

    vix_image
  end
end

Run.run()
Process.sleep(:infinity)
Fermuch
FermuchOP•2mo ago
Hey! Thanks for the answer. I'll take a better look at what I'm doing in the source. Thanks for pointing that out!

Did you find this page helpful?