SMM
Software Mansion / MembraneAske

Dynamically starting children to Demux Mp4 tracks

I want to convert this to take arbitrary user uploaded Mp4 files where the tracks can have different indexes: 
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
      # Video
      |> via_out(Pad.ref(:output, 1))
      |> child(:vido_tmp, %Membrane.H264.Parser{
        output_stream_structure: :annexb
      })
      |> do_video()
      |> child(:video_out, %Membrane.H264.Parser{
        generate_best_effort_timestamps: %{framerate: {25, 1}},
        output_stream_structure: :avc1
      }),
      # Audio
      get_child(:demuxer)
      |> via_out(Pad.ref(:output, 2))
      |> do_audio(),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
      get_child(:video_out) |> get_child(:muxer),
      get_child(:audio_out) |> get_child(:muxer)
    ]
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
      # Video
      |> via_out(Pad.ref(:output, 1))
      |> child(:vido_tmp, %Membrane.H264.Parser{
        output_stream_structure: :annexb
      })
      |> do_video()
      |> child(:video_out, %Membrane.H264.Parser{
        generate_best_effort_timestamps: %{framerate: {25, 1}},
        output_stream_structure: :avc1
      }),
      # Audio
      get_child(:demuxer)
      |> via_out(Pad.ref(:output, 2))
      |> do_audio(),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
      get_child(:video_out) |> get_child(:muxer),
      get_child(:audio_out) |> get_child(:muxer)
    ]
I tried converting the above to: 
  @impl true
  def handle_init(_ctx, _path) do
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
    ]

    {[spec: structure], %{}}
  end

  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.map(fn
        {track_id, %Membrane.H264{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_video()

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    {[spec: spec], state}
  end
  @impl true
  def handle_init(_ctx, _path) do
    structure = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM),
      # Mux
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})
    ]

    {[spec: structure], %{}}
  end

  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.map(fn
        {track_id, %Membrane.H264{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_video()

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    {[spec: spec], state}
  end
but now the demuxer doesn't run, because there are no outputs to pull data.. How do I add dynamic children, based on the track types?
A
Aske•32d ago
I get this error: 
[error] GenServer #PID<0.2412.0> terminating
** (ArgumentError) errors were found at the given arguments:

  * 1st argument: not a nonempty list

    :erlang.hd([])
    (membrane_mp4_plugin 0.34.1) lib/membrane_mp4/muxer/isom.ex:130: Membrane.MP4.Muxer.ISOM.handle_demand/5
    (membrane_core 1.0.1) lib/membrane/core/callback_handler.ex:139: Membrane.Core.CallbackHandler.exec_callback/4
[error] GenServer #PID<0.2412.0> terminating
** (ArgumentError) errors were found at the given arguments:

  * 1st argument: not a nonempty list

    :erlang.hd([])
    (membrane_mp4_plugin 0.34.1) lib/membrane_mp4/muxer/isom.ex:130: Membrane.MP4.Muxer.ISOM.handle_demand/5
    (membrane_core 1.0.1) lib/membrane/core/callback_handler.ex:139: Membrane.Core.CallbackHandler.exec_callback/4
I guess I could attempt to pick apart the output of: 
iex> File.stream!(path, [], 2048) |> Enum.take(50) |> IO.iodata_to_binary() |> Membrane.MP4.Container.parse()
{:ok,
 [
   ftyp: %{
     children: [],
     fields: %{
       compatible_brands: ["isom", "iso2", "avc1", "mp41"],
       major_brand: "isom",
       major_brand_version: 512
     }
   },
   moov: %{
     children: [
       mvhd: %{
         children: [],
         fields: %{
           creation_time: 0,
           duration: 87872,
           flags: 0,
           matrix_value_A: {1, 0},
iex> File.stream!(path, [], 2048) |> Enum.take(50) |> IO.iodata_to_binary() |> Membrane.MP4.Container.parse()
{:ok,
 [
   ftyp: %{
     children: [],
     fields: %{
       compatible_brands: ["isom", "iso2", "avc1", "mp41"],
       major_brand: "isom",
       major_brand_version: 512
     }
   },
   moov: %{
     children: [
       mvhd: %{
         children: [],
         fields: %{
           creation_time: 0,
           duration: 87872,
           flags: 0,
           matrix_value_A: {1, 0},
but not sure that's a good approach. Ah.. I see another person posted a very similar question here, with no solution: https://discord.com/channels/464786597288738816/1088162780537954406/1088162780537954406 Okay figured it out now: 
  @impl true
  def handle_init(_ctx, _path) do
    spec = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
    ]

    {[spec: spec], %{}}
  end
  @impl true
  def handle_init(_ctx, _path) do
    spec = [
      child(:video_source, %Membrane.File.Source{
        location: @input_file
      })
      |> child(:demuxer, Membrane.MP4.Demuxer.ISOM)
    ]

    {[spec: spec], %{}}
  end
  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.flat_map(fn
        {track_id, %Membrane.H264{}} ->
          vid =
            get_child(:demuxer)
            |> via_out(Pad.ref(:output, track_id))
            |> do_video()

          [vid]

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    # Mux to MP4
    muxer =
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})

    {[spec: [muxer | spec]], state}
  end
  @impl true
  def handle_child_notification({:new_tracks, tracks}, :demuxer, _ctx, state) do
    spec =
      tracks
      |> Enum.flat_map(fn
        {track_id, %Membrane.H264{}} ->
          vid =
            get_child(:demuxer)
            |> via_out(Pad.ref(:output, track_id))
            |> do_video()

          [vid]

        {track_id, %Membrane.AAC{}} ->
          get_child(:demuxer)
          |> via_out(Pad.ref(:output, track_id))
          |> do_audio()
      end)

    # Mux to MP4
    muxer =
      child(:muxer, Membrane.MP4.Muxer.ISOM)
      |> child(:sink, %Membrane.File.Sink{location: @out_file})

    {[spec: [muxer | spec]], state}
  end
Want results from more Discord servers?
Add your server
More Posts
H264.FFmpeg.Decoded frames to MP4.MuxerI'm attempting to open a local mp4, demux it, and write it back to an mp4, just to get started. I waWiring up Javascript FE Using membrane-webrtc-jsSorry if this obvious. I'm looking through the example in the membrane_rtc_engine (link below). It'sFilter with `push` flow_controlHello, I have a filter that transcribes audio as it receives it by sending it to a transcription serLL-HLS broadcastingHello everyone! I am trying to make LL-HLS broadcasting work. I used the demo from webrtc_to_hls aPipeline children started twiceHello, I'm seeing children in a Membrane pipeline get started twice: I think this might be an issuWriting a `Bin` queuing content from multiple remote files@skillet wrote in https://discord.com/channels/464786597288738816/1007192081107791902/12244914186265Split audio file into 20mb chunksIm trying to figure out how to take the file at this URL, and send it to OpenAI in chunks of 20mb: hbundlex nifs and libasanIs it anyone build nifs with libasan support? Even if I put compiler_flags ["-fno-omit-frame-pointerunifex seg fault on handle_destroy_stateHi, i'm implementing g772.1 decoder/encoder plugin and have issue with handle_destroy state. I've taDeveloping an advanced Jellyfish use caseHey I've been using jellyfish to develop a platform for essentially one-on-one calls between two peo