Membrane.AAC.Parser - Not enough information provided to parse the stream.
Hello, I am receiving audio via my reolink camera and I am trying to send it to an RTMP server.
Here is the info from my camera using ffmpeg:
Here are the relevant parts of the pipeline:
When I run it, I get the following error message:
How do I properly pass the information to the audio parser?
Duration: N/A, start: 0.000250, bitrate: N/A
Stream #0:0: Video: h264 (High), yuv420p(progressive), 2560x1920, 30 fps, 30 tbr, 90k tbn
Stream #0:1: Audio: aac (LC), 16000 Hz, mono, fltp
Duration: N/A, start: 0.000250, bitrate: N/A
Stream #0:0: Video: h264 (High), yuv420p(progressive), 2560x1920, 30 fps, 30 tbr, 90k tbn
Stream #0:1: Audio: aac (LC), 16000 Hz, mono, fltp
def handle_init(_ctx, _opts) do
spec = [
child(:source, %Membrane.RTSP.Source{
transport: :tcp,
allowed_media_types: [:video, :audio],
stream_uri: @rtsp_url
}),
child(:rtmp_sink, %Membrane.RTMP.Sink{rtmp_url: @rtmp_url, tracks: [:video, :audio]}),
child(:audio_depayloader, %Membrane.RTP.AAC.Depayloader{
mode: :lbr
}),
child(:audio_parser, %Membrane.AAC.Parser{
#out_encapsulation: :none
}),
child(:audio_realtimer, Membrane.Realtimer),
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_depayloader)
|> get_child(:audio_parser)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
]
{[spec: spec], %{}}
end
def handle_init(_ctx, _opts) do
spec = [
child(:source, %Membrane.RTSP.Source{
transport: :tcp,
allowed_media_types: [:video, :audio],
stream_uri: @rtsp_url
}),
child(:rtmp_sink, %Membrane.RTMP.Sink{rtmp_url: @rtmp_url, tracks: [:video, :audio]}),
child(:audio_depayloader, %Membrane.RTP.AAC.Depayloader{
mode: :lbr
}),
child(:audio_parser, %Membrane.AAC.Parser{
#out_encapsulation: :none
}),
child(:audio_realtimer, Membrane.Realtimer),
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_depayloader)
|> get_child(:audio_parser)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
]
{[spec: spec], %{}}
end
Evaluation process terminated - {:membrane_child_crash, :audio_parser, {%RuntimeError{message: "Not enough information provided to parse the stream. \nProfile, channels and sample rate need to be provided in one of the following ways:\n - ADTS encapsulation\n - Stream format, either explicitly or through `:config` field\n - `:audio_specific_config` option of this element\nHowever they evaluated to: \n - Profile: nil\n - Channels: nil\n - Sample rate: nil\n"}, [{Membrane.AAC.Parser, :handle_stream_format, 4, [file: ~c"lib/membrane/aac/parser.ex", line: 139, error_info: %{module: Exception}]}, ...
Evaluation process terminated - {:membrane_child_crash, :audio_parser, {%RuntimeError{message: "Not enough information provided to parse the stream. \nProfile, channels and sample rate need to be provided in one of the following ways:\n - ADTS encapsulation\n - Stream format, either explicitly or through `:config` field\n - `:audio_specific_config` option of this element\nHowever they evaluated to: \n - Profile: nil\n - Channels: nil\n - Sample rate: nil\n"}, [{Membrane.AAC.Parser, :handle_stream_format, 4, [file: ~c"lib/membrane/aac/parser.ex", line: 139, error_info: %{module: Exception}]}, ...
9 Replies
I went and asked ChatGPT and it suggest to use
:hbr for the :audio_depayloader and audio_specific_config: <<0x12, 0x08>> for the aac parser. Now the stream is working but I can't hear any audio. There surely is audio as I am able to hear birds and cars over the reolink app. Any ideas?Hello @phillipp ! I would say that the
Membrane.RTSP.Source (which is a bin, meaning that it "wraps" some elements under the hood) should spawn the appropriate depayloader and parser on its own, and pass to them information like audio_specific_config read from the exchanged RTSP messages.
Have you tried removing the audio_depayloader from your pipeline? (the parser might still be needed to change the encapsulation so that it matches RTMP capabilities)@varsill
I messed around a bit more with ChatGPT and it suggested me to resample the audio and that did the trick. The below pipeline is working for me.
spec = [
child(:source, %Membrane.RTSP.Source{
transport: :tcp,
allowed_media_types: [:video, :audio],
stream_uri: @rtsp_url
}),
child(:video_parser, %Membrane.H264.Parser{
output_stream_structure: :avc3,
}),
child(:video_realtimer, Membrane.Realtimer),
child(:rtmp_sink, %Membrane.RTMP.Sink{rtmp_url: @rtmp_url, tracks: [:video, :audio]}),
child(:audio_depayloader, %Membrane.RTP.AAC.Depayloader{
mode: :hbr
}),
child(:audio_parser, %Membrane.AAC.Parser{
out_encapsulation: :ADTS,
audio_specific_config: AACConfigHelper.generate(:aac_lc, 16000, 1)
}),
child(:aac_decoder, Membrane.AAC.FDK.Decoder),
child(:audio_resampler, %Membrane.FFmpeg.SWResample.Converter{
output_stream_format: %Membrane.RawAudio{channels: 2, sample_format: :s16le, sample_rate: 44_100}
}),
child(:aac_encoder, Membrane.AAC.FDK.Encoder),
child(:audio_realtimer, Membrane.Realtimer),
get_child(:source)
|> via_out(Pad.ref(:output, "track1"))
|> get_child(:video_parser)
|> get_child(:video_realtimer)
|> via_in(Pad.ref(:video, 0))
|> get_child(:rtmp_sink),
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_depayloader)
|> get_child(:audio_parser)
|> get_child(:aac_decoder)
|> get_child(:audio_resampler)
|> get_child(:aac_encoder)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
]
spec = [
child(:source, %Membrane.RTSP.Source{
transport: :tcp,
allowed_media_types: [:video, :audio],
stream_uri: @rtsp_url
}),
child(:video_parser, %Membrane.H264.Parser{
output_stream_structure: :avc3,
}),
child(:video_realtimer, Membrane.Realtimer),
child(:rtmp_sink, %Membrane.RTMP.Sink{rtmp_url: @rtmp_url, tracks: [:video, :audio]}),
child(:audio_depayloader, %Membrane.RTP.AAC.Depayloader{
mode: :hbr
}),
child(:audio_parser, %Membrane.AAC.Parser{
out_encapsulation: :ADTS,
audio_specific_config: AACConfigHelper.generate(:aac_lc, 16000, 1)
}),
child(:aac_decoder, Membrane.AAC.FDK.Decoder),
child(:audio_resampler, %Membrane.FFmpeg.SWResample.Converter{
output_stream_format: %Membrane.RawAudio{channels: 2, sample_format: :s16le, sample_rate: 44_100}
}),
child(:aac_encoder, Membrane.AAC.FDK.Encoder),
child(:audio_realtimer, Membrane.Realtimer),
get_child(:source)
|> via_out(Pad.ref(:output, "track1"))
|> get_child(:video_parser)
|> get_child(:video_realtimer)
|> via_in(Pad.ref(:video, 0))
|> get_child(:rtmp_sink),
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_depayloader)
|> get_child(:audio_parser)
|> get_child(:aac_decoder)
|> get_child(:audio_resampler)
|> get_child(:aac_encoder)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
]
Good to hear that it's working for you! If you have time, you can try removing that depayloader though (and the resampling part too) as it's possible that the stream can be properly handled with its native sampling frequency etc. - a benefit coming with such an approach is that it would be less resource-consuming.
@varsill So, I left out the depayloader and the resampling part. The audio pipeline looks like this now:
and i am getting:
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_parser)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
get_child(:source)
|> via_out(Pad.ref(:output, "track2"))
|> get_child(:audio_parser)
|> get_child(:audio_realtimer)
|> via_in(Pad.ref(:audio, 0))
|> get_child(:rtmp_sink)
Evaluation process terminated - {:membrane_child_crash, :audio_parser, {%Membrane.StreamFormatError{message: "Stream format: %Membrane.RTP{payload_format: nil} is not matching accepted format pattern `AAC` or `Membrane.RemoteStream` in def_input_pad for pad :input in Membrane.AAC.Parser. Beware that stream format is forwarded by the default implemetation of handle_stream_format in filters.\n"}, [{Membrane.Core.Element.StreamFormatController, :"-validate_stream_format!/3-fun-1-", 4, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 120]}, {Enum, :"-reduce/3-lists^foldl/2-0-", 3, [file: ~c"lib/enum.ex", line: 2546]}, {Membrane.Core.Element.StreamFormatController, :validate_stream_format!, 3, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 105]}, {Membrane.Core.Element.StreamFormatController, :exec_handle_stream_format, 4, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 77]}, {Membrane.Core.Element, :handle_info, 2, [file: ~c"lib/membrane/core/element.ex", line: 235]}, {:gen_server, :try_handle_info, 3, [file: ~c"gen_server.erl", line: 2345]}, {:gen_server, :handle_msg, 6, [file: ~c"gen_server.erl", line: 2433]}, {:proc_lib, :init_p_do_apply, 3, [file: ~c"proc_lib.erl", line: 329]}]}}
Evaluation process terminated - {:membrane_child_crash, :audio_parser, {%Membrane.StreamFormatError{message: "Stream format: %Membrane.RTP{payload_format: nil} is not matching accepted format pattern `AAC` or `Membrane.RemoteStream` in def_input_pad for pad :input in Membrane.AAC.Parser. Beware that stream format is forwarded by the default implemetation of handle_stream_format in filters.\n"}, [{Membrane.Core.Element.StreamFormatController, :"-validate_stream_format!/3-fun-1-", 4, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 120]}, {Enum, :"-reduce/3-lists^foldl/2-0-", 3, [file: ~c"lib/enum.ex", line: 2546]}, {Membrane.Core.Element.StreamFormatController, :validate_stream_format!, 3, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 105]}, {Membrane.Core.Element.StreamFormatController, :exec_handle_stream_format, 4, [file: ~c"lib/membrane/core/element/stream_format_controller.ex", line: 77]}, {Membrane.Core.Element, :handle_info, 2, [file: ~c"lib/membrane/core/element.ex", line: 235]}, {:gen_server, :try_handle_info, 3, [file: ~c"gen_server.erl", line: 2345]}, {:gen_server, :handle_msg, 6, [file: ~c"gen_server.erl", line: 2433]}, {:proc_lib, :init_p_do_apply, 3, [file: ~c"proc_lib.erl", line: 329]}]}}
Hi @phillipp
What do you get as tracks when you receive this message
{:set_up_tracks, tracks} ?@Billal this are the tracks:
audio:
video:
had to split it up because of discord character limit
[
%{
type: :audio,
rtpmap: %ExSDP.Attribute.RTPMapping{
payload_type: 97,
encoding: "MPEG4-GENERIC",
clock_rate: 16000,
params: 1
},
fmtp: %ExSDP.Attribute.FMTP{
pt: 97,
profile_level_id: 1,
level_asymmetry_allowed: nil,
packetization_mode: nil,
max_mbps: nil,
max_smbps: nil,
max_fs: nil,
max_dpb: nil,
max_br: nil,
sprop_parameter_sets: nil,
profile_space: nil,
profile_id: nil,
tier_flag: nil,
level_id: nil,
interop_constraints: nil,
sprop_vps: nil,
sprop_sps: nil,
sprop_pps: nil,
maxaveragebitrate: nil,
maxplaybackrate: nil,
sprop_maxcapturerate: nil,
maxptime: nil,
ptime: nil,
minptime: nil,
stereo: nil,
cbr: nil,
useinbandfec: nil,
usedtx: nil,
max_fr: nil,
profile: nil,
level_idx: nil,
tier: nil,
apt: nil,
rtx_time: nil,
repair_window: nil,
dtmf_tones: nil,
redundant_payloads: nil,
bitrate: nil,
streamtype: 5,
config: <<20, 8>>,
mode: :AAC_hbr,
objecttype: nil,
constantsize: nil,
constantduration: nil,
maxdisplacement: nil,
de_interleavebuffersize: nil,
...
},
control_path: "track2",
framerate: nil
}
]
[
%{
type: :audio,
rtpmap: %ExSDP.Attribute.RTPMapping{
payload_type: 97,
encoding: "MPEG4-GENERIC",
clock_rate: 16000,
params: 1
},
fmtp: %ExSDP.Attribute.FMTP{
pt: 97,
profile_level_id: 1,
level_asymmetry_allowed: nil,
packetization_mode: nil,
max_mbps: nil,
max_smbps: nil,
max_fs: nil,
max_dpb: nil,
max_br: nil,
sprop_parameter_sets: nil,
profile_space: nil,
profile_id: nil,
tier_flag: nil,
level_id: nil,
interop_constraints: nil,
sprop_vps: nil,
sprop_sps: nil,
sprop_pps: nil,
maxaveragebitrate: nil,
maxplaybackrate: nil,
sprop_maxcapturerate: nil,
maxptime: nil,
ptime: nil,
minptime: nil,
stereo: nil,
cbr: nil,
useinbandfec: nil,
usedtx: nil,
max_fr: nil,
profile: nil,
level_idx: nil,
tier: nil,
apt: nil,
rtx_time: nil,
repair_window: nil,
dtmf_tones: nil,
redundant_payloads: nil,
bitrate: nil,
streamtype: 5,
config: <<20, 8>>,
mode: :AAC_hbr,
objecttype: nil,
constantsize: nil,
constantduration: nil,
maxdisplacement: nil,
de_interleavebuffersize: nil,
...
},
control_path: "track2",
framerate: nil
}
]
[
%{
type: :video,
rtpmap: %ExSDP.Attribute.RTPMapping{
payload_type: 96,
encoding: "H264",
clock_rate: 90000,
params: nil
},
fmtp: %ExSDP.Attribute.FMTP{
pt: 96,
profile_level_id: 6553650,
level_asymmetry_allowed: nil,
packetization_mode: 1,
max_mbps: nil,
max_smbps: nil,
max_fs: nil,
max_dpb: nil,
max_br: nil,
sprop_parameter_sets: %{
sps: <<103, 100, 0, 50, 172, 210, 0, 160, 3, 198, 132, 0, 0, 15, 164, 0,
3, 170, 112, 16>>,
pps: <<104, 234, 143, 44>>
},
profile_space: nil,
profile_id: nil,
tier_flag: nil,
level_id: nil,
interop_constraints: nil,
sprop_vps: nil,
sprop_sps: nil,
sprop_pps: nil,
maxaveragebitrate: nil,
maxplaybackrate: nil,
sprop_maxcapturerate: nil,
maxptime: nil,
ptime: nil,
minptime: nil,
stereo: nil,
cbr: nil,
useinbandfec: nil,
usedtx: nil,
max_fr: nil,
profile: nil,
level_idx: nil,
tier: nil,
apt: nil,
rtx_time: nil,
repair_window: nil,
dtmf_tones: nil,
redundant_payloads: nil,
bitrate: nil,
streamtype: nil,
config: nil,
mode: nil,
objecttype: nil,
constantsize: nil,
constantduration: nil,
maxdisplacement: nil,
...
},
control_path: "track1",
framerate: nil
}
]
[
%{
type: :video,
rtpmap: %ExSDP.Attribute.RTPMapping{
payload_type: 96,
encoding: "H264",
clock_rate: 90000,
params: nil
},
fmtp: %ExSDP.Attribute.FMTP{
pt: 96,
profile_level_id: 6553650,
level_asymmetry_allowed: nil,
packetization_mode: 1,
max_mbps: nil,
max_smbps: nil,
max_fs: nil,
max_dpb: nil,
max_br: nil,
sprop_parameter_sets: %{
sps: <<103, 100, 0, 50, 172, 210, 0, 160, 3, 198, 132, 0, 0, 15, 164, 0,
3, 170, 112, 16>>,
pps: <<104, 234, 143, 44>>
},
profile_space: nil,
profile_id: nil,
tier_flag: nil,
level_id: nil,
interop_constraints: nil,
sprop_vps: nil,
sprop_sps: nil,
sprop_pps: nil,
maxaveragebitrate: nil,
maxplaybackrate: nil,
sprop_maxcapturerate: nil,
maxptime: nil,
ptime: nil,
minptime: nil,
stereo: nil,
cbr: nil,
useinbandfec: nil,
usedtx: nil,
max_fr: nil,
profile: nil,
level_idx: nil,
tier: nil,
apt: nil,
rtx_time: nil,
repair_window: nil,
dtmf_tones: nil,
redundant_payloads: nil,
bitrate: nil,
streamtype: nil,
config: nil,
mode: nil,
objecttype: nil,
constantsize: nil,
constantduration: nil,
maxdisplacement: nil,
...
},
control_path: "track1",
framerate: nil
}
]
Ok, I see the issue.
I'll try to fix it and create a new release by tomorrow
@phillipp try the new version
@Billal yeah, now it works. thanks!