Added
Link Here
|
0 |
- |
1 |
From 569e6df7ac64e586c7cbdc0ccd2ccf7d34f6984d Mon Sep 17 00:00:00 2001 |
|
|
2 |
From: Evgeniy Khramtsov <evgeniy@khramtsov.org> |
3 |
Date: Wed, 8 Sep 2021 21:37:04 +0300 |
4 |
Subject: [PATCH] Revert bug 1702646 |
5 |
|
6 |
Revert: |
7 |
https://github.com/mozilla/gecko-dev/commit/0a9065018a56a7812b15411051143c2c8f7b1a08 |
8 |
https://github.com/mozilla/gecko-dev/commit/d2f2ea20bbc9e1e7bc6858ea19d689624f27055a |
9 |
https://github.com/mozilla/gecko-dev/commit/c95e8979b6b673658cdc51313d01db388bc3ff5d |
10 |
https://github.com/mozilla/gecko-dev/commit/1f72372ece815dd5c59e9a34ca0754ea84124713 |
11 |
https://github.com/mozilla/gecko-dev/commit/81d623f7fc5f4e9acbe31203bc9a0868078dbe09 |
12 |
|
13 |
See: |
14 |
https://bugzilla.mozilla.org/show_bug.cgi?id=1702646 |
15 |
https://bugzilla.mozilla.org/show_bug.cgi?id=1725810 |
16 |
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=257639 |
17 |
--- |
18 |
dom/media/AudioSegment.h | 45 --- |
19 |
dom/media/MediaTrackGraph.cpp | 369 +++---------------- |
20 |
dom/media/MediaTrackGraph.h | 16 +- |
21 |
dom/media/MediaTrackGraphImpl.h | 123 +------ |
22 |
dom/media/gtest/TestAudioInputProcessing.cpp | 10 +- |
23 |
dom/media/webrtc/MediaEngineWebRTCAudio.cpp | 127 +++---- |
24 |
dom/media/webrtc/MediaEngineWebRTCAudio.h | 19 +- |
25 |
7 files changed, 133 insertions(+), 576 deletions(-) |
26 |
|
27 |
diff --git dom/media/AudioSegment.h dom/media/AudioSegment.h |
28 |
index 4ec71c92bc..a0aee574e5 100644 |
29 |
--- dom/media/AudioSegment.h |
30 |
+++ dom/media/AudioSegment.h |
31 |
@@ -377,51 +377,6 @@ class AudioSegment : public MediaSegmentBase<AudioSegment, AudioChunk> { |
32 |
chunk->mBufferFormat = AUDIO_FORMAT_S16; |
33 |
chunk->mPrincipalHandle = aPrincipalHandle; |
34 |
} |
35 |
- void AppendSegment(const AudioSegment* aSegment, |
36 |
- const PrincipalHandle& aPrincipalHandle) { |
37 |
- MOZ_ASSERT(aSegment); |
38 |
- |
39 |
- for (const AudioChunk& c : aSegment->mChunks) { |
40 |
- AudioChunk* chunk = AppendChunk(c.GetDuration()); |
41 |
- chunk->mBuffer = c.mBuffer; |
42 |
- chunk->mChannelData = c.mChannelData; |
43 |
- chunk->mBufferFormat = c.mBufferFormat; |
44 |
- chunk->mPrincipalHandle = aPrincipalHandle; |
45 |
- } |
46 |
- } |
47 |
- template <typename T> |
48 |
- void AppendFromInterleavedBuffer(const T* aBuffer, size_t aFrames, |
49 |
- uint32_t aChannels, |
50 |
- const PrincipalHandle& aPrincipalHandle) { |
51 |
- MOZ_ASSERT(aChannels >= 1 && aChannels <= 8, "Support up to 8 channels"); |
52 |
- |
53 |
- CheckedInt<size_t> bufferSize(sizeof(T)); |
54 |
- bufferSize *= aFrames; |
55 |
- bufferSize *= aChannels; |
56 |
- RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize); |
57 |
- AutoTArray<const T*, 8> channels; |
58 |
- if (aChannels == 1) { |
59 |
- PodCopy(static_cast<T*>(buffer->Data()), aBuffer, aFrames); |
60 |
- channels.AppendElement(static_cast<T*>(buffer->Data())); |
61 |
- } else { |
62 |
- channels.SetLength(aChannels); |
63 |
- AutoTArray<T*, 8> writeChannels; |
64 |
- writeChannels.SetLength(aChannels); |
65 |
- T* samples = static_cast<T*>(buffer->Data()); |
66 |
- |
67 |
- size_t offset = 0; |
68 |
- for (uint32_t i = 0; i < aChannels; ++i) { |
69 |
- channels[i] = writeChannels[i] = samples + offset; |
70 |
- offset += aFrames; |
71 |
- } |
72 |
- |
73 |
- DeinterleaveAndConvertBuffer(aBuffer, aFrames, aChannels, |
74 |
- writeChannels.Elements()); |
75 |
- } |
76 |
- |
77 |
- MOZ_ASSERT(aChannels == channels.Length()); |
78 |
- AppendFrames(buffer.forget(), channels, aFrames, aPrincipalHandle); |
79 |
- } |
80 |
// Consumes aChunk, and returns a pointer to the persistent copy of aChunk |
81 |
// in the segment. |
82 |
AudioChunk* AppendAndConsumeChunk(AudioChunk&& aChunk) { |
83 |
diff --git dom/media/MediaTrackGraph.cpp dom/media/MediaTrackGraph.cpp |
84 |
index 3b5d376028..14f918558d 100644 |
85 |
--- dom/media/MediaTrackGraph.cpp |
86 |
+++ dom/media/MediaTrackGraph.cpp |
87 |
@@ -62,157 +62,6 @@ LazyLogModule gMediaTrackGraphLog("MediaTrackGraph"); |
88 |
*/ |
89 |
static nsTHashMap<nsUint32HashKey, MediaTrackGraphImpl*> gGraphs; |
90 |
|
91 |
-void NativeInputTrack::AudioDataBuffers::SetOutputData(AudioDataValue* aBuffer, |
92 |
- size_t aFrames, |
93 |
- uint32_t aChannels, |
94 |
- TrackRate aRate) { |
95 |
- mOutputData = Some(BufferInfo{aBuffer, aFrames, aChannels, aRate}); |
96 |
-} |
97 |
- |
98 |
-void NativeInputTrack::AudioDataBuffers::SetInputData(AudioDataValue* aBuffer, |
99 |
- size_t aFrames, |
100 |
- uint32_t aChannels, |
101 |
- TrackRate aRate) { |
102 |
- mInputData = Some(BufferInfo{aBuffer, aFrames, aChannels, aRate}); |
103 |
-} |
104 |
- |
105 |
-void NativeInputTrack::AudioDataBuffers::Clear(Scope aScope) { |
106 |
- if (aScope & Scope::Input) { |
107 |
- mInputData.take(); |
108 |
- } |
109 |
- |
110 |
- if (aScope & Scope::Output) { |
111 |
- mOutputData.take(); |
112 |
- } |
113 |
-} |
114 |
- |
115 |
-NativeInputTrack* NativeInputTrack::Create(MediaTrackGraphImpl* aGraph) { |
116 |
- MOZ_ASSERT(NS_IsMainThread()); |
117 |
- |
118 |
- NativeInputTrack* track = new NativeInputTrack(aGraph->GraphRate()); |
119 |
- aGraph->AddTrack(track); |
120 |
- return track; |
121 |
-} |
122 |
- |
123 |
-size_t NativeInputTrack::AddUser() { |
124 |
- MOZ_ASSERT(NS_IsMainThread()); |
125 |
- mUserCount += 1; |
126 |
- return mUserCount; |
127 |
-} |
128 |
- |
129 |
-size_t NativeInputTrack::RemoveUser() { |
130 |
- MOZ_ASSERT(NS_IsMainThread()); |
131 |
- MOZ_ASSERT(mUserCount > 0); |
132 |
- mUserCount -= 1; |
133 |
- return mUserCount; |
134 |
-} |
135 |
- |
136 |
-void NativeInputTrack::DestroyImpl() { |
137 |
- MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning()); |
138 |
- if (mDataHolder) { |
139 |
- mDataHolder->Clear(static_cast<AudioDataBuffers::Scope>( |
140 |
- AudioDataBuffers::Scope::Input | AudioDataBuffers::Scope::Output)); |
141 |
- } |
142 |
- ProcessedMediaTrack::DestroyImpl(); |
143 |
-} |
144 |
- |
145 |
-void NativeInputTrack::ProcessInput(GraphTime aFrom, GraphTime aTo, |
146 |
- uint32_t aFlags) { |
147 |
- MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning()); |
148 |
- TRACE_COMMENT("NativeInputTrack %p", this); |
149 |
- |
150 |
- if (!mDataHolder || !mDataHolder->mInputData) { |
151 |
- return; |
152 |
- } |
153 |
- |
154 |
- // One NotifyInputData might have multiple following ProcessInput calls, but |
155 |
- // we only process one input per NotifyInputData call. |
156 |
- NativeInputTrack::AudioDataBuffers::BufferInfo inputInfo = |
157 |
- mDataHolder->mInputData.extract(); |
158 |
- |
159 |
- MOZ_ASSERT(mInputChannels == inputInfo.mChannels); |
160 |
- MOZ_ASSERT(inputInfo.mChannels >= 1 && inputInfo.mChannels <= 8, |
161 |
- "Support up to 8 channels"); |
162 |
- |
163 |
- GetData<AudioSegment>()->Clear(); |
164 |
- GetData<AudioSegment>()->AppendFromInterleavedBuffer( |
165 |
- inputInfo.mBuffer, inputInfo.mFrames, inputInfo.mChannels, |
166 |
- PRINCIPAL_HANDLE_NONE); |
167 |
- |
168 |
- LOG(LogLevel::Verbose, |
169 |
- ("NativeInputTrack %p Appending %zu frames of raw audio", this, |
170 |
- inputInfo.mFrames)); |
171 |
-} |
172 |
- |
173 |
-uint32_t NativeInputTrack::NumberOfChannels() const { |
174 |
- MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning()); |
175 |
- return mInputChannels; |
176 |
-} |
177 |
- |
178 |
-void NativeInputTrack::InitDataHolderIfNeeded() { |
179 |
- MOZ_ASSERT(mGraph->OnGraphThreadOrNotRunning()); |
180 |
- if (!mDataHolder) { |
181 |
- mDataHolder.emplace(); |
182 |
- } |
183 |
-} |
184 |
- |
185 |
-void NativeInputTrack::NotifyOutputData(MediaTrackGraphImpl* aGraph, |
186 |
- AudioDataValue* aBuffer, size_t aFrames, |
187 |
- TrackRate aRate, uint32_t aChannels) { |
188 |
- MOZ_ASSERT(aGraph->OnGraphThreadOrNotRunning()); |
189 |
- MOZ_ASSERT(aGraph == mGraph, "Receive output data from another graph"); |
190 |
- MOZ_ASSERT(mDataHolder); |
191 |
- mDataHolder->SetOutputData(aBuffer, aFrames, aChannels, aRate); |
192 |
- for (auto& listener : mDataUsers) { |
193 |
- listener->NotifyOutputData(aGraph, mDataHolder->mOutputData.value()); |
194 |
- } |
195 |
-} |
196 |
- |
197 |
-void NativeInputTrack::NotifyInputStopped(MediaTrackGraphImpl* aGraph) { |
198 |
- MOZ_ASSERT(aGraph->OnGraphThreadOrNotRunning()); |
199 |
- MOZ_ASSERT(aGraph == mGraph, |
200 |
- "Receive input stopped signal from another graph"); |
201 |
- MOZ_ASSERT(mDataHolder); |
202 |
- mInputChannels = 0; |
203 |
- mDataHolder->Clear(AudioDataBuffers::Scope::Input); |
204 |
- for (auto& listener : mDataUsers) { |
205 |
- listener->NotifyInputStopped(aGraph); |
206 |
- } |
207 |
-} |
208 |
- |
209 |
-void NativeInputTrack::NotifyInputData(MediaTrackGraphImpl* aGraph, |
210 |
- const AudioDataValue* aBuffer, |
211 |
- size_t aFrames, TrackRate aRate, |
212 |
- uint32_t aChannels, |
213 |
- uint32_t aAlreadyBuffered) { |
214 |
- MOZ_ASSERT(aGraph->OnGraphThreadOrNotRunning()); |
215 |
- MOZ_ASSERT(aGraph == mGraph, "Receive input data from another graph"); |
216 |
- |
217 |
- MOZ_ASSERT(mDataHolder); |
218 |
- MOZ_ASSERT(aChannels); |
219 |
- if (!mInputChannels) { |
220 |
- mInputChannels = aChannels; |
221 |
- } |
222 |
- mDataHolder->SetInputData(const_cast<AudioDataValue*>(aBuffer), aFrames, |
223 |
- aChannels, aRate); |
224 |
- for (auto& listener : mDataUsers) { |
225 |
- listener->NotifyInputData(aGraph, mDataHolder->mInputData.value(), |
226 |
- aAlreadyBuffered); |
227 |
- } |
228 |
-} |
229 |
- |
230 |
-void NativeInputTrack::DeviceChanged(MediaTrackGraphImpl* aGraph) { |
231 |
- MOZ_ASSERT(aGraph->OnGraphThreadOrNotRunning()); |
232 |
- MOZ_ASSERT(aGraph == mGraph, |
233 |
- "Receive device changed signal from another graph"); |
234 |
- MOZ_ASSERT(mDataHolder); |
235 |
- mDataHolder->Clear(static_cast<AudioDataBuffers::Scope>( |
236 |
- AudioDataBuffers::Scope::Input | AudioDataBuffers::Scope::Output)); |
237 |
- for (auto& listener : mDataUsers) { |
238 |
- listener->DeviceChanged(aGraph); |
239 |
- } |
240 |
-} |
241 |
- |
242 |
MediaTrackGraphImpl::~MediaTrackGraphImpl() { |
243 |
MOZ_ASSERT(mTracks.IsEmpty() && mSuspendedTracks.IsEmpty(), |
244 |
"All tracks should have been destroyed by messages from the main " |
245 |
@@ -430,7 +279,7 @@ bool MediaTrackGraphImpl::AudioTrackPresent() { |
246 |
// XXX For some reason, there are race conditions when starting an audio input |
247 |
// where we find no active audio tracks. In any case, if we have an active |
248 |
// audio input we should not allow a switch back to a SystemClockDriver |
249 |
- if (!audioTrackPresent && mDeviceTrackMap.Count() != 0) { |
250 |
+ if (!audioTrackPresent && mInputDeviceUsers.Count() != 0) { |
251 |
NS_WARNING("No audio tracks, but full-duplex audio is enabled!!!!!"); |
252 |
audioTrackPresent = true; |
253 |
} |
254 |
@@ -779,51 +628,23 @@ TrackTime MediaTrackGraphImpl::PlayAudio(AudioMixer* aMixer, |
255 |
return ticksWritten; |
256 |
} |
257 |
|
258 |
-ProcessedMediaTrack* MediaTrackGraphImpl::GetDeviceTrack( |
259 |
- CubebUtils::AudioDeviceID aID) { |
260 |
- MOZ_ASSERT(NS_IsMainThread()); |
261 |
- |
262 |
- RefPtr<NativeInputTrack>& t = mDeviceTracks.LookupOrInsertWith( |
263 |
- aID, [self = RefPtr<MediaTrackGraphImpl>(this), aID] { |
264 |
- NativeInputTrack* track = NativeInputTrack::Create(self); |
265 |
- LOG(LogLevel::Debug, |
266 |
- ("Create NativeInputTrack %p for device %p", track, aID)); |
267 |
- return do_AddRef(track); |
268 |
- }); |
269 |
- |
270 |
- return t.get(); |
271 |
-} |
272 |
- |
273 |
void MediaTrackGraphImpl::OpenAudioInputImpl(CubebUtils::AudioDeviceID aID, |
274 |
- AudioDataListener* aListener, |
275 |
- NativeInputTrack* aInputTrack) { |
276 |
+ AudioDataListener* aListener) { |
277 |
MOZ_ASSERT(OnGraphThread()); |
278 |
- LOG(LogLevel::Debug, |
279 |
- ("%p OpenAudioInputImpl: NativeInputTrack %p for device %p", this, |
280 |
- aInputTrack, aID)); |
281 |
- |
282 |
- if (mDeviceTrackMap.Count() > 0 && !mDeviceTrackMap.Get(aID, nullptr)) { |
283 |
+ // Only allow one device per MTG (hence, per document), but allow opening a |
284 |
+ // device multiple times |
285 |
+ nsTArray<RefPtr<AudioDataListener>>& listeners = |
286 |
+ mInputDeviceUsers.LookupOrInsert(aID); |
287 |
+ if (listeners.IsEmpty() && mInputDeviceUsers.Count() > 1) { |
288 |
// We don't support opening multiple input device in a graph for now. |
289 |
- LOG(LogLevel::Debug, ("%p Device %p is not native device. Cannot open %p!", |
290 |
- this, aID, aInputTrack)); |
291 |
+ listeners.RemoveElement(aID); |
292 |
return; |
293 |
} |
294 |
|
295 |
- LOG(LogLevel::Debug, |
296 |
- ("%p Device %p is native device. Open %p", this, aID, aInputTrack)); |
297 |
- |
298 |
- // Only allow one device per MTG (hence, per document), but allow opening a |
299 |
- // device multiple times |
300 |
- NativeInputTrack* track = mDeviceTrackMap.LookupOrInsertWith( |
301 |
- aID, [inputTrack = RefPtr<NativeInputTrack>(aInputTrack)] { |
302 |
- return inputTrack.get(); |
303 |
- }); |
304 |
- MOZ_ASSERT(track); |
305 |
- track->InitDataHolderIfNeeded(); |
306 |
- |
307 |
- nsTArray<RefPtr<AudioDataListener>>& listeners = track->mDataUsers; |
308 |
MOZ_ASSERT(!listeners.Contains(aListener), "Don't add a listener twice."); |
309 |
+ |
310 |
listeners.AppendElement(aListener); |
311 |
+ |
312 |
if (listeners.Length() == 1) { // first open for this device |
313 |
mInputDeviceID = aID; |
314 |
// Switch Drivers since we're adding input (to input-only or full-duplex) |
315 |
@@ -844,60 +665,28 @@ nsresult MediaTrackGraphImpl::OpenAudioInput(CubebUtils::AudioDeviceID aID, |
316 |
class Message : public ControlMessage { |
317 |
public: |
318 |
Message(MediaTrackGraphImpl* aGraph, CubebUtils::AudioDeviceID aID, |
319 |
- AudioDataListener* aListener, NativeInputTrack* aInputTrack) |
320 |
+ AudioDataListener* aListener) |
321 |
: ControlMessage(nullptr), |
322 |
mGraph(aGraph), |
323 |
mID(aID), |
324 |
- mListener(aListener), |
325 |
- mInputTrack(aInputTrack) {} |
326 |
- void Run() override { |
327 |
- mGraph->OpenAudioInputImpl(mID, mListener, mInputTrack); |
328 |
- } |
329 |
+ mListener(aListener) {} |
330 |
+ void Run() override { mGraph->OpenAudioInputImpl(mID, mListener); } |
331 |
MediaTrackGraphImpl* mGraph; |
332 |
CubebUtils::AudioDeviceID mID; |
333 |
RefPtr<AudioDataListener> mListener; |
334 |
- NativeInputTrack* mInputTrack; |
335 |
}; |
336 |
- |
337 |
- auto result = mDeviceTracks.Lookup(aID); |
338 |
- MOZ_ASSERT(result); |
339 |
- MOZ_ASSERT(result.Data()); |
340 |
- size_t users = result.Data()->AddUser(); |
341 |
- |
342 |
- LOG(LogLevel::Debug, |
343 |
- ("%p OpenInput: NativeInputTrack %p for device %p has %zu users now", |
344 |
- this, result.Data().get(), aID, users)); |
345 |
- |
346 |
// XXX Check not destroyed! |
347 |
- this->AppendMessage( |
348 |
- MakeUnique<Message>(this, aID, aListener, result.Data().get())); |
349 |
+ this->AppendMessage(MakeUnique<Message>(this, aID, aListener)); |
350 |
return NS_OK; |
351 |
} |
352 |
|
353 |
void MediaTrackGraphImpl::CloseAudioInputImpl(CubebUtils::AudioDeviceID aID, |
354 |
- AudioDataListener* aListener, |
355 |
- NativeInputTrack* aInputTrack) { |
356 |
+ AudioDataListener* aListener) { |
357 |
MOZ_ASSERT(OnGraphThread()); |
358 |
|
359 |
- LOG(LogLevel::Debug, |
360 |
- ("%p CloseAudioInputImpl: NativeInputTrack %p for device %p", this, |
361 |
- aInputTrack, aID)); |
362 |
- |
363 |
- auto result = mDeviceTrackMap.Lookup(aID); |
364 |
- if (!result) { |
365 |
- LOG(LogLevel::Debug, |
366 |
- ("%p Device %p is not native device. Do nothing for %p", this, aID, |
367 |
- aInputTrack)); |
368 |
- return; |
369 |
- } |
370 |
- |
371 |
- LOG(LogLevel::Debug, |
372 |
- ("%p Device %p is native device. Close %p", this, aID, aInputTrack)); |
373 |
- |
374 |
- NativeInputTrack* track = result.Data(); |
375 |
- MOZ_ASSERT(track == aInputTrack); |
376 |
- nsTArray<RefPtr<AudioDataListener>>& listeners = track->mDataUsers; |
377 |
- bool wasPresent = listeners.RemoveElement(aListener); |
378 |
+ auto listeners = mInputDeviceUsers.Lookup(aID); |
379 |
+ MOZ_ASSERT(listeners); |
380 |
+ bool wasPresent = listeners->RemoveElement(aListener); |
381 |
MOZ_ASSERT(wasPresent); |
382 |
|
383 |
if (wasPresent) { |
384 |
@@ -907,22 +696,13 @@ void MediaTrackGraphImpl::CloseAudioInputImpl(CubebUtils::AudioDeviceID aID, |
385 |
// Breaks the cycle between the MTG and the listener. |
386 |
aListener->Disconnect(this); |
387 |
|
388 |
- if (!listeners.IsEmpty()) { |
389 |
- LOG(LogLevel::Debug, |
390 |
- ("%p NativeInputTrack %p for device %p still has consumer", this, track, |
391 |
- aID)); |
392 |
+ if (!listeners->IsEmpty()) { |
393 |
+ // There is still a consumer for this audio input device |
394 |
return; |
395 |
} |
396 |
|
397 |
- LOG(LogLevel::Debug, |
398 |
- ("%p NativeInputTrack %p for device %p has no consumer now", this, track, |
399 |
- aID)); |
400 |
- |
401 |
mInputDeviceID = nullptr; // reset to default |
402 |
- |
403 |
- bool r = mDeviceTrackMap.Remove(aID); |
404 |
- MOZ_ASSERT(r); |
405 |
- Unused << r; |
406 |
+ Unused << mInputDeviceUsers.Remove(aID); |
407 |
|
408 |
// Switch Drivers since we're adding or removing an input (to nothing/system |
409 |
// or output only) |
410 |
@@ -989,55 +769,17 @@ void MediaTrackGraphImpl::CloseAudioInput(CubebUtils::AudioDeviceID aID, |
411 |
class Message : public ControlMessage { |
412 |
public: |
413 |
Message(MediaTrackGraphImpl* aGraph, CubebUtils::AudioDeviceID aID, |
414 |
- AudioDataListener* aListener, NativeInputTrack* aInputTrack) |
415 |
+ AudioDataListener* aListener) |
416 |
: ControlMessage(nullptr), |
417 |
mGraph(aGraph), |
418 |
mID(aID), |
419 |
- mListener(aListener), |
420 |
- mInputTrack(aInputTrack) {} |
421 |
- void Run() override { |
422 |
- mGraph->CloseAudioInputImpl(mID, mListener, mInputTrack); |
423 |
- } |
424 |
+ mListener(aListener) {} |
425 |
+ void Run() override { mGraph->CloseAudioInputImpl(mID, mListener); } |
426 |
MediaTrackGraphImpl* mGraph; |
427 |
CubebUtils::AudioDeviceID mID; |
428 |
RefPtr<AudioDataListener> mListener; |
429 |
- NativeInputTrack* mInputTrack; |
430 |
}; |
431 |
- |
432 |
- auto result = mDeviceTracks.Lookup(aID); |
433 |
- MOZ_ASSERT(result); |
434 |
- MOZ_ASSERT(result.Data()); |
435 |
- size_t users = result.Data()->RemoveUser(); |
436 |
- |
437 |
- LOG(LogLevel::Debug, |
438 |
- ("%p: CloseInput: NativeInputTrack %p for device %p has %zu users now", |
439 |
- this, result.Data().get(), aID, users)); |
440 |
- |
441 |
- this->AppendMessage( |
442 |
- MakeUnique<Message>(this, aID, aListener, result.Data().get())); |
443 |
- |
444 |
- // Remove the NativeInputTrack from mDeviceTracks if no AudioInputTrack needs |
445 |
- // it, so NativeInputTrack::Create can create a new NativeInputTrack when it's |
446 |
- // called for the same aID. The paired value in mDeviceTrackMap will be |
447 |
- // removed later in CloseAudioInputImpl. The NativeInputTrack will still be |
448 |
- // alive after it's removed from mDeviceTracks since AddTrack called via |
449 |
- // NativeInputTrack::Create will call NS_ADDREF to it and it will be alive |
450 |
- // until its NS_RELEASE is called via NativeInputTrack::DestroyImpl(). |
451 |
- // Note that NativeInputTrack::Destroy() must be called after the above |
452 |
- // message is appended so NativeInputTrack::DestroyImpl() will be run after |
453 |
- // CloseAudioInputImpl(). Therefore, the NativeInputTrack will be alive before |
454 |
- // it's removed from mDeviceTrackMap in CloseAudioInputImpl() |
455 |
- if (users == 0) { |
456 |
- LOG(LogLevel::Debug, |
457 |
- ("%p: CloseInput: NativeInputTrack %p for device %p is removed from " |
458 |
- "mDeviceTracks", |
459 |
- this, result.Data().get(), aID)); |
460 |
- |
461 |
- result.Data()->Destroy(); |
462 |
- bool r = mDeviceTracks.Remove(aID); |
463 |
- MOZ_ASSERT(r); |
464 |
- Unused << r; |
465 |
- } |
466 |
+ this->AppendMessage(MakeUnique<Message>(this, aID, aListener)); |
467 |
} |
468 |
|
469 |
// All AudioInput listeners get the same speaker data (at least for now). |
470 |
@@ -1049,7 +791,7 @@ void MediaTrackGraphImpl::NotifyOutputData(AudioDataValue* aBuffer, |
471 |
// device. |
472 |
// The absence of an input consumer is enough to know we need to bail out |
473 |
// here. |
474 |
- if (!mDeviceTrackMap.Contains(mInputDeviceID)) { |
475 |
+ if (!mInputDeviceUsers.Contains(mInputDeviceID)) { |
476 |
return; |
477 |
} |
478 |
#else |
479 |
@@ -1059,16 +801,14 @@ void MediaTrackGraphImpl::NotifyOutputData(AudioDataValue* aBuffer, |
480 |
#endif |
481 |
// When/if we decide to support multiple input devices per graph, this needs |
482 |
// to loop over them. |
483 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
484 |
- MOZ_ASSERT(result); |
485 |
- NativeInputTrack* track = result.Data(); |
486 |
- MOZ_ASSERT(track); |
487 |
- track->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels); |
488 |
+ for (auto& listener : *mInputDeviceUsers.Lookup(mInputDeviceID)) { |
489 |
+ listener->NotifyOutputData(this, aBuffer, aFrames, aRate, aChannels); |
490 |
+ } |
491 |
} |
492 |
|
493 |
void MediaTrackGraphImpl::NotifyInputStopped() { |
494 |
#ifdef ANDROID |
495 |
- if (!mDeviceTrackMap.Contains(mInputDeviceID)) { |
496 |
+ if (!mInputDeviceUsers.Contains(mInputDeviceID)) { |
497 |
return; |
498 |
} |
499 |
#else |
500 |
@@ -1076,11 +816,9 @@ void MediaTrackGraphImpl::NotifyInputStopped() { |
501 |
return; |
502 |
} |
503 |
#endif |
504 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
505 |
- MOZ_ASSERT(result); |
506 |
- NativeInputTrack* track = result.Data(); |
507 |
- MOZ_ASSERT(track); |
508 |
- track->NotifyInputStopped(this); |
509 |
+ for (auto& listener : *mInputDeviceUsers.Lookup(mInputDeviceID)) { |
510 |
+ listener->NotifyInputStopped(this); |
511 |
+ } |
512 |
} |
513 |
|
514 |
void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, |
515 |
@@ -1088,7 +826,7 @@ void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, |
516 |
uint32_t aChannels, |
517 |
uint32_t aAlreadyBuffered) { |
518 |
#ifdef ANDROID |
519 |
- if (!mDeviceTrackMap.Contains(mInputDeviceID)) { |
520 |
+ if (!mInputDeviceUsers.Contains(mInputDeviceID)) { |
521 |
return; |
522 |
} |
523 |
#else |
524 |
@@ -1100,18 +838,17 @@ void MediaTrackGraphImpl::NotifyInputData(const AudioDataValue* aBuffer, |
525 |
return; |
526 |
} |
527 |
#endif |
528 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
529 |
- MOZ_ASSERT(result); |
530 |
- NativeInputTrack* track = result.Data(); |
531 |
- MOZ_ASSERT(track); |
532 |
- track->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels, |
533 |
- aAlreadyBuffered); |
534 |
+ for (auto& listener : *mInputDeviceUsers.Lookup(mInputDeviceID)) { |
535 |
+ listener->NotifyInputData(this, aBuffer, aFrames, aRate, aChannels, |
536 |
+ aAlreadyBuffered); |
537 |
+ } |
538 |
} |
539 |
|
540 |
void MediaTrackGraphImpl::DeviceChangedImpl() { |
541 |
MOZ_ASSERT(OnGraphThread()); |
542 |
+ |
543 |
#ifdef ANDROID |
544 |
- if (!mDeviceTrackMap.Contains(mInputDeviceID)) { |
545 |
+ if (!mInputDeviceUsers.Contains(mInputDeviceID)) { |
546 |
return; |
547 |
} |
548 |
#else |
549 |
@@ -1119,11 +856,10 @@ void MediaTrackGraphImpl::DeviceChangedImpl() { |
550 |
return; |
551 |
} |
552 |
#endif |
553 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
554 |
- MOZ_ASSERT(result); |
555 |
- NativeInputTrack* track = result.Data(); |
556 |
- MOZ_ASSERT(track); |
557 |
- track->DeviceChanged(this); |
558 |
+ |
559 |
+ for (auto& listener : *mInputDeviceUsers.Lookup(mInputDeviceID)) { |
560 |
+ listener->DeviceChanged(this); |
561 |
+ } |
562 |
} |
563 |
|
564 |
void MediaTrackGraphImpl::SetMaxOutputChannelCount(uint32_t aMaxChannelCount) { |
565 |
@@ -1361,10 +1097,6 @@ void MediaTrackGraphImpl::ProduceDataForTracksBlockByBlock( |
566 |
for (uint32_t i = aTrackIndex; i < mTracks.Length(); ++i) { |
567 |
ProcessedMediaTrack* pt = mTracks[i]->AsProcessedTrack(); |
568 |
if (pt) { |
569 |
- if (pt->AsNativeInputTrack()) { |
570 |
- // NativeInputTracks are processed in Process. Skip them. |
571 |
- continue; |
572 |
- } |
573 |
pt->ProcessInput( |
574 |
mProcessedTime, next, |
575 |
(next == mStateComputedTime) ? ProcessedMediaTrack::ALLOW_END : 0); |
576 |
@@ -1509,23 +1241,12 @@ void MediaTrackGraphImpl::Process(AudioMixer* aMixer) { |
577 |
bool doneAllProducing = false; |
578 |
const GraphTime oldProcessedTime = mProcessedTime; |
579 |
|
580 |
- // Process NativeInputTracks first since they are data source of other tracks |
581 |
- for (uint32_t i = 0; i < mTracks.Length(); ++i) { |
582 |
- NativeInputTrack* track = mTracks[i]->AsNativeInputTrack(); |
583 |
- if (track) { |
584 |
- track->ProcessInput(mProcessedTime, mStateComputedTime, |
585 |
- ProcessedMediaTrack::ALLOW_END); |
586 |
- } |
587 |
- } |
588 |
- |
589 |
// Figure out what each track wants to do |
590 |
for (uint32_t i = 0; i < mTracks.Length(); ++i) { |
591 |
MediaTrack* track = mTracks[i]; |
592 |
if (!doneAllProducing) { |
593 |
ProcessedMediaTrack* pt = track->AsProcessedTrack(); |
594 |
- // NativeInputTrack::ProcessInput are called above so we can skip them |
595 |
- bool isNativeInputTrack = track->AsNativeInputTrack(); |
596 |
- if (pt && !isNativeInputTrack) { |
597 |
+ if (pt) { |
598 |
AudioNodeTrack* n = track->AsAudioNodeTrack(); |
599 |
if (n) { |
600 |
#ifdef DEBUG |
601 |
diff --git dom/media/MediaTrackGraph.h dom/media/MediaTrackGraph.h |
602 |
index d85cb502b9..1c34060abb 100644 |
603 |
--- dom/media/MediaTrackGraph.h |
604 |
+++ dom/media/MediaTrackGraph.h |
605 |
@@ -97,7 +97,6 @@ class MediaTrack; |
606 |
class MediaTrackGraph; |
607 |
class MediaTrackGraphImpl; |
608 |
class MediaTrackListener; |
609 |
-class NativeInputTrack; |
610 |
class ProcessedMediaTrack; |
611 |
class SourceMediaTrack; |
612 |
|
613 |
@@ -107,14 +106,6 @@ class AudioDataListenerInterface { |
614 |
virtual ~AudioDataListenerInterface() = default; |
615 |
|
616 |
public: |
617 |
- // Information for the interleaved buffer coming from the audio callbacks |
618 |
- struct BufferInfo { |
619 |
- AudioDataValue* mBuffer = nullptr; |
620 |
- size_t mFrames = 0; |
621 |
- uint32_t mChannels = 0; |
622 |
- TrackRate mRate = 0; |
623 |
- }; |
624 |
- |
625 |
/* These are for cubeb audio input & output streams: */ |
626 |
/** |
627 |
* Output data to speakers, for use as the "far-end" data for echo |
628 |
@@ -122,7 +113,8 @@ class AudioDataListenerInterface { |
629 |
* chunks. |
630 |
*/ |
631 |
virtual void NotifyOutputData(MediaTrackGraphImpl* aGraph, |
632 |
- BufferInfo aInfo) = 0; |
633 |
+ AudioDataValue* aBuffer, size_t aFrames, |
634 |
+ TrackRate aRate, uint32_t aChannels) = 0; |
635 |
/** |
636 |
* An AudioCallbackDriver with an input stream signaling that it has stopped |
637 |
* for any reason and the AudioDataListener will not be notified of input data |
638 |
@@ -134,7 +126,8 @@ class AudioDataListenerInterface { |
639 |
* guaranteed to be in any particular size chunks. |
640 |
*/ |
641 |
virtual void NotifyInputData(MediaTrackGraphImpl* aGraph, |
642 |
- const BufferInfo aInfo, |
643 |
+ const AudioDataValue* aBuffer, size_t aFrames, |
644 |
+ TrackRate aRate, uint32_t aChannels, |
645 |
uint32_t aAlreadyBuffered) = 0; |
646 |
|
647 |
/** |
648 |
@@ -389,7 +382,6 @@ class MediaTrack : public mozilla::LinkedListElement<MediaTrack> { |
649 |
virtual ForwardedInputTrack* AsForwardedInputTrack() { return nullptr; } |
650 |
virtual CrossGraphTransmitter* AsCrossGraphTransmitter() { return nullptr; } |
651 |
virtual CrossGraphReceiver* AsCrossGraphReceiver() { return nullptr; } |
652 |
- virtual NativeInputTrack* AsNativeInputTrack() { return nullptr; } |
653 |
|
654 |
// These Impl methods perform the core functionality of the control methods |
655 |
// above, on the media graph thread. |
656 |
diff --git dom/media/MediaTrackGraphImpl.h dom/media/MediaTrackGraphImpl.h |
657 |
index 8baeb25a7d..d0c3ac8499 100644 |
658 |
--- dom/media/MediaTrackGraphImpl.h |
659 |
+++ dom/media/MediaTrackGraphImpl.h |
660 |
@@ -11,7 +11,6 @@ |
661 |
#include "AudioMixer.h" |
662 |
#include "GraphDriver.h" |
663 |
#include "mozilla/Atomics.h" |
664 |
-#include "mozilla/Maybe.h" |
665 |
#include "mozilla/Monitor.h" |
666 |
#include "mozilla/TimeStamp.h" |
667 |
#include "mozilla/UniquePtr.h" |
668 |
@@ -35,79 +34,6 @@ template <typename T> |
669 |
class LinkedList; |
670 |
class GraphRunner; |
671 |
|
672 |
-// MediaTrack subclass storing the raw audio data from microphone. |
673 |
-class NativeInputTrack : public ProcessedMediaTrack { |
674 |
- ~NativeInputTrack() = default; |
675 |
- explicit NativeInputTrack(TrackRate aSampleRate) |
676 |
- : ProcessedMediaTrack(aSampleRate, MediaSegment::AUDIO, |
677 |
- new AudioSegment()) {} |
678 |
- |
679 |
- public: |
680 |
- // Main Thread API |
681 |
- static NativeInputTrack* Create(MediaTrackGraphImpl* aGraph); |
682 |
- |
683 |
- size_t AddUser(); |
684 |
- size_t RemoveUser(); |
685 |
- |
686 |
- // Graph Thread APIs, for ProcessedMediaTrack |
687 |
- void DestroyImpl() override; |
688 |
- void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; |
689 |
- uint32_t NumberOfChannels() const override; |
690 |
- |
691 |
- // Graph thread APIs: Redirect calls from GraphDriver to mDataUsers |
692 |
- void NotifyOutputData(MediaTrackGraphImpl* aGraph, AudioDataValue* aBuffer, |
693 |
- size_t aFrames, TrackRate aRate, uint32_t aChannels); |
694 |
- void NotifyInputStopped(MediaTrackGraphImpl* aGraph); |
695 |
- void NotifyInputData(MediaTrackGraphImpl* aGraph, |
696 |
- const AudioDataValue* aBuffer, size_t aFrames, |
697 |
- TrackRate aRate, uint32_t aChannels, |
698 |
- uint32_t aAlreadyBuffered); |
699 |
- void DeviceChanged(MediaTrackGraphImpl* aGraph); |
700 |
- |
701 |
- // Other Graph Thread APIs |
702 |
- void InitDataHolderIfNeeded(); |
703 |
- |
704 |
- // Any thread |
705 |
- NativeInputTrack* AsNativeInputTrack() override { return this; } |
706 |
- |
707 |
- public: |
708 |
- // Only accessed on the graph thread. |
709 |
- nsTArray<RefPtr<AudioDataListener>> mDataUsers; |
710 |
- |
711 |
- private: |
712 |
- class AudioDataBuffers { |
713 |
- public: |
714 |
- AudioDataBuffers() = default; |
715 |
- void SetOutputData(AudioDataValue* aBuffer, size_t aFrames, |
716 |
- uint32_t aChannels, TrackRate aRate); |
717 |
- void SetInputData(AudioDataValue* aBuffer, size_t aFrames, |
718 |
- uint32_t aChannels, TrackRate aRate); |
719 |
- |
720 |
- enum Scope : unsigned char { |
721 |
- Input = 0x01, |
722 |
- Output = 0x02, |
723 |
- }; |
724 |
- void Clear(Scope aScope); |
725 |
- |
726 |
- typedef AudioDataListenerInterface::BufferInfo BufferInfo; |
727 |
- // Storing the audio output data coming from NotifyOutputData |
728 |
- Maybe<BufferInfo> mOutputData; |
729 |
- // Storing the audio input data coming from NotifyInputData |
730 |
- Maybe<BufferInfo> mInputData; |
731 |
- }; |
732 |
- |
733 |
- // Only accessed on the graph thread. |
734 |
- // Storing the audio data coming from GraphDriver directly. |
735 |
- Maybe<AudioDataBuffers> mDataHolder; |
736 |
- |
737 |
- // Only accessed on the graph thread. |
738 |
- uint32_t mInputChannels = 0; |
739 |
- |
740 |
- // Only accessed on the main thread. |
741 |
- // When this becomes zero, this NativeInputTrack is no longer needed. |
742 |
- int32_t mUserCount = 0; |
743 |
-}; |
744 |
- |
745 |
/** |
746 |
* A per-track update message passed from the media graph thread to the |
747 |
* main thread. |
748 |
@@ -462,28 +388,20 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
749 |
}; |
750 |
TrackTime PlayAudio(AudioMixer* aMixer, const TrackKeyAndVolume& aTkv, |
751 |
GraphTime aPlayedTime); |
752 |
- |
753 |
- /* Called on the main thread when AudioInputTrack requests audio data from an |
754 |
- * input device aID. */ |
755 |
- ProcessedMediaTrack* GetDeviceTrack(CubebUtils::AudioDeviceID aID); |
756 |
- |
757 |
/* Runs off a message on the graph thread when something requests audio from |
758 |
* an input audio device of ID aID, and delivers the input audio frames to |
759 |
* aListener. */ |
760 |
void OpenAudioInputImpl(CubebUtils::AudioDeviceID aID, |
761 |
- AudioDataListener* aListener, |
762 |
- NativeInputTrack* aInputTrack); |
763 |
+ AudioDataListener* aListener); |
764 |
/* Called on the main thread when something requests audio from an input |
765 |
* audio device aID. */ |
766 |
virtual nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID, |
767 |
AudioDataListener* aListener) override; |
768 |
- |
769 |
/* Runs off a message on the graph when input audio from aID is not needed |
770 |
* anymore, for a particular track. It can be that other tracks still need |
771 |
* audio from this audio input device. */ |
772 |
void CloseAudioInputImpl(CubebUtils::AudioDeviceID aID, |
773 |
- AudioDataListener* aListener, |
774 |
- NativeInputTrack* aInputTrack); |
775 |
+ AudioDataListener* aListener); |
776 |
/* Called on the main thread when input audio from aID is not needed |
777 |
* anymore, for a particular track. It can be that other tracks still need |
778 |
* audio from this audio input device. */ |
779 |
@@ -578,12 +496,12 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
780 |
MOZ_ASSERT(OnGraphThreadOrNotRunning()); |
781 |
|
782 |
#ifdef ANDROID |
783 |
- if (!mDeviceTrackMap.Contains(mInputDeviceID)) { |
784 |
+ if (!mInputDeviceUsers.Contains(mInputDeviceID)) { |
785 |
return 0; |
786 |
} |
787 |
#else |
788 |
if (!mInputDeviceID) { |
789 |
- MOZ_ASSERT(mDeviceTrackMap.Count() == 0, |
790 |
+ MOZ_ASSERT(mInputDeviceUsers.Count() == 0, |
791 |
"If running on a platform other than android," |
792 |
"an explicit device id should be present"); |
793 |
return 0; |
794 |
@@ -592,12 +510,7 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
795 |
uint32_t maxInputChannels = 0; |
796 |
// When/if we decide to support multiple input device per graph, this needs |
797 |
// loop over them. |
798 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
799 |
- MOZ_ASSERT(result); |
800 |
- if (!result) { |
801 |
- return maxInputChannels; |
802 |
- } |
803 |
- for (const auto& listener : result.Data()->mDataUsers) { |
804 |
+ for (const auto& listener : *mInputDeviceUsers.Lookup(mInputDeviceID)) { |
805 |
maxInputChannels = std::max(maxInputChannels, |
806 |
listener->RequestedInputChannelCount(this)); |
807 |
} |
808 |
@@ -607,8 +520,8 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
809 |
AudioInputType AudioInputDevicePreference() { |
810 |
MOZ_ASSERT(OnGraphThreadOrNotRunning()); |
811 |
|
812 |
- auto result = mDeviceTrackMap.Lookup(mInputDeviceID); |
813 |
- if (!result) { |
814 |
+ auto listeners = mInputDeviceUsers.Lookup(mInputDeviceID); |
815 |
+ if (!listeners) { |
816 |
return AudioInputType::Unknown; |
817 |
} |
818 |
bool voiceInput = false; |
819 |
@@ -617,7 +530,7 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
820 |
|
821 |
// If at least one track is considered to be voice, |
822 |
// XXX This could use short-circuit evaluation resp. std::any_of. |
823 |
- for (const auto& listener : result.Data()->mDataUsers) { |
824 |
+ for (const auto& listener : *listeners) { |
825 |
voiceInput |= listener->IsVoiceInput(this); |
826 |
} |
827 |
if (voiceInput) { |
828 |
@@ -842,7 +755,7 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
829 |
/** |
830 |
* Devices to use for cubeb input & output, or nullptr for default device. |
831 |
* A MediaTrackGraph always has an output (even if silent). |
832 |
- * If `mDeviceTrackMap.Count() != 0`, this MediaTrackGraph wants audio |
833 |
+ * If `mInputDeviceUsers.Count() != 0`, this MediaTrackGraph wants audio |
834 |
* input. |
835 |
* |
836 |
* All mInputDeviceID access is on the graph thread except for reads via |
837 |
@@ -853,13 +766,12 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
838 |
*/ |
839 |
std::atomic<CubebUtils::AudioDeviceID> mInputDeviceID; |
840 |
CubebUtils::AudioDeviceID mOutputDeviceID; |
841 |
- |
842 |
- // Maps AudioDeviceID to a device track that delivers audio input/output |
843 |
- // data and send device-changed signals to its listeners. This is only |
844 |
- // touched on the graph thread. The NativeInputTrack* here is used for |
845 |
- // for bookkeeping on the graph thread. The owner of the NativeInputTrack is |
846 |
- // mDeviceTracks, which is only touched by main thread. |
847 |
- nsTHashMap<CubebUtils::AudioDeviceID, NativeInputTrack*> mDeviceTrackMap; |
848 |
+ // Maps AudioDeviceID to an array of their users (that are listeners). This is |
849 |
+ // used to deliver audio input frames and to notify the listeners that the |
850 |
+ // audio device that delivers the audio frames has changed. |
851 |
+ // This is only touched on the graph thread. |
852 |
+ nsTHashMap<nsVoidPtrHashKey, nsTArray<RefPtr<AudioDataListener>>> |
853 |
+ mInputDeviceUsers; |
854 |
|
855 |
/** |
856 |
* List of resume operations waiting for a switch to an AudioCallbackDriver. |
857 |
@@ -1131,11 +1043,6 @@ class MediaTrackGraphImpl : public MediaTrackGraph, |
858 |
* ctor, and the read/write only on the graph thread. |
859 |
*/ |
860 |
uint32_t mMaxOutputChannelCount; |
861 |
- |
862 |
- /* |
863 |
- * Hold the NativeInputTrack for a certain device |
864 |
- */ |
865 |
- nsTHashMap<CubebUtils::AudioDeviceID, RefPtr<NativeInputTrack>> mDeviceTracks; |
866 |
}; |
867 |
|
868 |
} // namespace mozilla |
869 |
diff --git dom/media/gtest/TestAudioInputProcessing.cpp dom/media/gtest/TestAudioInputProcessing.cpp |
870 |
index ed5d14fcb8..c78a56080a 100644 |
871 |
--- dom/media/gtest/TestAudioInputProcessing.cpp |
872 |
+++ dom/media/gtest/TestAudioInputProcessing.cpp |
873 |
@@ -70,11 +70,8 @@ TEST(TestAudioInputProcessing, UnaccountedPacketizerBuffering) |
874 |
processedTime = 0; |
875 |
nextTime = MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(nrFrames); |
876 |
generator.GenerateInterleaved(buffer.Elements(), nrFrames); |
877 |
- aip->NotifyInputData(graph, |
878 |
- AudioInputProcessing::BufferInfo{ |
879 |
- buffer.Elements(), nrFrames, channels, rate}, |
880 |
+ aip->NotifyInputData(graph, buffer.Elements(), nrFrames, rate, channels, |
881 |
nextTime - nrFrames); |
882 |
- aip->ProcessInput(graph, nullptr); |
883 |
aip->Pull(graph, processedTime, nextTime, segment.GetDuration(), &segment, |
884 |
true, &ended); |
885 |
EXPECT_EQ(aip->NumBufferedFrames(graph), 24U); |
886 |
@@ -90,11 +87,8 @@ TEST(TestAudioInputProcessing, UnaccountedPacketizerBuffering) |
887 |
processedTime = nextTime; |
888 |
nextTime = MediaTrackGraphImpl::RoundUpToEndOfAudioBlock(2 * nrFrames); |
889 |
generator.GenerateInterleaved(buffer.Elements(), nrFrames); |
890 |
- aip->NotifyInputData(graph, |
891 |
- AudioInputProcessing::BufferInfo{ |
892 |
- buffer.Elements(), nrFrames, channels, rate}, |
893 |
+ aip->NotifyInputData(graph, buffer.Elements(), nrFrames, rate, channels, |
894 |
nextTime - (2 * nrFrames)); |
895 |
- aip->ProcessInput(graph, nullptr); |
896 |
aip->Pull(graph, processedTime, nextTime, segment.GetDuration(), &segment, |
897 |
true, &ended); |
898 |
EXPECT_EQ(aip->NumBufferedFrames(graph), 120U); |
899 |
diff --git dom/media/webrtc/MediaEngineWebRTCAudio.cpp dom/media/webrtc/MediaEngineWebRTCAudio.cpp |
900 |
index fb59fc195f..77da55c0a1 100644 |
901 |
--- dom/media/webrtc/MediaEngineWebRTCAudio.cpp |
902 |
+++ dom/media/webrtc/MediaEngineWebRTCAudio.cpp |
903 |
@@ -867,20 +867,21 @@ void AudioInputProcessing::Pull(MediaTrackGraphImpl* aGraph, GraphTime aFrom, |
904 |
} |
905 |
|
906 |
void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph, |
907 |
- BufferInfo aInfo) { |
908 |
+ AudioDataValue* aBuffer, |
909 |
+ size_t aFrames, TrackRate aRate, |
910 |
+ uint32_t aChannels) { |
911 |
MOZ_ASSERT(aGraph->OnGraphThread()); |
912 |
MOZ_ASSERT(mEnabled); |
913 |
|
914 |
- if (!mPacketizerOutput || |
915 |
- mPacketizerOutput->mPacketSize != aInfo.mRate / 100u || |
916 |
- mPacketizerOutput->mChannels != aInfo.mChannels) { |
917 |
+ if (!mPacketizerOutput || mPacketizerOutput->mPacketSize != aRate / 100u || |
918 |
+ mPacketizerOutput->mChannels != aChannels) { |
919 |
// It's ok to drop the audio still in the packetizer here: if this changes, |
920 |
// we changed devices or something. |
921 |
mPacketizerOutput = MakeUnique<AudioPacketizer<AudioDataValue, float>>( |
922 |
- aInfo.mRate / 100, aInfo.mChannels); |
923 |
+ aRate / 100, aChannels); |
924 |
} |
925 |
|
926 |
- mPacketizerOutput->Input(aInfo.mBuffer, aInfo.mFrames); |
927 |
+ mPacketizerOutput->Input(aBuffer, aFrames); |
928 |
|
929 |
while (mPacketizerOutput->PacketsAvailable()) { |
930 |
uint32_t samplesPerPacket = |
931 |
@@ -899,11 +900,11 @@ void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph, |
932 |
uint32_t channelCountFarend = 0; |
933 |
uint32_t framesPerPacketFarend = 0; |
934 |
|
935 |
- // Downmix from aInfo.mChannels to MAX_CHANNELS if needed. We always have |
936 |
- // floats here, the packetized performed the conversion. |
937 |
- if (aInfo.mChannels > MAX_CHANNELS) { |
938 |
+ // Downmix from aChannels to MAX_CHANNELS if needed. We always have floats |
939 |
+ // here, the packetized performed the conversion. |
940 |
+ if (aChannels > MAX_CHANNELS) { |
941 |
AudioConverter converter( |
942 |
- AudioConfig(aInfo.mChannels, 0, AudioConfig::FORMAT_FLT), |
943 |
+ AudioConfig(aChannels, 0, AudioConfig::FORMAT_FLT), |
944 |
AudioConfig(MAX_CHANNELS, 0, AudioConfig::FORMAT_FLT)); |
945 |
framesPerPacketFarend = mPacketizerOutput->mPacketSize; |
946 |
framesPerPacketFarend = |
947 |
@@ -913,9 +914,9 @@ void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph, |
948 |
deinterleavedPacketDataChannelPointers.SetLength(MAX_CHANNELS); |
949 |
} else { |
950 |
interleavedFarend = packet; |
951 |
- channelCountFarend = aInfo.mChannels; |
952 |
+ channelCountFarend = aChannels; |
953 |
framesPerPacketFarend = mPacketizerOutput->mPacketSize; |
954 |
- deinterleavedPacketDataChannelPointers.SetLength(aInfo.mChannels); |
955 |
+ deinterleavedPacketDataChannelPointers.SetLength(aChannels); |
956 |
} |
957 |
|
958 |
MOZ_ASSERT(interleavedFarend && |
959 |
@@ -941,7 +942,7 @@ void AudioInputProcessing::NotifyOutputData(MediaTrackGraphImpl* aGraph, |
960 |
|
961 |
// Having the same config for input and output means we potentially save |
962 |
// some CPU. |
963 |
- StreamConfig inputConfig(aInfo.mRate, channelCountFarend, false); |
964 |
+ StreamConfig inputConfig(aRate, channelCountFarend, false); |
965 |
StreamConfig outputConfig = inputConfig; |
966 |
|
967 |
// Passing the same pointers here saves a copy inside this function. |
968 |
@@ -1078,34 +1079,45 @@ void AudioInputProcessing::PacketizeAndProcess(MediaTrackGraphImpl* aGraph, |
969 |
} |
970 |
} |
971 |
|
972 |
-void AudioInputProcessing::ProcessInput(MediaTrackGraphImpl* aGraph, |
973 |
- const AudioSegment* aSegment) { |
974 |
- MOZ_ASSERT(aGraph); |
975 |
- MOZ_ASSERT(aGraph->OnGraphThread()); |
976 |
- |
977 |
- if (mEnded || !mEnabled || !mLiveFramesAppended || !mInputData) { |
978 |
+template <typename T> |
979 |
+void AudioInputProcessing::InsertInGraph(MediaTrackGraphImpl* aGraph, |
980 |
+ const T* aBuffer, size_t aFrames, |
981 |
+ uint32_t aChannels) { |
982 |
+ if (mEnded) { |
983 |
return; |
984 |
} |
985 |
|
986 |
- // One NotifyInputData might have multiple following ProcessInput calls, but |
987 |
- // we only process one input per NotifyInputData call. |
988 |
- BufferInfo inputInfo = mInputData.extract(); |
989 |
+ MOZ_ASSERT(aChannels >= 1 && aChannels <= 8, "Support up to 8 channels"); |
990 |
|
991 |
- // If some processing is necessary, packetize and insert in the WebRTC.org |
992 |
- // code. Otherwise, directly insert the mic data in the MTG, bypassing all |
993 |
- // processing. |
994 |
- if (PassThrough(aGraph)) { |
995 |
- if (aSegment) { |
996 |
- mSegment.AppendSegment(aSegment, mPrincipal); |
997 |
- } else { |
998 |
- mSegment.AppendFromInterleavedBuffer(inputInfo.mBuffer, inputInfo.mFrames, |
999 |
- inputInfo.mChannels, mPrincipal); |
1000 |
- } |
1001 |
+ CheckedInt<size_t> bufferSize(sizeof(T)); |
1002 |
+ bufferSize *= aFrames; |
1003 |
+ bufferSize *= aChannels; |
1004 |
+ RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize); |
1005 |
+ AutoTArray<const T*, 8> channels; |
1006 |
+ if (aChannels == 1) { |
1007 |
+ PodCopy(static_cast<T*>(buffer->Data()), aBuffer, aFrames); |
1008 |
+ channels.AppendElement(static_cast<T*>(buffer->Data())); |
1009 |
} else { |
1010 |
- MOZ_ASSERT(aGraph->GraphRate() == inputInfo.mRate); |
1011 |
- PacketizeAndProcess(aGraph, inputInfo.mBuffer, inputInfo.mFrames, |
1012 |
- inputInfo.mRate, inputInfo.mChannels); |
1013 |
+ channels.SetLength(aChannels); |
1014 |
+ AutoTArray<T*, 8> write_channels; |
1015 |
+ write_channels.SetLength(aChannels); |
1016 |
+ T* samples = static_cast<T*>(buffer->Data()); |
1017 |
+ |
1018 |
+ size_t offset = 0; |
1019 |
+ for (uint32_t i = 0; i < aChannels; ++i) { |
1020 |
+ channels[i] = write_channels[i] = samples + offset; |
1021 |
+ offset += aFrames; |
1022 |
+ } |
1023 |
+ |
1024 |
+ DeinterleaveAndConvertBuffer(aBuffer, aFrames, aChannels, |
1025 |
+ write_channels.Elements()); |
1026 |
} |
1027 |
+ |
1028 |
+ LOG_FRAME("AudioInputProcessing %p Appending %zu frames of raw audio", this, |
1029 |
+ aFrames); |
1030 |
+ |
1031 |
+ MOZ_ASSERT(aChannels == channels.Length()); |
1032 |
+ mSegment.AppendFrames(buffer.forget(), channels, aFrames, mPrincipal); |
1033 |
} |
1034 |
|
1035 |
void AudioInputProcessing::NotifyInputStopped(MediaTrackGraphImpl* aGraph) { |
1036 |
@@ -1119,13 +1131,14 @@ void AudioInputProcessing::NotifyInputStopped(MediaTrackGraphImpl* aGraph) { |
1037 |
if (mPacketizerInput) { |
1038 |
mPacketizerInput->Clear(); |
1039 |
} |
1040 |
- mInputData.take(); |
1041 |
} |
1042 |
|
1043 |
// Called back on GraphDriver thread! |
1044 |
// Note this can be called back after ::Stop() |
1045 |
void AudioInputProcessing::NotifyInputData(MediaTrackGraphImpl* aGraph, |
1046 |
- const BufferInfo aInfo, |
1047 |
+ const AudioDataValue* aBuffer, |
1048 |
+ size_t aFrames, TrackRate aRate, |
1049 |
+ uint32_t aChannels, |
1050 |
uint32_t aAlreadyBuffered) { |
1051 |
MOZ_ASSERT(aGraph->OnGraphThread()); |
1052 |
TRACE(); |
1053 |
@@ -1139,7 +1152,14 @@ void AudioInputProcessing::NotifyInputData(MediaTrackGraphImpl* aGraph, |
1054 |
mLiveBufferingAppended = aAlreadyBuffered; |
1055 |
} |
1056 |
|
1057 |
- mInputData = Some(aInfo); |
1058 |
+ // If some processing is necessary, packetize and insert in the WebRTC.org |
1059 |
+ // code. Otherwise, directly insert the mic data in the MTG, bypassing all |
1060 |
+ // processing. |
1061 |
+ if (PassThrough(aGraph)) { |
1062 |
+ InsertInGraph<AudioDataValue>(aGraph, aBuffer, aFrames, aChannels); |
1063 |
+ } else { |
1064 |
+ PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels); |
1065 |
+ } |
1066 |
} |
1067 |
|
1068 |
#define ResetProcessingIfNeeded(_processing) \ |
1069 |
@@ -1173,7 +1193,6 @@ void AudioInputProcessing::DeviceChanged(MediaTrackGraphImpl* aGraph) { |
1070 |
void AudioInputProcessing::End() { |
1071 |
mEnded = true; |
1072 |
mSegment.Clear(); |
1073 |
- mInputData.take(); |
1074 |
} |
1075 |
|
1076 |
TrackTime AudioInputProcessing::NumBufferedFrames( |
1077 |
@@ -1230,28 +1249,6 @@ void AudioInputTrack::DestroyImpl() { |
1078 |
void AudioInputTrack::ProcessInput(GraphTime aFrom, GraphTime aTo, |
1079 |
uint32_t aFlags) { |
1080 |
TRACE_COMMENT("AudioInputTrack %p", this); |
1081 |
- |
1082 |
- // Check if there is a connected NativeInputTrack |
1083 |
- NativeInputTrack* source = nullptr; |
1084 |
- if (!mInputs.IsEmpty()) { |
1085 |
- for (const MediaInputPort* input : mInputs) { |
1086 |
- MOZ_ASSERT(input->GetSource()); |
1087 |
- if (input->GetSource()->AsNativeInputTrack()) { |
1088 |
- source = input->GetSource()->AsNativeInputTrack(); |
1089 |
- break; |
1090 |
- } |
1091 |
- } |
1092 |
- } |
1093 |
- |
1094 |
- // Push the input data from the connected NativeInputTrack to mInputProcessing |
1095 |
- if (source) { |
1096 |
- MOZ_ASSERT(source->GraphImpl() == GraphImpl()); |
1097 |
- MOZ_ASSERT(source->mSampleRate == mSampleRate); |
1098 |
- MOZ_ASSERT(GraphImpl()->GraphRate() == mSampleRate); |
1099 |
- mInputProcessing->ProcessInput(GraphImpl(), |
1100 |
- source->GetData<AudioSegment>()); |
1101 |
- } |
1102 |
- |
1103 |
bool ended = false; |
1104 |
mInputProcessing->Pull( |
1105 |
GraphImpl(), aFrom, aTo, TrackTimeToGraphTime(GetEnd()), |
1106 |
@@ -1275,10 +1272,6 @@ nsresult AudioInputTrack::OpenAudioInput(CubebUtils::AudioDeviceID aId, |
1107 |
MOZ_ASSERT(!mInputListener); |
1108 |
MOZ_ASSERT(mDeviceId.isNothing()); |
1109 |
mInputListener = aListener; |
1110 |
- ProcessedMediaTrack* input = GraphImpl()->GetDeviceTrack(aId); |
1111 |
- MOZ_ASSERT(input); |
1112 |
- LOG("Open device %p (InputTrack=%p) for Mic source %p", aId, input, this); |
1113 |
- mPort = AllocateInputPort(input); |
1114 |
mDeviceId.emplace(aId); |
1115 |
return GraphImpl()->OpenAudioInput(aId, aListener); |
1116 |
} |
1117 |
@@ -1289,11 +1282,7 @@ void AudioInputTrack::CloseAudioInput() { |
1118 |
if (!mInputListener) { |
1119 |
return; |
1120 |
} |
1121 |
- MOZ_ASSERT(mPort); |
1122 |
MOZ_ASSERT(mDeviceId.isSome()); |
1123 |
- LOG("Close device %p (InputTrack=%p) for Mic source %p ", mDeviceId.value(), |
1124 |
- mPort->GetSource(), this); |
1125 |
- mPort->Destroy(); |
1126 |
GraphImpl()->CloseAudioInput(mDeviceId.extract(), mInputListener); |
1127 |
mInputListener = nullptr; |
1128 |
} |
1129 |
diff --git dom/media/webrtc/MediaEngineWebRTCAudio.h dom/media/webrtc/MediaEngineWebRTCAudio.h |
1130 |
index 46a66d9a7a..9065b28f7a 100644 |
1131 |
--- dom/media/webrtc/MediaEngineWebRTCAudio.h |
1132 |
+++ dom/media/webrtc/MediaEngineWebRTCAudio.h |
1133 |
@@ -141,9 +141,13 @@ class AudioInputProcessing : public AudioDataListener { |
1134 |
GraphTime aTrackEnd, AudioSegment* aSegment, |
1135 |
bool aLastPullThisIteration, bool* aEnded); |
1136 |
|
1137 |
- void NotifyOutputData(MediaTrackGraphImpl* aGraph, BufferInfo aInfo) override; |
1138 |
+ void NotifyOutputData(MediaTrackGraphImpl* aGraph, AudioDataValue* aBuffer, |
1139 |
+ size_t aFrames, TrackRate aRate, |
1140 |
+ uint32_t aChannels) override; |
1141 |
void NotifyInputStopped(MediaTrackGraphImpl* aGraph) override; |
1142 |
- void NotifyInputData(MediaTrackGraphImpl* aGraph, const BufferInfo aInfo, |
1143 |
+ void NotifyInputData(MediaTrackGraphImpl* aGraph, |
1144 |
+ const AudioDataValue* aBuffer, size_t aFrames, |
1145 |
+ TrackRate aRate, uint32_t aChannels, |
1146 |
uint32_t aAlreadyBuffered) override; |
1147 |
bool IsVoiceInput(MediaTrackGraphImpl* aGraph) const override { |
1148 |
// If we're passing data directly without AEC or any other process, this |
1149 |
@@ -163,8 +167,9 @@ class AudioInputProcessing : public AudioDataListener { |
1150 |
|
1151 |
void Disconnect(MediaTrackGraphImpl* aGraph) override; |
1152 |
|
1153 |
- // aSegment stores the unprocessed non-interleaved audio input data from mic |
1154 |
- void ProcessInput(MediaTrackGraphImpl* aGraph, const AudioSegment* aSegment); |
1155 |
+ template <typename T> |
1156 |
+ void InsertInGraph(MediaTrackGraphImpl* aGraph, const T* aBuffer, |
1157 |
+ size_t aFrames, uint32_t aChannels); |
1158 |
|
1159 |
void PacketizeAndProcess(MediaTrackGraphImpl* aGraph, |
1160 |
const AudioDataValue* aBuffer, size_t aFrames, |
1161 |
@@ -242,8 +247,6 @@ class AudioInputProcessing : public AudioDataListener { |
1162 |
bool mEnabled; |
1163 |
// Whether or not we've ended and removed the AudioInputTrack. |
1164 |
bool mEnded; |
1165 |
- // Store the unprocessed interleaved audio input data |
1166 |
- Maybe<BufferInfo> mInputData; |
1167 |
}; |
1168 |
|
1169 |
// MediaTrack subclass tailored for MediaEngineWebRTCMicrophoneSource. |
1170 |
@@ -251,10 +254,6 @@ class AudioInputTrack : public ProcessedMediaTrack { |
1171 |
// Only accessed on the graph thread. |
1172 |
RefPtr<AudioInputProcessing> mInputProcessing; |
1173 |
|
1174 |
- // Only accessed on the main thread. Link to the track producing raw audio |
1175 |
- // input data. Graph thread should use mInputs to get the source |
1176 |
- RefPtr<MediaInputPort> mPort; |
1177 |
- |
1178 |
// Only accessed on the main thread. Used for bookkeeping on main thread, such |
1179 |
// that CloseAudioInput can be idempotent. |
1180 |
// XXX Should really be a CubebUtils::AudioDeviceID, but they aren't |
1181 |
-- |
1182 |
2.32.0 |
1183 |
|