Implement a ChannelMixingAudioProcessor.

PiperOrigin-RevId: 506886903

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAudioEndToEndTest.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.net.Uri;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.MediaItem;
+import com.google.android.exoplayer2.audio.AudioProcessor;
+import com.google.android.exoplayer2.audio.ToInt16PcmAudioProcessor;
+import com.google.common.collect.ImmutableList;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * End-to-end instrumentation test for {@link Transformer} audio test cases that cannot be tested
+ * using robolectric.
+ */
+@RunWith(AndroidJUnit4.class)
+public class TransformerAudioEndToEndTest {
+  private final Context context = ApplicationProvider.getApplicationContext();
+
+  @Test
+  public void mixMonoToStereo_outputsStereo() throws Exception {
+    String testId = "mixMonoToStereo_outputsStereo";
+
+    Effects effects =
+        createForAudioProcessors(
+            new ChannelMixingAudioProcessor(
+                ChannelMixingMatrix.create(
+                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
+            .setRemoveVideo(true)
+            .setEffects(effects)
+            .build();
+
+    TransformationTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.transformationResult.channelCount).isEqualTo(2);
+  }
+
+  @Test
+  public void mixingChannels_outputsFloatPcm() throws Exception {
+    final String testId = "mixingChannels_outputsFloatPcm";
+
+    Effects effects =
+        createForAudioProcessors(
+            new ChannelMixingAudioProcessor(
+                ChannelMixingMatrix.create(
+                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
+            .setRemoveVideo(true)
+            .setEffects(effects)
+            .build();
+
+    TransformationTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_FLOAT);
+  }
+
+  @Test
+  public void mixChannelsThenToInt16Pcm_outputsInt16Pcm() throws Exception {
+    final String testId = "mixChannelsThenToInt16Pcm_outputsInt16Pcm";
+
+    Effects effects =
+        createForAudioProcessors(
+            new ChannelMixingAudioProcessor(
+                ChannelMixingMatrix.create(
+                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)),
+            new ToInt16PcmAudioProcessor());
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
+            .setRemoveVideo(true)
+            .setEffects(effects)
+            .build();
+
+    TransformationTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.transformationResult.pcmEncoding).isEqualTo(C.ENCODING_PCM_16BIT);
+  }
+
+  private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) {
+    return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of());
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessor.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
+
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.audio.AudioProcessor;
+import com.google.android.exoplayer2.audio.BaseAudioProcessor;
+import java.nio.ByteBuffer;
+
+/**
+ * An {@link AudioProcessor} that handles mixing and scaling audio channels.
+ *
+ * <p>The following encodings are supported as input:
+ *
+ * <ul>
+ *   <li>{@link C#ENCODING_PCM_16BIT}
+ *   <li>{@link C#ENCODING_PCM_FLOAT}
+ * </ul>
+ *
+ * The output is {@link C#ENCODING_PCM_FLOAT}.
+ */
+/* package */ final class ChannelMixingAudioProcessor extends BaseAudioProcessor {
+
+  @Nullable private ChannelMixingMatrix pendingMatrix;
+  @Nullable private ChannelMixingMatrix matrix;
+  @Nullable private AudioMixingAlgorithm pendingAlgorithm;
+  @Nullable private AudioMixingAlgorithm algorithm;
+
+  public ChannelMixingAudioProcessor(ChannelMixingMatrix matrix) {
+    pendingMatrix = matrix;
+  }
+
+  public void setMatrix(ChannelMixingMatrix matrix) {
+    pendingMatrix = matrix;
+  }
+
+  @Override
+  protected AudioFormat onConfigure(AudioFormat inputAudioFormat)
+      throws UnhandledAudioFormatException {
+    checkStateNotNull(pendingMatrix);
+    // TODO(b/252538025): Allow for a mapping of input channel count -> matrix to be passed in.
+    if (inputAudioFormat.channelCount != pendingMatrix.getInputChannelCount()) {
+      throw new UnhandledAudioFormatException(
+          "Channel count must match mixing matrix", inputAudioFormat);
+    }
+
+    if (pendingMatrix.isIdentity()) {
+      return AudioFormat.NOT_SET;
+    }
+
+    // TODO(b/264926272): Allow config of output PCM config when other AudioMixingAlgorithms exist.
+    AudioFormat pendingOutputAudioFormat =
+        new AudioFormat(
+            inputAudioFormat.sampleRate,
+            pendingMatrix.getOutputChannelCount(),
+            C.ENCODING_PCM_FLOAT);
+
+    pendingAlgorithm = AudioMixingAlgorithm.create(pendingOutputAudioFormat);
+    if (!pendingAlgorithm.supportsSourceAudioFormat(inputAudioFormat)) {
+      throw new UnhandledAudioFormatException(inputAudioFormat);
+    }
+
+    return pendingOutputAudioFormat;
+  }
+
+  @Override
+  protected void onFlush() {
+    algorithm = pendingAlgorithm;
+    matrix = pendingMatrix;
+  }
+
+  @Override
+  protected void onReset() {
+    pendingAlgorithm = null;
+    algorithm = null;
+    pendingMatrix = null;
+    matrix = null;
+  }
+
+  @Override
+  public void queueInput(ByteBuffer inputBuffer) {
+    int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame;
+    ByteBuffer outputBuffer =
+        replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame);
+    checkNotNull(algorithm)
+        .mix(inputBuffer, inputAudioFormat, checkNotNull(matrix), inputFramesToMix, outputBuffer);
+    outputBuffer.flip();
+  }
+}

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessorTest.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.transformer;
+
+import static com.google.android.exoplayer2.testutil.TestUtil.createByteBuffer;
+import static com.google.android.exoplayer2.testutil.TestUtil.createFloatArray;
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
+import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link ChannelMixingAudioProcessor}. */
+@RunWith(AndroidJUnit4.class)
+public class ChannelMixingAudioProcessorTest {
+
+  private static final AudioFormat AUDIO_FORMAT_48KHZ_STEREO_16BIT =
+      new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 2, C.ENCODING_PCM_16BIT);
+
+  @Test
+  public void configure_outputAudioFormat_matchesChannelCountOfMatrix() throws Exception {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(
+            ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1));
+
+    AudioFormat outputAudioFormat = audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
+    assertThat(outputAudioFormat.channelCount).isEqualTo(1);
+  }
+
+  @Test
+  public void configure_invalidInputAudioChannelCount_throws() {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(
+            ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2));
+
+    assertThrows(
+        UnhandledAudioFormatException.class,
+        () -> audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT));
+  }
+
+  @Test
+  public void reconfigure_withDifferentMatrix_outputsCorrectChannelCount() throws Exception {
+    ChannelMixingMatrix stereoTo1 =
+        ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1);
+    ChannelMixingMatrix stereoTo6 =
+        new ChannelMixingMatrix(
+            /* inputChannelCount= */ 2,
+            /* outputChannelCount= */ 6,
+            new float[] {
+              /* L channel factors */ 0.5f, 0.1f, 0.1f, 0.1f, 0.1f, 0.1f,
+              /* R channel factors */ 0.1f, 0.5f, 0.1f, 0.1f, 0.1f, 0.1f
+            });
+
+    ChannelMixingAudioProcessor channelMixingAudioProcessor =
+        new ChannelMixingAudioProcessor(stereoTo1);
+    AudioFormat outputAudioFormat =
+        channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
+    assertThat(outputAudioFormat.channelCount).isEqualTo(1);
+    channelMixingAudioProcessor.flush();
+
+    channelMixingAudioProcessor.setMatrix(stereoTo6);
+    outputAudioFormat = channelMixingAudioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
+    assertThat(outputAudioFormat.channelCount).isEqualTo(6);
+  }
+
+  @Test
+  public void isActive_afterConfigureWithCustomMixingMatrix_returnsTrue() throws Exception {
+    float[] coefficients =
+        new float[] {
+          /* L channel factors */ 0.5f, 0.5f, 0.0f,
+          /* R channel factors */ 0.0f, 0.5f, 0.5f
+        };
+
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(
+            new ChannelMixingMatrix(
+                /* inputChannelCount= */ 3, /* outputChannelCount= */ 2, coefficients));
+
+    AudioFormat outputAudioFormat =
+        audioProcessor.configure(
+            new AudioFormat(/* sampleRate= */ 48000, /* channelCount= */ 3, C.ENCODING_PCM_16BIT));
+
+    assertThat(audioProcessor.isActive()).isTrue();
+    assertThat(outputAudioFormat.channelCount).isEqualTo(2);
+  }
+
+  @Test
+  public void isActive_afterConfigureWithIdentityMatrix_returnsFalse() throws Exception {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(
+            ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 2));
+
+    audioProcessor.configure(AUDIO_FORMAT_48KHZ_STEREO_16BIT);
+    assertThat(audioProcessor.isActive()).isFalse();
+  }
+
+  @Test
+  public void numberOfFramesOutput_matchesNumberOfFramesInput() throws Exception {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(
+            ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1));
+
+    AudioFormat inputAudioFormat =
+        new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 2, C.ENCODING_PCM_16BIT);
+    AudioFormat outputAudioFormat = audioProcessor.configure(inputAudioFormat);
+    audioProcessor.flush();
+    audioProcessor.queueInput(
+        ByteBuffer.allocateDirect(inputAudioFormat.sampleRate * inputAudioFormat.bytesPerFrame)
+            .order(ByteOrder.nativeOrder()));
+
+    assertThat(audioProcessor.getOutput().remaining() / outputAudioFormat.bytesPerFrame)
+        .isEqualTo(44100);
+  }
+
+  @Test
+  public void output_stereoToMono_asExpected() throws Exception {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 1));
+
+    AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT);
+    audioProcessor.configure(inputAudioFormat);
+    audioProcessor.flush();
+
+    audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}));
+
+    assertThat(createFloatArray(audioProcessor.getOutput()))
+        .usingTolerance(1.0e-5)
+        .containsExactly(new float[] {0.15f, 0.35f, 0.55f})
+        .inOrder();
+  }
+
+  @Test
+  public void output_scaled_asExpected() throws Exception {
+    ChannelMixingAudioProcessor audioProcessor =
+        new ChannelMixingAudioProcessor(ChannelMixingMatrix.create(2, 2).scaleBy(0.5f));
+
+    AudioFormat inputAudioFormat = new AudioFormat(44100, 2, C.ENCODING_PCM_FLOAT);
+    audioProcessor.configure(inputAudioFormat);
+    audioProcessor.flush();
+
+    audioProcessor.queueInput(createByteBuffer(new float[] {0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f}));
+
+    assertThat(createFloatArray(audioProcessor.getOutput()))
+        .usingTolerance(1.0e-5)
+        .containsExactly(new float[] {0.05f, 0.1f, 0.15f, 0.2f, 0.25f, 0.3f})
+        .inOrder();
+  }
+}