Channel mix to 16-bit int not float

Previously `ChannelMixingAudioProcessor` output float because it was
implemented using the audio mixer's float mixing support.

Move the implementation over to just using the `ChannelMixingMatrix` and make
it publicly visible in the common module so it can be used by apps for both
playback and export.

Also resolve a TODO that no longer had a bug attached by implementing support
for putting multiple mixing matrices to handle different input audio channel
counts, and fix some nits in the test code.

Tested via unit tests and manually configuring a `ChannelMixingAudioProcessor`
in the transformer demo app and playing an audio stream that identifies
channels, and verifying that they are remapped as expected.

PiperOrigin-RevId: 523653901

library/common/src/main/java/com/google/android/exoplayer2/audio/ChannelMixingAudioProcessor.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.audio;
+
+import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
+
+import android.util.SparseArray;
+import androidx.annotation.Nullable;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.util.Util;
+import java.nio.ByteBuffer;
+
+/**
+ * An {@link AudioProcessor} that handles mixing and scaling audio channels. Call {@link
+ * #putChannelMixingMatrix(ChannelMixingMatrix)} specifying mixing matrices to apply for each
+ * possible input channel count before using the audio processor. Input and output are 16-bit PCM.
+ */
+public final class ChannelMixingAudioProcessor extends BaseAudioProcessor {
+
+  private final SparseArray<ChannelMixingMatrix> matrixByInputChannelCount;
+
+  /** Creates a new audio processor for mixing and scaling audio channels. */
+  public ChannelMixingAudioProcessor() {
+    matrixByInputChannelCount = new SparseArray<>();
+  }
+
+  /**
+   * Stores a channel mixing matrix for processing audio with a given {@link
+   * ChannelMixingMatrix#getInputChannelCount() channel count}. Overwrites any previously stored
+   * matrix for the same input channel count.
+   */
+  public void putChannelMixingMatrix(ChannelMixingMatrix matrix) {
+    int inputChannelCount = matrix.getInputChannelCount();
+    matrixByInputChannelCount.put(inputChannelCount, matrix);
+  }
+
+  @Override
+  protected AudioFormat onConfigure(AudioFormat inputAudioFormat)
+      throws UnhandledAudioFormatException {
+    if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
+      throw new UnhandledAudioFormatException(inputAudioFormat);
+    }
+    @Nullable
+    ChannelMixingMatrix channelMixingMatrix =
+        matrixByInputChannelCount.get(inputAudioFormat.channelCount);
+    if (channelMixingMatrix == null) {
+      throw new UnhandledAudioFormatException(
+          "No mixing matrix for input channel count", inputAudioFormat);
+    }
+    if (channelMixingMatrix.isIdentity()) {
+      return AudioFormat.NOT_SET;
+    }
+    return new AudioFormat(
+        inputAudioFormat.sampleRate,
+        channelMixingMatrix.getOutputChannelCount(),
+        C.ENCODING_PCM_16BIT);
+  }
+
+  @Override
+  public void queueInput(ByteBuffer inputBuffer) {
+    ChannelMixingMatrix channelMixingMatrix =
+        checkStateNotNull(matrixByInputChannelCount.get(inputAudioFormat.channelCount));
+
+    int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame;
+    ByteBuffer outputBuffer =
+        replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame);
+    int inputChannelCount = channelMixingMatrix.getInputChannelCount();
+    int outputChannelCount = channelMixingMatrix.getOutputChannelCount();
+    float[] outputFrame = new float[outputChannelCount];
+    while (inputBuffer.hasRemaining()) {
+      for (int inputChannelIndex = 0; inputChannelIndex < inputChannelCount; inputChannelIndex++) {
+        short inputValue = inputBuffer.getShort();
+        for (int outputChannelIndex = 0;
+            outputChannelIndex < outputChannelCount;
+            outputChannelIndex++) {
+          outputFrame[outputChannelIndex] +=
+              channelMixingMatrix.getMixingCoefficient(inputChannelIndex, outputChannelIndex)
+                  * inputValue;
+        }
+      }
+      for (int outputChannelIndex = 0;
+          outputChannelIndex < outputChannelCount;
+          outputChannelIndex++) {
+        short shortValue =
+            (short)
+                Util.constrainValue(
+                    outputFrame[outputChannelIndex], Short.MIN_VALUE, Short.MAX_VALUE);
+        outputBuffer.put((byte) (shortValue & 0xFF));
+        outputBuffer.put((byte) ((shortValue >> 8) & 0xFF));
+        outputFrame[outputChannelIndex] = 0;
+      }
+    }
+    outputBuffer.flip();
+  }
+}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingMatrix.java → library/common/src/main/java/com/google/android/exoplayer2/audio/ChannelMixingMatrix.java

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * See the License for the specific language governing permissions and
  * limitations under the License.
  * limitations under the License.
  */
  */
-package com.google.android.exoplayer2.transformer;
+package com.google.android.exoplayer2.audio;
 
 
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 import static com.google.android.exoplayer2.util.Assertions.checkArgument;
 
 
@@ -39,7 +39,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument;
  *          0 0.7]</pre>
  *          0 0.7]</pre>
  * </ul>
  * </ul>
  */
  */
-/* package */ final class ChannelMixingMatrix {
+public final class ChannelMixingMatrix {
   private final int inputChannelCount;
   private final int inputChannelCount;
   private final int outputChannelCount;
   private final int outputChannelCount;
   private final float[] coefficients;
   private final float[] coefficients;

library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAudioEndToEndTest.java

@@ -26,8 +26,9 @@ import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.MediaItem;
 import com.google.android.exoplayer2.MediaItem;
 import com.google.android.exoplayer2.audio.AudioProcessor;
 import com.google.android.exoplayer2.audio.AudioProcessor;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
+import com.google.android.exoplayer2.audio.ChannelMixingAudioProcessor;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import com.google.android.exoplayer2.audio.TeeAudioProcessor;
 import com.google.android.exoplayer2.audio.TeeAudioProcessor;
-import com.google.android.exoplayer2.audio.ToInt16PcmAudioProcessor;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
@@ -71,11 +72,10 @@ public class TransformerAudioEndToEndTest {
   public void mixMonoToStereo_outputsStereo() throws Exception {
   public void mixMonoToStereo_outputsStereo() throws Exception {
     String testId = "mixMonoToStereo_outputsStereo";
     String testId = "mixMonoToStereo_outputsStereo";
 
 
-    Effects effects =
-        createForAudioProcessors(
-            new ChannelMixingAudioProcessor(
-                ChannelMixingMatrix.create(
-                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)));
+    ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor();
+    channelMixingAudioProcessor.putChannelMixingMatrix(
+        ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2));
+    Effects effects = createForAudioProcessors(channelMixingAudioProcessor);
     EditedMediaItem editedMediaItem =
     EditedMediaItem editedMediaItem =
         new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
         new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
             .setRemoveVideo(true)
             .setRemoveVideo(true)
@@ -90,60 +90,6 @@ public class TransformerAudioEndToEndTest {
     assertThat(result.exportResult.channelCount).isEqualTo(2);
     assertThat(result.exportResult.channelCount).isEqualTo(2);
   }
   }
 
 
-  @Test
-  public void channelMixing_outputsFloatPcm() throws Exception {
-    final String testId = "channelMixing_outputsFloatPcm";
-    FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink();
-
-    Effects effects =
-        createForAudioProcessors(
-            new ChannelMixingAudioProcessor(
-                ChannelMixingMatrix.create(
-                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)),
-            new TeeAudioProcessor(audioFormatTracker));
-    EditedMediaItem editedMediaItem =
-        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
-            .setRemoveVideo(true)
-            .setEffects(effects)
-            .build();
-
-    new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
-        .build()
-        .run(testId, editedMediaItem);
-
-    ImmutableList<AudioFormat> audioFormats = audioFormatTracker.getFlushedAudioFormats().asList();
-    assertThat(audioFormats).hasSize(1);
-    assertThat(audioFormats.get(0).encoding).isEqualTo(C.ENCODING_PCM_FLOAT);
-  }
-
-  @Test
-  public void channelMixingThenToInt16Pcm_outputsInt16Pcm() throws Exception {
-    final String testId = "channelMixingThenToInt16Pcm_outputsInt16Pcm";
-
-    FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink();
-
-    Effects effects =
-        createForAudioProcessors(
-            new ChannelMixingAudioProcessor(
-                ChannelMixingMatrix.create(
-                    /* inputChannelCount= */ 1, /* outputChannelCount= */ 2)),
-            new ToInt16PcmAudioProcessor(),
-            new TeeAudioProcessor(audioFormatTracker));
-    EditedMediaItem editedMediaItem =
-        new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))
-            .setRemoveVideo(true)
-            .setEffects(effects)
-            .build();
-
-    new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
-        .build()
-        .run(testId, editedMediaItem);
-
-    ImmutableList<AudioFormat> audioFormats = audioFormatTracker.getFlushedAudioFormats().asList();
-    assertThat(audioFormats).hasSize(1);
-    assertThat(audioFormats.get(0).encoding).isEqualTo(C.ENCODING_PCM_16BIT);
-  }
-
   private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) {
   private static Effects createForAudioProcessors(AudioProcessor... audioProcessors) {
     return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of());
     return new Effects(ImmutableList.copyOf(audioProcessors), ImmutableList.of());
   }
   }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioMixerImpl.java

@@ -26,6 +26,7 @@ import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
 import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.util.Util;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.ByteOrder;

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioMixingAlgorithm.java

@@ -19,6 +19,7 @@ import android.annotation.SuppressLint;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
 import com.google.android.exoplayer2.audio.AudioProcessor.UnhandledAudioFormatException;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ChannelMixingAudioProcessor.java

-/*
- * Copyright 2023 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.google.android.exoplayer2.transformer;
-
-import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
-import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
-
-import androidx.annotation.Nullable;
-import com.google.android.exoplayer2.C;
-import com.google.android.exoplayer2.audio.AudioProcessor;
-import com.google.android.exoplayer2.audio.BaseAudioProcessor;
-import java.nio.ByteBuffer;
-
-/**
- * An {@link AudioProcessor} that handles mixing and scaling audio channels.
- *
- * <p>The following encodings are supported as input:
- *
- * <ul>
- *   <li>{@link C#ENCODING_PCM_16BIT}
- *   <li>{@link C#ENCODING_PCM_FLOAT}
- * </ul>
- *
- * The output is {@link C#ENCODING_PCM_FLOAT}.
- */
-/* package */ final class ChannelMixingAudioProcessor extends BaseAudioProcessor {
-
-  @Nullable private ChannelMixingMatrix pendingMatrix;
-  @Nullable private ChannelMixingMatrix matrix;
-  @Nullable private AudioMixingAlgorithm pendingAlgorithm;
-  @Nullable private AudioMixingAlgorithm algorithm;
-
-  public ChannelMixingAudioProcessor(ChannelMixingMatrix matrix) {
-    pendingMatrix = matrix;
-  }
-
-  public void setMatrix(ChannelMixingMatrix matrix) {
-    pendingMatrix = matrix;
-  }
-
-  @Override
-  protected AudioFormat onConfigure(AudioFormat inputAudioFormat)
-      throws UnhandledAudioFormatException {
-    checkStateNotNull(pendingMatrix);
-    // TODO(b/252538025): Allow for a mapping of input channel count -> matrix to be passed in.
-    if (inputAudioFormat.channelCount != pendingMatrix.getInputChannelCount()) {
-      throw new UnhandledAudioFormatException(
-          "Channel count must match mixing matrix", inputAudioFormat);
-    }
-
-    if (pendingMatrix.isIdentity()) {
-      return AudioFormat.NOT_SET;
-    }
-
-    // TODO(b/264926272): Allow config of output PCM config when other AudioMixingAlgorithms exist.
-    AudioFormat pendingOutputAudioFormat =
-        new AudioFormat(
-            inputAudioFormat.sampleRate,
-            pendingMatrix.getOutputChannelCount(),
-            C.ENCODING_PCM_FLOAT);
-
-    pendingAlgorithm = AudioMixingAlgorithm.create(pendingOutputAudioFormat);
-    if (!pendingAlgorithm.supportsSourceAudioFormat(inputAudioFormat)) {
-      throw new UnhandledAudioFormatException(inputAudioFormat);
-    }
-
-    return pendingOutputAudioFormat;
-  }
-
-  @Override
-  protected void onFlush() {
-    algorithm = pendingAlgorithm;
-    matrix = pendingMatrix;
-  }
-
-  @Override
-  protected void onReset() {
-    pendingAlgorithm = null;
-    algorithm = null;
-    pendingMatrix = null;
-    matrix = null;
-  }
-
-  @Override
-  public void queueInput(ByteBuffer inputBuffer) {
-    int inputFramesToMix = inputBuffer.remaining() / inputAudioFormat.bytesPerFrame;
-    ByteBuffer outputBuffer =
-        replaceOutputBuffer(inputFramesToMix * outputAudioFormat.bytesPerFrame);
-    checkNotNull(algorithm)
-        .mix(inputBuffer, inputAudioFormat, checkNotNull(matrix), inputFramesToMix, outputBuffer);
-    outputBuffer.flip();
-  }
-}

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FloatAudioMixingAlgorithm.java

@@ -21,6 +21,7 @@ import android.annotation.SuppressLint;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 
 
 /** An {@link AudioMixingAlgorithm} which mixes into float samples. */
 /** An {@link AudioMixingAlgorithm} which mixes into float samples. */

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ChannelMixingMatrixTest.java

@@ -18,6 +18,7 @@ package com.google.android.exoplayer2.transformer;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertThat;
 
 
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import org.junit.Test;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runner.RunWith;
 
 

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FloatAudioMixingAlgorithmTest.java

@@ -23,6 +23,7 @@ import static com.google.common.truth.Truth.assertWithMessage;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
+import com.google.android.exoplayer2.audio.ChannelMixingMatrix;
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 import org.junit.Test;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runner.RunWith;