Render last frame even if have not read BUFFER_FLAG_END_OF_STREAM

If the limited number of input buffers causes reading of all samples except the last one conveying end of stream, then the last frame will not be rendered.

PiperOrigin-RevId: 525974445

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -1247,7 +1247,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       return true;
     }
 
-    if (hasReadStreamToEnd()) {
+    if (hasReadStreamToEnd() || buffer.isLastSample()) {
       // Notify output queue of the last buffer's timestamp.
       lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
     }

library/core/src/main/java/com/google/android/exoplayer2/source/SampleQueue.java

@@ -714,6 +714,9 @@ public class SampleQueue implements TrackOutput {
     }
 
     buffer.setFlags(flags[relativeReadIndex]);
+    if (readPosition == (length - 1) && (loadingFinished || isLastSampleQueued)) {
+      buffer.addFlag(C.BUFFER_FLAG_LAST_SAMPLE);
+    }
     buffer.timeUs = timesUs[relativeReadIndex];
     if (buffer.timeUs < startTimeUs) {
       buffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY);

library/core/src/test/java/com/google/android/exoplayer2/source/SampleQueueTest.java

@@ -355,6 +355,32 @@ public final class SampleQueueTest {
   }
 
   @Test
+  public void readSingleSampleWithLoadingFinished() {
+    sampleQueue.sampleData(new ParsableByteArray(DATA), ALLOCATION_SIZE);
+    sampleQueue.format(FORMAT_1);
+    sampleQueue.sampleMetadata(1000, C.BUFFER_FLAG_KEY_FRAME, ALLOCATION_SIZE, 0, null);
+
+    assertAllocationCount(1);
+    // If formatRequired, should read the format rather than the sample.
+    assertReadFormat(true, FORMAT_1);
+    // Otherwise should read the sample with loading finished.
+    assertReadLastSample(
+        1000,
+        /* isKeyFrame= */ true,
+        /* isDecodeOnly= */ false,
+        /* isEncrypted= */ false,
+        DATA,
+        /* offset= */ 0,
+        ALLOCATION_SIZE);
+    // Allocation should still be held.
+    assertAllocationCount(1);
+
+    sampleQueue.discardToRead();
+    // The allocation should have been released.
+    assertAllocationCount(0);
+  }
+
+  @Test
   public void readMultiSamples() {
     writeTestData();
     assertThat(sampleQueue.getLargestQueuedTimestampUs()).isEqualTo(LAST_SAMPLE_TIMESTAMP);
@@ -1642,13 +1668,27 @@ public final class SampleQueueTest {
             FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK,
             /* loadingFinished= */ false);
     assertSampleBufferReadResult(
-        flagsOnlyBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted);
+        flagsOnlyBuffer,
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ false);
 
     // Check that peek yields the expected values.
     clearFormatHolderAndInputBuffer();
     result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ false);
     assertSampleBufferReadResult(
-        result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length);
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ false,
+        sampleData,
+        offset,
+        length);
 
     // Check that read yields the expected values.
     clearFormatHolderAndInputBuffer();
@@ -1656,7 +1696,85 @@ public final class SampleQueueTest {
         sampleQueue.read(
             formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ false);
     assertSampleBufferReadResult(
-        result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length);
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ false,
+        sampleData,
+        offset,
+        length);
+  }
+
+  /**
+   * Asserts {@link SampleQueue#read} returns {@link C#RESULT_BUFFER_READ} and that the buffer is
+   * filled with the specified sample data. Also asserts that being the last sample and loading is
+   * finished, that the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set.
+   *
+   * @param timeUs The expected buffer timestamp.
+   * @param isKeyFrame The expected keyframe flag.
+   * @param isDecodeOnly The expected decodeOnly flag.
+   * @param isEncrypted The expected encrypted flag.
+   * @param sampleData An array containing the expected sample data.
+   * @param offset The offset in {@code sampleData} of the expected sample data.
+   * @param length The length of the expected sample data.
+   */
+  private void assertReadLastSample(
+      long timeUs,
+      boolean isKeyFrame,
+      boolean isDecodeOnly,
+      boolean isEncrypted,
+      byte[] sampleData,
+      int offset,
+      int length) {
+    // Check that peek whilst omitting data yields the expected values.
+    formatHolder.format = null;
+    DecoderInputBuffer flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
+    int result =
+        sampleQueue.read(
+            formatHolder,
+            flagsOnlyBuffer,
+            FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK,
+            /* loadingFinished= */ true);
+    assertSampleBufferReadResult(
+        flagsOnlyBuffer,
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ true);
+
+    // Check that peek yields the expected values.
+    clearFormatHolderAndInputBuffer();
+    result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ true);
+    assertSampleBufferReadResult(
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ true,
+        sampleData,
+        offset,
+        length);
+
+    // Check that read yields the expected values.
+    clearFormatHolderAndInputBuffer();
+    result =
+        sampleQueue.read(
+            formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ true);
+    assertSampleBufferReadResult(
+        result,
+        timeUs,
+        isKeyFrame,
+        isDecodeOnly,
+        isEncrypted,
+        /* isLastSample= */ true,
+        sampleData,
+        offset,
+        length);
   }
 
   private void assertSampleBufferReadResult(
@@ -1665,7 +1783,8 @@ public final class SampleQueueTest {
       long timeUs,
       boolean isKeyFrame,
       boolean isDecodeOnly,
-      boolean isEncrypted) {
+      boolean isEncrypted,
+      boolean isLastSample) {
     assertThat(result).isEqualTo(RESULT_BUFFER_READ);
     // formatHolder should not be populated.
     assertThat(formatHolder.format).isNull();
@@ -1674,6 +1793,7 @@ public final class SampleQueueTest {
     assertThat(inputBuffer.isKeyFrame()).isEqualTo(isKeyFrame);
     assertThat(inputBuffer.isDecodeOnly()).isEqualTo(isDecodeOnly);
     assertThat(inputBuffer.isEncrypted()).isEqualTo(isEncrypted);
+    assertThat(inputBuffer.isLastSample()).isEqualTo(isLastSample);
   }
 
   private void assertSampleBufferReadResult(
@@ -1682,11 +1802,12 @@ public final class SampleQueueTest {
       boolean isKeyFrame,
       boolean isDecodeOnly,
       boolean isEncrypted,
+      boolean isLastSample,
       byte[] sampleData,
       int offset,
       int length) {
     assertSampleBufferReadResult(
-        inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted);
+        inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, isLastSample);
     // inputBuffer should be populated with data.
     inputBuffer.flip();
     assertThat(inputBuffer.data.limit()).isEqualTo(length);

library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java

@@ -31,11 +31,14 @@ import static org.robolectric.Shadows.shadowOf;
 import android.content.Context;
 import android.graphics.SurfaceTexture;
 import android.hardware.display.DisplayManager;
+import android.media.MediaCodec;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaCodecInfo.CodecProfileLevel;
 import android.media.MediaFormat;
+import android.os.Bundle;
 import android.os.Handler;
 import android.os.Looper;
+import android.os.PersistableBundle;
 import android.os.SystemClock;
 import android.view.Display;
 import android.view.Surface;
@@ -49,14 +52,20 @@ import com.google.android.exoplayer2.RendererCapabilities;
 import com.google.android.exoplayer2.RendererCapabilities.Capabilities;
 import com.google.android.exoplayer2.RendererConfiguration;
 import com.google.android.exoplayer2.analytics.PlayerId;
+import com.google.android.exoplayer2.decoder.CryptoInfo;
+import com.google.android.exoplayer2.decoder.DecoderCounters;
 import com.google.android.exoplayer2.drm.DrmSessionEventListener;
 import com.google.android.exoplayer2.drm.DrmSessionManager;
+import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter;
 import com.google.android.exoplayer2.mediacodec.MediaCodecInfo;
 import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
+import com.google.android.exoplayer2.mediacodec.SynchronousMediaCodecAdapter;
 import com.google.android.exoplayer2.testutil.FakeSampleStream;
 import com.google.android.exoplayer2.upstream.DefaultAllocator;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -116,6 +125,7 @@ public class MediaCodecVideoRendererTest {
   private Looper testMainLooper;
   private Surface surface;
   private MediaCodecVideoRenderer mediaCodecVideoRenderer;
+  private MediaCodecSelector mediaCodecSelector;
   @Nullable private Format currentOutputFormat;
 
   @Mock private VideoRendererEventListener eventListener;
@@ -123,7 +133,7 @@ public class MediaCodecVideoRendererTest {
   @Before
   public void setUp() throws Exception {
     testMainLooper = Looper.getMainLooper();
-    MediaCodecSelector mediaCodecSelector =
+    mediaCodecSelector =
         (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) ->
             Collections.singletonList(
                 MediaCodecInfo.newInstance(
@@ -207,6 +217,65 @@ public class MediaCodecVideoRendererTest {
   }
 
   @Test
+  public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEndOfStream()
+      throws Exception {
+    ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
+        ArgumentCaptor.forClass(DecoderCounters.class);
+    FakeSampleStream fakeSampleStream =
+        new FakeSampleStream(
+            new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
+            /* mediaSourceEventDispatcher= */ null,
+            DrmSessionManager.DRM_UNSUPPORTED,
+            new DrmSessionEventListener.EventDispatcher(),
+            /* initialFormat= */ VIDEO_H264,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
+                oneByteSample(/* timeUs= */ 10_000),
+                oneByteSample(/* timeUs= */ 20_000), // Last buffer.
+                END_OF_STREAM_ITEM));
+    fakeSampleStream.writeData(/* startPositionUs= */ 0);
+    // Seek to time after samples.
+    fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true);
+    mediaCodecVideoRenderer =
+        new MediaCodecVideoRenderer(
+            ApplicationProvider.getApplicationContext(),
+            new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
+            mediaCodecSelector,
+            /* allowedJoiningTimeMs= */ 0,
+            /* enableDecoderFallback= */ false,
+            /* eventHandler= */ new Handler(testMainLooper),
+            /* eventListener= */ eventListener,
+            /* maxDroppedFramesToNotify= */ 1);
+    mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
+    mediaCodecVideoRenderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {VIDEO_H264},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 0,
+        /* offsetUs= */ 0);
+
+    mediaCodecVideoRenderer.start();
+    mediaCodecVideoRenderer.setCurrentStreamFinal();
+    mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
+    // Call to render should have read all samples up to but not including the END_OF_STREAM_ITEM.
+    assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isFalse();
+    int posUs = 30_000;
+    while (!mediaCodecVideoRenderer.isEnded()) {
+      mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
+      posUs += 40_000;
+    }
+    shadowOf(testMainLooper).idle();
+
+    verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
+    verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
+    assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
+    assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
+  }
+
+  @Test
   public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception {
     FakeSampleStream fakeSampleStream =
         new FakeSampleStream(
@@ -1193,4 +1262,146 @@ public class MediaCodecVideoRendererTest {
         .setHeight(height)
         .build();
   }
+
+  private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit
+      extends ForwardingSynchronousMediaCodecAdapter {
+    /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */
+    public static final class Factory implements MediaCodecAdapter.Factory {
+      private final int bufferLimit;
+
+      Factory(int bufferLimit) {
+        this.bufferLimit = bufferLimit;
+      }
+
+      @Override
+      public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException {
+        return new ForwardingSynchronousMediaCodecAdapterWithBufferLimit(
+            bufferLimit, new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration));
+      }
+    }
+
+    private int bufferCounter;
+
+    ForwardingSynchronousMediaCodecAdapterWithBufferLimit(
+        int bufferCounter, MediaCodecAdapter adapter) {
+      super(adapter);
+      this.bufferCounter = bufferCounter;
+    }
+
+    @Override
+    public int dequeueInputBufferIndex() {
+      if (bufferCounter > 0) {
+        bufferCounter--;
+        return super.dequeueInputBufferIndex();
+      }
+      return -1;
+    }
+
+    @Override
+    public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
+      int outputIndex = super.dequeueOutputBufferIndex(bufferInfo);
+      if (outputIndex > 0) {
+        bufferCounter++;
+      }
+      return outputIndex;
+    }
+  }
+
+  private abstract static class ForwardingSynchronousMediaCodecAdapter
+      implements MediaCodecAdapter {
+    private final MediaCodecAdapter adapter;
+
+    ForwardingSynchronousMediaCodecAdapter(MediaCodecAdapter adapter) {
+      this.adapter = adapter;
+    }
+
+    @Override
+    public int dequeueInputBufferIndex() {
+      return adapter.dequeueInputBufferIndex();
+    }
+
+    @Override
+    public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
+      return adapter.dequeueOutputBufferIndex(bufferInfo);
+    }
+
+    @Override
+    public MediaFormat getOutputFormat() {
+      return adapter.getOutputFormat();
+    }
+
+    @Nullable
+    @Override
+    public ByteBuffer getInputBuffer(int index) {
+      return adapter.getInputBuffer(index);
+    }
+
+    @Nullable
+    @Override
+    public ByteBuffer getOutputBuffer(int index) {
+      return adapter.getOutputBuffer(index);
+    }
+
+    @Override
+    public void queueInputBuffer(
+        int index, int offset, int size, long presentationTimeUs, int flags) {
+      adapter.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
+    }
+
+    @Override
+    public void queueSecureInputBuffer(
+        int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
+      adapter.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags);
+    }
+
+    @Override
+    public void releaseOutputBuffer(int index, boolean render) {
+      adapter.releaseOutputBuffer(index, render);
+    }
+
+    @Override
+    public void releaseOutputBuffer(int index, long renderTimeStampNs) {
+      adapter.releaseOutputBuffer(index, renderTimeStampNs);
+    }
+
+    @Override
+    public void flush() {
+      adapter.flush();
+    }
+
+    @Override
+    public void release() {
+      adapter.release();
+    }
+
+    @Override
+    public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) {
+      adapter.setOnFrameRenderedListener(listener, handler);
+    }
+
+    @Override
+    public void setOutputSurface(Surface surface) {
+      adapter.setOutputSurface(surface);
+    }
+
+    @Override
+    public void setParameters(Bundle params) {
+      adapter.setParameters(params);
+    }
+
+    @Override
+    public void setVideoScalingMode(int scalingMode) {
+      adapter.setVideoScalingMode(scalingMode);
+    }
+
+    @Override
+    public boolean needsReconfiguration() {
+      return adapter.needsReconfiguration();
+    }
+
+    @Override
+    public PersistableBundle getMetrics() {
+      return adapter.getMetrics();
+    }
+  }
 }

library/decoder/src/main/java/com/google/android/exoplayer2/decoder/Buffer.java

@@ -47,6 +47,11 @@ public abstract class Buffer {
     return getFlag(C.BUFFER_FLAG_KEY_FRAME);
   }
 
+  /** Returns whether the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set. */
+  public final boolean isLastSample() {
+    return getFlag(C.BUFFER_FLAG_LAST_SAMPLE);
+  }
+
   /** Returns whether the {@link C#BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA} flag is set. */
   public final boolean hasSupplementalData() {
     return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA);

testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeMediaPeriod.java

@@ -336,7 +336,8 @@ public class FakeMediaPeriod implements MediaPeriod {
     lastSeekPositionUs = seekPositionUs;
     boolean seekedInsideStreams = true;
     for (FakeSampleStream sampleStream : sampleStreams) {
-      seekedInsideStreams &= sampleStream.seekToUs(seekPositionUs);
+      seekedInsideStreams &=
+          sampleStream.seekToUs(seekPositionUs, /* allowTimeBeyondBuffer= */ false);
     }
     if (!seekedInsideStreams) {
       for (FakeSampleStream sampleStream : sampleStreams) {

testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeSampleStream.java

@@ -202,10 +202,12 @@ public class FakeSampleStream implements SampleStream {
    * Seeks the stream to a new position using already available data in the queue.
    *
    * @param positionUs The new position, in microseconds.
+   * @param allowTimeBeyondBuffer Whether the operation can succeed if timeUs is beyond the end of
+   *     the queue, by seeking to the last sample (or keyframe).
    * @return Whether seeking inside the available data was possible.
    */
-  public boolean seekToUs(long positionUs) {
-    return sampleQueue.seekTo(positionUs, /* allowTimeBeyondBuffer= */ false);
+  public boolean seekToUs(long positionUs, boolean allowTimeBeyondBuffer) {
+    return sampleQueue.seekTo(positionUs, allowTimeBeyondBuffer);
   }
 
   /**