Skip to content

SDK / exoplayer

Go to a project
Commit 5335b258 by eguven Committed by Oliver Woodman
Options

Make SceneRenderer implement VideoFrameMetadataListener and CameraMotionListener 

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=211084127
parent 514edb69
Inline Side-by-side
Showing with 134 additions and 127 deletions
library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
......@@ -513,9 +513,7 @@ public class PlayerView extends FrameLayout {
if (surfaceView instanceof TextureView) {
oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) {
oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView));
((SphericalSurfaceView) surfaceView).setVideoComponent(null);
} else if (surfaceView instanceof SurfaceView) {
oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
}
......@@ -541,9 +539,7 @@ public class PlayerView extends FrameLayout {
if (surfaceView instanceof TextureView) {
newVideoComponent.setVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) {
newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView));
newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
} else if (surfaceView instanceof SurfaceView) {
newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
}
......
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java
......@@ -21,7 +21,6 @@ import android.annotation.TargetApi;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.video.spherical.Projection;
import java.nio.FloatBuffer;
import org.checkerframework.checker.nullness.qual.Nullable;
......@@ -143,13 +142,19 @@ import org.checkerframework.checker.nullness.qual.Nullable;
}
/**
* Renders the mesh. This must be called on the GL thread.
* Renders the mesh. If the projection hasn't been set, does nothing. This must be called on the
* GL thread.
*
* @param textureId GL_TEXTURE_EXTERNAL_OES used for this mesh.
* @param mvpMatrix The Model View Projection matrix.
* @param eyeType An {@link EyeType} value.
*/
/* package */ void draw(int textureId, float[] mvpMatrix, int eyeType) {
MeshData meshData = eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData;
if (meshData == null) {
return;
}
// Configure shader.
GLES20.glUseProgram(program);
checkGlError();
......@@ -174,9 +179,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
GLES20.glUniform1i(textureHandle, 0);
checkGlError();
MeshData meshData =
Assertions.checkNotNull(eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData);
// Load position data.
GLES20.glVertexAttribPointer(
positionHandle,
......
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java
......@@ -21,49 +21,68 @@ import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.support.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection;
import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Renders a GL Scene.
*
* <p>All methods should be called only on the GL thread unless GL thread is stopped.
*/
/*package*/ final class SceneRenderer {
/** Renders a GL Scene. */
/*package*/ class SceneRenderer implements VideoFrameMetadataListener, CameraMotionListener {
private final AtomicBoolean frameAvailable;
private final AtomicBoolean resetRotationAtNextFrame;
private final ProjectionRenderer projectionRenderer;
private final FrameRotationQueue frameRotationQueue;
private final TimedValueQueue<Long> sampleTimestampQueue;
private final TimedValueQueue<Projection> projectionQueue;
private final float[] rotationMatrix;
private final float[] tempMatrix;
// Used by GL thread only
private int textureId;
private @MonotonicNonNull SurfaceTexture surfaceTexture;
private @Nullable Projection pendingProjection;
private long pendingProjectionTimeNs;
private long lastFrameTimestamp;
private boolean resetRotationAtNextFrame;
public SceneRenderer(
Projection projection,
FrameRotationQueue frameRotationQueue,
TimedValueQueue<Long> sampleTimestampQueue) {
this.frameRotationQueue = frameRotationQueue;
this.sampleTimestampQueue = sampleTimestampQueue;
// Used by other threads only
private volatile @C.StreamType int defaultStereoMode;
private @C.StreamType int lastStereoMode;
private @Nullable byte[] lastProjectionData;
// Methods called on any thread.
public SceneRenderer() {
frameAvailable = new AtomicBoolean();
resetRotationAtNextFrame = new AtomicBoolean(true);
projectionRenderer = new ProjectionRenderer();
projectionRenderer.setProjection(projection);
frameRotationQueue = new FrameRotationQueue();
sampleTimestampQueue = new TimedValueQueue<>();
projectionQueue = new TimedValueQueue<>();
rotationMatrix = new float[16];
tempMatrix = new float[16];
resetRotation();
defaultStereoMode = C.STEREO_MODE_MONO;
lastStereoMode = Format.NO_VALUE;
}
/**
* Sets the default stereo mode. If the played video doesn't contain a stereo mode the default one
* is used.
*
* @param stereoMode A {@link C.StereoMode} value.
*/
public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
defaultStereoMode = stereoMode;
}
// Methods called on GL thread.
/** Initializes the renderer. */
public SurfaceTexture init() {
// Set the background frame color. This is only visible if the display mesh isn't a full sphere.
......@@ -79,21 +98,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return surfaceTexture;
}
public void resetRotation() {
resetRotationAtNextFrame = true;
}
/** Sets a {@link Projection} to be used to display video. */
public void setProjection(Projection projection, long timeNs) {
pendingProjection = projection;
pendingProjectionTimeNs = timeNs;
}
/**
* Draws the scene with a given eye pose and type.
*
* @param viewProjectionMatrix 16 element GL matrix.
* @param eyeType an {@link EyeType} value
* @param eyeType An {@link EyeType} value
*/
public void drawFrame(float[] viewProjectionMatrix, int eyeType) {
// glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve
......@@ -104,20 +113,73 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (frameAvailable.compareAndSet(true, false)) {
Assertions.checkNotNull(surfaceTexture).updateTexImage();
checkGlError();
if (resetRotationAtNextFrame) {
if (resetRotationAtNextFrame.compareAndSet(true, false)) {
Matrix.setIdentityM(rotationMatrix, 0);
}
lastFrameTimestamp = surfaceTexture.getTimestamp();
Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp);
if (sampleTimestamp != null) {
frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp);
long lastFrameTimestampNs = surfaceTexture.getTimestamp();
Long sampleTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs);
if (sampleTimestampUs != null) {
frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestampUs);
}
Projection projection = projectionQueue.pollFloor(lastFrameTimestampNs);
if (projection != null) {
projectionRenderer.setProjection(projection);
}
}
if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) {
projectionRenderer.setProjection(pendingProjection);
pendingProjection = null;
}
Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
projectionRenderer.draw(textureId, tempMatrix, eyeType);
}
// Methods called on playback thread.
// VideoFrameMetadataListener implementation.
@Override
public void onVideoFrameAboutToBeRendered(
long presentationTimeUs, long releaseTimeNs, Format format) {
sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
}
// CameraMotionListener implementation.
@Override
public void onCameraMotion(long timeUs, float[] rotation) {
frameRotationQueue.setRotation(timeUs, rotation);
}
@Override
public void onCameraMotionReset() {
sampleTimestampQueue.clear();
frameRotationQueue.reset();
resetRotationAtNextFrame.set(true);
}
/**
* Sets projection data and stereo mode of the media to be played.
*
* @param projectionData Contains the projection data to be rendered.
* @param stereoMode A {@link C.StereoMode} value.
* @param timeNs When then new projection should be used.
*/
private void setProjection(
@Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
byte[] oldProjectionData = lastProjectionData;
int oldStereoMode = lastStereoMode;
lastProjectionData = projectionData;
lastStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
if (oldStereoMode == lastStereoMode && Arrays.equals(oldProjectionData, lastProjectionData)) {
return;
}
Projection projectionFromData = null;
if (lastProjectionData != null) {
projectionFromData = ProjectionDecoder.decode(lastProjectionData, lastStereoMode);
}
Projection projection =
projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
? projectionFromData
: Projection.createEquirectangular(lastStereoMode);
projectionQueue.add(timeNs, projection);
}
}
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java
......@@ -37,17 +37,10 @@ import android.view.Display;
import android.view.Surface;
import android.view.WindowManager;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection;
import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
import java.util.Arrays;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
......@@ -62,8 +55,7 @@ import javax.microedition.khronos.opengles.GL10;
* match what they expect.
*/
@TargetApi(15)
public final class SphericalSurfaceView extends GLSurfaceView
implements VideoFrameMetadataListener, CameraMotionListener {
public final class SphericalSurfaceView extends GLSurfaceView {
/**
* This listener can be used to be notified when the {@link Surface} associated with this view is
......@@ -94,15 +86,12 @@ public final class SphericalSurfaceView extends GLSurfaceView
private final PhoneOrientationListener phoneOrientationListener;
private final Renderer renderer;
private final Handler mainHandler;
private final TimedValueQueue<Long> sampleTimestampQueue;
private final FrameRotationQueue frameRotationQueue;
private final TouchTracker touchTracker;
private final SceneRenderer scene;
private @Nullable SurfaceListener surfaceListener;
private @Nullable SurfaceTexture surfaceTexture;
private @Nullable Surface surface;
private @C.StreamType int defaultStereoMode;
private @C.StreamType int currentStereoMode;
private @Nullable byte[] currentProjectionData;
private @Nullable Player.VideoComponent videoComponent;
public SphericalSurfaceView(Context context) {
this(context, null);
......@@ -122,12 +111,7 @@ public final class SphericalSurfaceView extends GLSurfaceView
int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR;
orientationSensor = sensorManager.getDefaultSensor(type);
defaultStereoMode = C.STEREO_MODE_MONO;
currentStereoMode = defaultStereoMode;
Projection projection = Projection.createEquirectangular(defaultStereoMode);
frameRotationQueue = new FrameRotationQueue();
sampleTimestampQueue = new TimedValueQueue<>();
SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue);
scene = new SceneRenderer();
renderer = new Renderer(scene);
touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES);
......@@ -147,12 +131,27 @@ public final class SphericalSurfaceView extends GLSurfaceView
* @param stereoMode A {@link C.StereoMode} value.
*/
public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
defaultStereoMode = stereoMode;
scene.setDefaultStereoMode(stereoMode);
}
/** Returns the {@link Surface} associated with this view. */
public @Nullable Surface getSurface() {
return surface;
/** Sets the {@link Player.VideoComponent} to use. */
public void setVideoComponent(@Nullable Player.VideoComponent newVideoComponent) {
if (newVideoComponent == videoComponent) {
return;
}
if (videoComponent != null) {
if (surface != null) {
videoComponent.clearVideoSurface(surface);
}
videoComponent.clearVideoFrameMetadataListener(scene);
videoComponent.clearCameraMotionListener(scene);
}
videoComponent = newVideoComponent;
if (videoComponent != null) {
videoComponent.setVideoFrameMetadataListener(scene);
videoComponent.setCameraMotionListener(scene);
videoComponent.setVideoSurface(surface);
}
}
/**
......@@ -169,29 +168,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
touchTracker.setSingleTapListener(listener);
}
// VideoFrameMetadataListener implementation.
@Override
public void onVideoFrameAboutToBeRendered(
long presentationTimeUs, long releaseTimeNs, Format format) {
sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
}
// CameraMotionListener implementation.
@Override
public void onCameraMotion(long timeUs, float[] rotation) {
frameRotationQueue.setRotation(timeUs, rotation);
}
@Override
public void onCameraMotionReset() {
sampleTimestampQueue.clear();
frameRotationQueue.reset();
queueEvent(renderer.scene::resetRotation);
}
@Override
public void onResume() {
super.onResume();
......@@ -253,35 +229,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
}
}
/**
* Sets projection data and stereo mode of the media to be played.
*
* @param projectionData Contains the projection data to be rendered.
* @param stereoMode A {@link C.StereoMode} value.
* @param timeNs When then new projection should be used.
*/
private void setProjection(
@Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
byte[] oldProjectionData = currentProjectionData;
int oldStereoMode = currentStereoMode;
currentProjectionData = projectionData;
currentStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
if (oldStereoMode == currentStereoMode
&& Arrays.equals(oldProjectionData, currentProjectionData)) {
return;
}
Projection projectionFromData = null;
if (currentProjectionData != null) {
projectionFromData = ProjectionDecoder.decode(currentProjectionData, currentStereoMode);
}
Projection projection =
projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
? projectionFromData
: Projection.createEquirectangular(currentStereoMode);
queueEvent(() -> renderer.scene.setProjection(projection, timeNs));
}
/** Detects sensor events and saves them as a matrix. */
private static class PhoneOrientationListener implements SensorEventListener {
private final float[] phoneInWorldSpaceMatrix = new float[16];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment