Skip to content

SDK / exoplayer

Go to a project
Commit 5335b258 by eguven Committed by Oliver Woodman
Options

Make SceneRenderer implement VideoFrameMetadataListener and CameraMotionListener 

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=211084127
parent 514edb69
Inline Side-by-side
Showing with 134 additions and 127 deletions
library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
...@@ -513,9 +513,7 @@ public class PlayerView extends FrameLayout { ...@@ -513,9 +513,7 @@ public class PlayerView extends FrameLayout {
if (surfaceView instanceof TextureView) { if (surfaceView instanceof TextureView) {
oldVideoComponent.clearVideoTextureView((TextureView) surfaceView); oldVideoComponent.clearVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) { } else if (surfaceView instanceof SphericalSurfaceView) {
oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); ((SphericalSurfaceView) surfaceView).setVideoComponent(null);
oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView));
} else if (surfaceView instanceof SurfaceView) { } else if (surfaceView instanceof SurfaceView) {
oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView); oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
} }
...@@ -541,9 +539,7 @@ public class PlayerView extends FrameLayout { ...@@ -541,9 +539,7 @@ public class PlayerView extends FrameLayout {
if (surfaceView instanceof TextureView) { if (surfaceView instanceof TextureView) {
newVideoComponent.setVideoTextureView((TextureView) surfaceView); newVideoComponent.setVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) { } else if (surfaceView instanceof SphericalSurfaceView) {
newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); ((SphericalSurfaceView) surfaceView).setVideoComponent(newVideoComponent);
newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView));
newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
} else if (surfaceView instanceof SurfaceView) { } else if (surfaceView instanceof SurfaceView) {
newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView); newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
} }
......
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/ProjectionRenderer.java
...@@ -21,7 +21,6 @@ import android.annotation.TargetApi; ...@@ -21,7 +21,6 @@ import android.annotation.TargetApi;
import android.opengl.GLES11Ext; import android.opengl.GLES11Ext;
import android.opengl.GLES20; import android.opengl.GLES20;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.video.spherical.Projection; import com.google.android.exoplayer2.video.spherical.Projection;
import java.nio.FloatBuffer; import java.nio.FloatBuffer;
import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.Nullable;
...@@ -143,13 +142,19 @@ import org.checkerframework.checker.nullness.qual.Nullable; ...@@ -143,13 +142,19 @@ import org.checkerframework.checker.nullness.qual.Nullable;
} }
/** /**
* Renders the mesh. This must be called on the GL thread. * Renders the mesh. If the projection hasn't been set, does nothing. This must be called on the
* GL thread.
* *
* @param textureId GL_TEXTURE_EXTERNAL_OES used for this mesh. * @param textureId GL_TEXTURE_EXTERNAL_OES used for this mesh.
* @param mvpMatrix The Model View Projection matrix. * @param mvpMatrix The Model View Projection matrix.
* @param eyeType An {@link EyeType} value. * @param eyeType An {@link EyeType} value.
*/ */
/* package */ void draw(int textureId, float[] mvpMatrix, int eyeType) { /* package */ void draw(int textureId, float[] mvpMatrix, int eyeType) {
MeshData meshData = eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData;
if (meshData == null) {
return;
}
// Configure shader. // Configure shader.
GLES20.glUseProgram(program); GLES20.glUseProgram(program);
checkGlError(); checkGlError();
...@@ -174,9 +179,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; ...@@ -174,9 +179,6 @@ import org.checkerframework.checker.nullness.qual.Nullable;
GLES20.glUniform1i(textureHandle, 0); GLES20.glUniform1i(textureHandle, 0);
checkGlError(); checkGlError();
MeshData meshData =
Assertions.checkNotNull(eyeType == EyeType.RIGHT ? rightMeshData : leftMeshData);
// Load position data. // Load position data.
GLES20.glVertexAttribPointer( GLES20.glVertexAttribPointer(
positionHandle, positionHandle,
......
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java
...@@ -21,49 +21,68 @@ import android.graphics.SurfaceTexture; ...@@ -21,49 +21,68 @@ import android.graphics.SurfaceTexture;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.Matrix; import android.opengl.Matrix;
import android.support.annotation.Nullable; import android.support.annotation.Nullable;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue; import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection; import com.google.android.exoplayer2.video.spherical.Projection;
import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /** Renders a GL Scene. */
* Renders a GL Scene. /*package*/ class SceneRenderer implements VideoFrameMetadataListener, CameraMotionListener {
*
* <p>All methods should be called only on the GL thread unless GL thread is stopped.
*/
/*package*/ final class SceneRenderer {
private final AtomicBoolean frameAvailable; private final AtomicBoolean frameAvailable;
private final AtomicBoolean resetRotationAtNextFrame;
private final ProjectionRenderer projectionRenderer; private final ProjectionRenderer projectionRenderer;
private final FrameRotationQueue frameRotationQueue; private final FrameRotationQueue frameRotationQueue;
private final TimedValueQueue<Long> sampleTimestampQueue; private final TimedValueQueue<Long> sampleTimestampQueue;
private final TimedValueQueue<Projection> projectionQueue;
private final float[] rotationMatrix; private final float[] rotationMatrix;
private final float[] tempMatrix; private final float[] tempMatrix;
// Used by GL thread only
private int textureId; private int textureId;
private @MonotonicNonNull SurfaceTexture surfaceTexture; private @MonotonicNonNull SurfaceTexture surfaceTexture;
private @Nullable Projection pendingProjection;
private long pendingProjectionTimeNs; // Used by other threads only
private long lastFrameTimestamp; private volatile @C.StreamType int defaultStereoMode;
private boolean resetRotationAtNextFrame; private @C.StreamType int lastStereoMode;
private @Nullable byte[] lastProjectionData;
public SceneRenderer(
Projection projection, // Methods called on any thread.
FrameRotationQueue frameRotationQueue,
TimedValueQueue<Long> sampleTimestampQueue) { public SceneRenderer() {
this.frameRotationQueue = frameRotationQueue;
this.sampleTimestampQueue = sampleTimestampQueue;
frameAvailable = new AtomicBoolean(); frameAvailable = new AtomicBoolean();
resetRotationAtNextFrame = new AtomicBoolean(true);
projectionRenderer = new ProjectionRenderer(); projectionRenderer = new ProjectionRenderer();
projectionRenderer.setProjection(projection); frameRotationQueue = new FrameRotationQueue();
sampleTimestampQueue = new TimedValueQueue<>();
projectionQueue = new TimedValueQueue<>();
rotationMatrix = new float[16]; rotationMatrix = new float[16];
tempMatrix = new float[16]; tempMatrix = new float[16];
resetRotation(); defaultStereoMode = C.STEREO_MODE_MONO;
lastStereoMode = Format.NO_VALUE;
} }
/**
* Sets the default stereo mode. If the played video doesn't contain a stereo mode the default one
* is used.
*
* @param stereoMode A {@link C.StereoMode} value.
*/
public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
defaultStereoMode = stereoMode;
}
// Methods called on GL thread.
/** Initializes the renderer. */ /** Initializes the renderer. */
public SurfaceTexture init() { public SurfaceTexture init() {
// Set the background frame color. This is only visible if the display mesh isn't a full sphere. // Set the background frame color. This is only visible if the display mesh isn't a full sphere.
...@@ -79,21 +98,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -79,21 +98,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return surfaceTexture; return surfaceTexture;
} }
public void resetRotation() {
resetRotationAtNextFrame = true;
}
/** Sets a {@link Projection} to be used to display video. */
public void setProjection(Projection projection, long timeNs) {
pendingProjection = projection;
pendingProjectionTimeNs = timeNs;
}
/** /**
* Draws the scene with a given eye pose and type. * Draws the scene with a given eye pose and type.
* *
* @param viewProjectionMatrix 16 element GL matrix. * @param viewProjectionMatrix 16 element GL matrix.
* @param eyeType an {@link EyeType} value * @param eyeType An {@link EyeType} value
*/ */
public void drawFrame(float[] viewProjectionMatrix, int eyeType) { public void drawFrame(float[] viewProjectionMatrix, int eyeType) {
// glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve // glClear isn't strictly necessary when rendering fully spherical panoramas, but it can improve
...@@ -104,20 +113,73 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -104,20 +113,73 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (frameAvailable.compareAndSet(true, false)) { if (frameAvailable.compareAndSet(true, false)) {
Assertions.checkNotNull(surfaceTexture).updateTexImage(); Assertions.checkNotNull(surfaceTexture).updateTexImage();
checkGlError(); checkGlError();
if (resetRotationAtNextFrame) { if (resetRotationAtNextFrame.compareAndSet(true, false)) {
Matrix.setIdentityM(rotationMatrix, 0); Matrix.setIdentityM(rotationMatrix, 0);
} }
lastFrameTimestamp = surfaceTexture.getTimestamp(); long lastFrameTimestampNs = surfaceTexture.getTimestamp();
Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp); Long sampleTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs);
if (sampleTimestamp != null) { if (sampleTimestampUs != null) {
frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp); frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestampUs);
}
Projection projection = projectionQueue.pollFloor(lastFrameTimestampNs);
if (projection != null) {
projectionRenderer.setProjection(projection);
} }
}
if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) {
projectionRenderer.setProjection(pendingProjection);
pendingProjection = null;
} }
Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0); Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
projectionRenderer.draw(textureId, tempMatrix, eyeType); projectionRenderer.draw(textureId, tempMatrix, eyeType);
} }
// Methods called on playback thread.
// VideoFrameMetadataListener implementation.
@Override
public void onVideoFrameAboutToBeRendered(
long presentationTimeUs, long releaseTimeNs, Format format) {
sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
}
// CameraMotionListener implementation.
@Override
public void onCameraMotion(long timeUs, float[] rotation) {
frameRotationQueue.setRotation(timeUs, rotation);
}
@Override
public void onCameraMotionReset() {
sampleTimestampQueue.clear();
frameRotationQueue.reset();
resetRotationAtNextFrame.set(true);
}
/**
* Sets projection data and stereo mode of the media to be played.
*
* @param projectionData Contains the projection data to be rendered.
* @param stereoMode A {@link C.StereoMode} value.
* @param timeNs When then new projection should be used.
*/
private void setProjection(
@Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
byte[] oldProjectionData = lastProjectionData;
int oldStereoMode = lastStereoMode;
lastProjectionData = projectionData;
lastStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
if (oldStereoMode == lastStereoMode && Arrays.equals(oldProjectionData, lastProjectionData)) {
return;
}
Projection projectionFromData = null;
if (lastProjectionData != null) {
projectionFromData = ProjectionDecoder.decode(lastProjectionData, lastStereoMode);
}
Projection projection =
projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
? projectionFromData
: Projection.createEquirectangular(lastStereoMode);
projectionQueue.add(timeNs, projection);
}
} }
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java
...@@ -37,17 +37,10 @@ import android.view.Display; ...@@ -37,17 +37,10 @@ import android.view.Display;
import android.view.Surface; import android.view.Surface;
import android.view.WindowManager; import android.view.WindowManager;
import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection;
import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
import java.util.Arrays;
import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10; import javax.microedition.khronos.opengles.GL10;
...@@ -62,8 +55,7 @@ import javax.microedition.khronos.opengles.GL10; ...@@ -62,8 +55,7 @@ import javax.microedition.khronos.opengles.GL10;
* match what they expect. * match what they expect.
*/ */
@TargetApi(15) @TargetApi(15)
public final class SphericalSurfaceView extends GLSurfaceView public final class SphericalSurfaceView extends GLSurfaceView {
implements VideoFrameMetadataListener, CameraMotionListener {
/** /**
* This listener can be used to be notified when the {@link Surface} associated with this view is * This listener can be used to be notified when the {@link Surface} associated with this view is
...@@ -94,15 +86,12 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -94,15 +86,12 @@ public final class SphericalSurfaceView extends GLSurfaceView
private final PhoneOrientationListener phoneOrientationListener; private final PhoneOrientationListener phoneOrientationListener;
private final Renderer renderer; private final Renderer renderer;
private final Handler mainHandler; private final Handler mainHandler;
private final TimedValueQueue<Long> sampleTimestampQueue;
private final FrameRotationQueue frameRotationQueue;
private final TouchTracker touchTracker; private final TouchTracker touchTracker;
private final SceneRenderer scene;
private @Nullable SurfaceListener surfaceListener; private @Nullable SurfaceListener surfaceListener;
private @Nullable SurfaceTexture surfaceTexture; private @Nullable SurfaceTexture surfaceTexture;
private @Nullable Surface surface; private @Nullable Surface surface;
private @C.StreamType int defaultStereoMode; private @Nullable Player.VideoComponent videoComponent;
private @C.StreamType int currentStereoMode;
private @Nullable byte[] currentProjectionData;
public SphericalSurfaceView(Context context) { public SphericalSurfaceView(Context context) {
this(context, null); this(context, null);
...@@ -122,12 +111,7 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -122,12 +111,7 @@ public final class SphericalSurfaceView extends GLSurfaceView
int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR; int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR;
orientationSensor = sensorManager.getDefaultSensor(type); orientationSensor = sensorManager.getDefaultSensor(type);
defaultStereoMode = C.STEREO_MODE_MONO; scene = new SceneRenderer();
currentStereoMode = defaultStereoMode;
Projection projection = Projection.createEquirectangular(defaultStereoMode);
frameRotationQueue = new FrameRotationQueue();
sampleTimestampQueue = new TimedValueQueue<>();
SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue);
renderer = new Renderer(scene); renderer = new Renderer(scene);
touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES); touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES);
...@@ -147,12 +131,27 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -147,12 +131,27 @@ public final class SphericalSurfaceView extends GLSurfaceView
* @param stereoMode A {@link C.StereoMode} value. * @param stereoMode A {@link C.StereoMode} value.
*/ */
public void setDefaultStereoMode(@C.StereoMode int stereoMode) { public void setDefaultStereoMode(@C.StereoMode int stereoMode) {
defaultStereoMode = stereoMode; scene.setDefaultStereoMode(stereoMode);
} }
/** Returns the {@link Surface} associated with this view. */ /** Sets the {@link Player.VideoComponent} to use. */
public @Nullable Surface getSurface() { public void setVideoComponent(@Nullable Player.VideoComponent newVideoComponent) {
return surface; if (newVideoComponent == videoComponent) {
return;
}
if (videoComponent != null) {
if (surface != null) {
videoComponent.clearVideoSurface(surface);
}
videoComponent.clearVideoFrameMetadataListener(scene);
videoComponent.clearCameraMotionListener(scene);
}
videoComponent = newVideoComponent;
if (videoComponent != null) {
videoComponent.setVideoFrameMetadataListener(scene);
videoComponent.setCameraMotionListener(scene);
videoComponent.setVideoSurface(surface);
}
} }
/** /**
...@@ -169,29 +168,6 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -169,29 +168,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
touchTracker.setSingleTapListener(listener); touchTracker.setSingleTapListener(listener);
} }
// VideoFrameMetadataListener implementation.
@Override
public void onVideoFrameAboutToBeRendered(
long presentationTimeUs, long releaseTimeNs, Format format) {
sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
}
// CameraMotionListener implementation.
@Override
public void onCameraMotion(long timeUs, float[] rotation) {
frameRotationQueue.setRotation(timeUs, rotation);
}
@Override
public void onCameraMotionReset() {
sampleTimestampQueue.clear();
frameRotationQueue.reset();
queueEvent(renderer.scene::resetRotation);
}
@Override @Override
public void onResume() { public void onResume() {
super.onResume(); super.onResume();
...@@ -253,35 +229,6 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -253,35 +229,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
} }
} }
/**
* Sets projection data and stereo mode of the media to be played.
*
* @param projectionData Contains the projection data to be rendered.
* @param stereoMode A {@link C.StereoMode} value.
* @param timeNs When then new projection should be used.
*/
private void setProjection(
@Nullable byte[] projectionData, @C.StereoMode int stereoMode, long timeNs) {
byte[] oldProjectionData = currentProjectionData;
int oldStereoMode = currentStereoMode;
currentProjectionData = projectionData;
currentStereoMode = stereoMode == Format.NO_VALUE ? defaultStereoMode : stereoMode;
if (oldStereoMode == currentStereoMode
&& Arrays.equals(oldProjectionData, currentProjectionData)) {
return;
}
Projection projectionFromData = null;
if (currentProjectionData != null) {
projectionFromData = ProjectionDecoder.decode(currentProjectionData, currentStereoMode);
}
Projection projection =
projectionFromData != null && ProjectionRenderer.isSupported(projectionFromData)
? projectionFromData
: Projection.createEquirectangular(currentStereoMode);
queueEvent(() -> renderer.scene.setProjection(projection, timeNs));
}
/** Detects sensor events and saves them as a matrix. */ /** Detects sensor events and saves them as a matrix. */
private static class PhoneOrientationListener implements SensorEventListener { private static class PhoneOrientationListener implements SensorEventListener {
private final float[] phoneInWorldSpaceMatrix = new float[16]; private final float[] phoneInWorldSpaceMatrix = new float[16];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment