Skip to content

SDK / exoplayer

Go to a project
Commit a429f481 by eguven Committed by Oliver Woodman
Options

Use camera motion metadata to stabilize 360 videos 

RELNOTES=true

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=210537375
parent ce1d8d6c
Inline Side-by-side
Showing with 593 additions and 64 deletions
RELEASENOTES.md
...@@ -98,7 +98,8 @@ ...@@ -98,7 +98,8 @@
* Use default Deserializers if non given to DownloadManager. * Use default Deserializers if non given to DownloadManager.
* 360: * 360:
* Add monoscopic 360 surface type to PlayerView. * Add monoscopic 360 surface type to PlayerView.
* Support VR180 videos. * Support
[VR180 video format](https://github.com/google/spatial-media/blob/master/docs/vr180.md).
* Deprecate `Player.DefaultEventListener` as selective listener overrides can * Deprecate `Player.DefaultEventListener` as selective listener overrides can
be directly made with the `Player.EventListener` interface. be directly made with the `Player.EventListener` interface.
* Deprecate `DefaultAnalyticsListener` as selective listener overrides can be * Deprecate `DefaultAnalyticsListener` as selective listener overrides can be
......
library/core/src/main/java/com/google/android/exoplayer2/C.java
...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.PlayerMessage.Target; ...@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.PlayerMessage.Target;
import com.google.android.exoplayer2.audio.AuxEffectInfo; import com.google.android.exoplayer2.audio.AuxEffectInfo;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import java.lang.annotation.Retention; import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy; import java.lang.annotation.RetentionPolicy;
import java.util.UUID; import java.util.UUID;
...@@ -595,34 +596,22 @@ public final class C { ...@@ -595,34 +596,22 @@ public final class C {
*/ */
public static final int DATA_TYPE_CUSTOM_BASE = 10000; public static final int DATA_TYPE_CUSTOM_BASE = 10000;
/** /** A type constant for tracks of unknown type. */
* A type constant for tracks of unknown type.
*/
public static final int TRACK_TYPE_UNKNOWN = -1; public static final int TRACK_TYPE_UNKNOWN = -1;
/** /** A type constant for tracks of some default type, where the type itself is unknown. */
* A type constant for tracks of some default type, where the type itself is unknown.
*/
public static final int TRACK_TYPE_DEFAULT = 0; public static final int TRACK_TYPE_DEFAULT = 0;
/** /** A type constant for audio tracks. */
* A type constant for audio tracks.
*/
public static final int TRACK_TYPE_AUDIO = 1; public static final int TRACK_TYPE_AUDIO = 1;
/** /** A type constant for video tracks. */
* A type constant for video tracks.
*/
public static final int TRACK_TYPE_VIDEO = 2; public static final int TRACK_TYPE_VIDEO = 2;
/** /** A type constant for text tracks. */
* A type constant for text tracks.
*/
public static final int TRACK_TYPE_TEXT = 3; public static final int TRACK_TYPE_TEXT = 3;
/** /** A type constant for metadata tracks. */
* A type constant for metadata tracks.
*/
public static final int TRACK_TYPE_METADATA = 4; public static final int TRACK_TYPE_METADATA = 4;
/** /** A type constant for camera motion tracks. */
* A type constant for a dummy or empty track. public static final int TRACK_TYPE_CAMERA_MOTION = 5;
*/ /** A type constant for a dummy or empty track. */
public static final int TRACK_TYPE_NONE = 5; public static final int TRACK_TYPE_NONE = 6;
/** /**
* Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or * Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or
* equal to this value. * equal to this value.
...@@ -655,36 +644,27 @@ public final class C { ...@@ -655,36 +644,27 @@ public final class C {
*/ */
public static final int SELECTION_REASON_CUSTOM_BASE = 10000; public static final int SELECTION_REASON_CUSTOM_BASE = 10000;
/** /** A default size in bytes for an individual allocation that forms part of a larger buffer. */
* A default size in bytes for an individual allocation that forms part of a larger buffer.
*/
public static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024; public static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024;
/** /** A default size in bytes for a video buffer. */
* A default size in bytes for a video buffer.
*/
public static final int DEFAULT_VIDEO_BUFFER_SIZE = 200 * DEFAULT_BUFFER_SEGMENT_SIZE; public static final int DEFAULT_VIDEO_BUFFER_SIZE = 200 * DEFAULT_BUFFER_SEGMENT_SIZE;
/** /** A default size in bytes for an audio buffer. */
* A default size in bytes for an audio buffer.
*/
public static final int DEFAULT_AUDIO_BUFFER_SIZE = 54 * DEFAULT_BUFFER_SEGMENT_SIZE; public static final int DEFAULT_AUDIO_BUFFER_SIZE = 54 * DEFAULT_BUFFER_SEGMENT_SIZE;
/** /** A default size in bytes for a text buffer. */
* A default size in bytes for a text buffer.
*/
public static final int DEFAULT_TEXT_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE; public static final int DEFAULT_TEXT_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
/** /** A default size in bytes for a metadata buffer. */
* A default size in bytes for a metadata buffer.
*/
public static final int DEFAULT_METADATA_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE; public static final int DEFAULT_METADATA_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
/** /** A default size in bytes for a camera motion buffer. */
* A default size in bytes for a muxed buffer (e.g. containing video, audio and text). public static final int DEFAULT_CAMERA_MOTION_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
*/
public static final int DEFAULT_MUXED_BUFFER_SIZE = DEFAULT_VIDEO_BUFFER_SIZE /** A default size in bytes for a muxed buffer (e.g. containing video, audio and text). */
+ DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE; public static final int DEFAULT_MUXED_BUFFER_SIZE =
DEFAULT_VIDEO_BUFFER_SIZE + DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE;
/** "cenc" scheme type name as defined in ISO/IEC 23001-7:2016. */ /** "cenc" scheme type name as defined in ISO/IEC 23001-7:2016. */
@SuppressWarnings("ConstantField") @SuppressWarnings("ConstantField")
...@@ -799,6 +779,13 @@ public final class C { ...@@ -799,6 +779,13 @@ public final class C {
public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6; public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6;
/** /**
* The type of a message that can be passed to a camera motion {@link Renderer} via {@link
* ExoPlayer#createMessage(Target)}. The message payload should be a {@link CameraMotionListener}
* instance, or null.
*/
public static final int MSG_SET_CAMERA_MOTION_LISTENER = 7;
/**
* Applications or extensions may define custom {@code MSG_*} constants that can be passed to * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
* {@link Renderer}s. These custom constants must be greater than or equal to this value. * {@link Renderer}s. These custom constants must be greater than or equal to this value.
*/ */
......
library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java
...@@ -35,6 +35,7 @@ import com.google.android.exoplayer2.text.TextRenderer; ...@@ -35,6 +35,7 @@ import com.google.android.exoplayer2.text.TextRenderer;
import com.google.android.exoplayer2.trackselection.TrackSelector; import com.google.android.exoplayer2.trackselection.TrackSelector;
import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; import com.google.android.exoplayer2.video.MediaCodecVideoRenderer;
import com.google.android.exoplayer2.video.VideoRendererEventListener; import com.google.android.exoplayer2.video.VideoRendererEventListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionRenderer;
import java.lang.annotation.Retention; import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy; import java.lang.annotation.RetentionPolicy;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
...@@ -182,6 +183,7 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -182,6 +183,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
extensionRendererMode, renderersList); extensionRendererMode, renderersList);
buildMetadataRenderers(context, metadataRendererOutput, eventHandler.getLooper(), buildMetadataRenderers(context, metadataRendererOutput, eventHandler.getLooper(),
extensionRendererMode, renderersList); extensionRendererMode, renderersList);
buildCameraMotionRenderers(context, extensionRendererMode, renderersList);
buildMiscellaneousRenderers(context, eventHandler, extensionRendererMode, renderersList); buildMiscellaneousRenderers(context, eventHandler, extensionRendererMode, renderersList);
return renderersList.toArray(new Renderer[renderersList.size()]); return renderersList.toArray(new Renderer[renderersList.size()]);
} }
...@@ -360,12 +362,14 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -360,12 +362,14 @@ public class DefaultRenderersFactory implements RenderersFactory {
* *
* @param context The {@link Context} associated with the player. * @param context The {@link Context} associated with the player.
* @param output An output for the renderers. * @param output An output for the renderers.
* @param outputLooper The looper associated with the thread on which the output should be * @param outputLooper The looper associated with the thread on which the output should be called.
* called.
* @param extensionRendererMode The extension renderer mode. * @param extensionRendererMode The extension renderer mode.
* @param out An array to which the built renderers should be appended. * @param out An array to which the built renderers should be appended.
*/ */
protected void buildTextRenderers(Context context, TextOutput output, Looper outputLooper, protected void buildTextRenderers(
Context context,
TextOutput output,
Looper outputLooper,
@ExtensionRendererMode int extensionRendererMode, @ExtensionRendererMode int extensionRendererMode,
ArrayList<Renderer> out) { ArrayList<Renderer> out) {
out.add(new TextRenderer(output, outputLooper)); out.add(new TextRenderer(output, outputLooper));
...@@ -376,17 +380,32 @@ public class DefaultRenderersFactory implements RenderersFactory { ...@@ -376,17 +380,32 @@ public class DefaultRenderersFactory implements RenderersFactory {
* *
* @param context The {@link Context} associated with the player. * @param context The {@link Context} associated with the player.
* @param output An output for the renderers. * @param output An output for the renderers.
* @param outputLooper The looper associated with the thread on which the output should be * @param outputLooper The looper associated with the thread on which the output should be called.
* called.
* @param extensionRendererMode The extension renderer mode. * @param extensionRendererMode The extension renderer mode.
* @param out An array to which the built renderers should be appended. * @param out An array to which the built renderers should be appended.
*/ */
protected void buildMetadataRenderers(Context context, MetadataOutput output, Looper outputLooper, protected void buildMetadataRenderers(
@ExtensionRendererMode int extensionRendererMode, ArrayList<Renderer> out) { Context context,
MetadataOutput output,
Looper outputLooper,
@ExtensionRendererMode int extensionRendererMode,
ArrayList<Renderer> out) {
out.add(new MetadataRenderer(output, outputLooper)); out.add(new MetadataRenderer(output, outputLooper));
} }
/** /**
* Builds camera motion renderers for use by the player.
*
* @param context The {@link Context} associated with the player.
* @param extensionRendererMode The extension renderer mode.
* @param out An array to which the built renderers should be appended.
*/
protected void buildCameraMotionRenderers(
Context context, @ExtensionRendererMode int extensionRendererMode, ArrayList<Renderer> out) {
out.add(new CameraMotionRenderer());
}
/**
* Builds any miscellaneous renderers used by the player. * Builds any miscellaneous renderers used by the player.
* *
* @param context The {@link Context} associated with the player. * @param context The {@link Context} associated with the player.
......
library/core/src/main/java/com/google/android/exoplayer2/Player.java
...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelectionArray; ...@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoListener; import com.google.android.exoplayer2.video.VideoListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import java.lang.annotation.Retention; import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy; import java.lang.annotation.RetentionPolicy;
...@@ -186,6 +187,21 @@ public interface Player { ...@@ -186,6 +187,21 @@ public interface Player {
void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener); void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener);
/** /**
* Sets a listener of camera motion events.
*
* @param listener The listener.
*/
void setCameraMotionListener(CameraMotionListener listener);
/**
* Clears the listener which receives camera motion events if it matches the one passed. Else
* does nothing.
*
* @param listener The listener to clear.
*/
void clearCameraMotionListener(CameraMotionListener listener);
/**
* Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView} * Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView}
* currently set on the player. * currently set on the player.
*/ */
......
library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
...@@ -53,6 +53,7 @@ import com.google.android.exoplayer2.util.Clock; ...@@ -53,6 +53,7 @@ import com.google.android.exoplayer2.util.Clock;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.VideoRendererEventListener; import com.google.android.exoplayer2.video.VideoRendererEventListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
...@@ -107,6 +108,7 @@ public class SimpleExoPlayer ...@@ -107,6 +108,7 @@ public class SimpleExoPlayer
private MediaSource mediaSource; private MediaSource mediaSource;
private List<Cue> currentCues; private List<Cue> currentCues;
private VideoFrameMetadataListener videoFrameMetadataListener; private VideoFrameMetadataListener videoFrameMetadataListener;
private CameraMotionListener cameraMotionListener;
/** /**
* @param context A {@link Context}. * @param context A {@link Context}.
...@@ -597,6 +599,36 @@ public class SimpleExoPlayer ...@@ -597,6 +599,36 @@ public class SimpleExoPlayer
} }
} }
@Override
public void setCameraMotionListener(CameraMotionListener listener) {
cameraMotionListener = listener;
for (Renderer renderer : renderers) {
if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) {
player
.createMessage(renderer)
.setType(C.MSG_SET_CAMERA_MOTION_LISTENER)
.setPayload(listener)
.send();
}
}
}
@Override
public void clearCameraMotionListener(CameraMotionListener listener) {
if (cameraMotionListener != listener) {
return;
}
for (Renderer renderer : renderers) {
if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) {
player
.createMessage(renderer)
.setType(C.MSG_SET_CAMERA_MOTION_LISTENER)
.setPayload(null)
.send();
}
}
}
/** /**
* Sets a listener to receive video events, removing all existing listeners. * Sets a listener to receive video events, removing all existing listeners.
* *
......
library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java
...@@ -599,6 +599,8 @@ public class EventLogger implements AnalyticsListener { ...@@ -599,6 +599,8 @@ public class EventLogger implements AnalyticsListener {
return "default"; return "default";
case C.TRACK_TYPE_METADATA: case C.TRACK_TYPE_METADATA:
return "metadata"; return "metadata";
case C.TRACK_TYPE_CAMERA_MOTION:
return "camera motion";
case C.TRACK_TYPE_NONE: case C.TRACK_TYPE_NONE:
return "none"; return "none";
case C.TRACK_TYPE_TEXT: case C.TRACK_TYPE_TEXT:
......
library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java
...@@ -328,9 +328,10 @@ public final class MimeTypes { ...@@ -328,9 +328,10 @@ public final class MimeTypes {
return C.TRACK_TYPE_TEXT; return C.TRACK_TYPE_TEXT;
} else if (APPLICATION_ID3.equals(mimeType) } else if (APPLICATION_ID3.equals(mimeType)
|| APPLICATION_EMSG.equals(mimeType) || APPLICATION_EMSG.equals(mimeType)
|| APPLICATION_SCTE35.equals(mimeType) || APPLICATION_SCTE35.equals(mimeType)) {
|| APPLICATION_CAMERA_MOTION.equals(mimeType)) {
return C.TRACK_TYPE_METADATA; return C.TRACK_TYPE_METADATA;
} else if (APPLICATION_CAMERA_MOTION.equals(mimeType)) {
return C.TRACK_TYPE_CAMERA_MOTION;
} else { } else {
return getTrackTypeForCustomMimeType(mimeType); return getTrackTypeForCustomMimeType(mimeType);
} }
......
library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java
...@@ -64,7 +64,7 @@ public final class TimedValueQueue<V> { ...@@ -64,7 +64,7 @@ public final class TimedValueQueue<V> {
/** /**
* Returns the value with the greatest timestamp which is less than or equal to the given * Returns the value with the greatest timestamp which is less than or equal to the given
* timestamp. Removes all older values including the returned one from the buffer. * timestamp. Removes all older values and the returned one from the buffer.
* *
* @param timestamp The timestamp value. * @param timestamp The timestamp value.
* @return The value with the greatest timestamp which is less than or equal to the given * @return The value with the greatest timestamp which is less than or equal to the given
......
library/core/src/main/java/com/google/android/exoplayer2/util/Util.java
...@@ -1443,6 +1443,8 @@ public final class Util { ...@@ -1443,6 +1443,8 @@ public final class Util {
return C.DEFAULT_TEXT_BUFFER_SIZE; return C.DEFAULT_TEXT_BUFFER_SIZE;
case C.TRACK_TYPE_METADATA: case C.TRACK_TYPE_METADATA:
return C.DEFAULT_METADATA_BUFFER_SIZE; return C.DEFAULT_METADATA_BUFFER_SIZE;
case C.TRACK_TYPE_CAMERA_MOTION:
return C.DEFAULT_CAMERA_MOTION_BUFFER_SIZE;
default: default:
throw new IllegalStateException(); throw new IllegalStateException();
} }
......
library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java 0 → 100644
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.video.spherical;
/** Listens camera motion. */
public interface CameraMotionListener {
/**
* Called when a new camera motion is read. This method is called on the playback thread.
*
* @param timeUs The presentation time of the data.
* @param rotation Angle axis orientation in radians representing the rotation from camera
* coordinate system to world coordinate system.
*/
void onCameraMotion(long timeUs, float[] rotation);
/** Called when the camera motion track position is reset. */
void onCameraMotionReset();
}
library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java 0 → 100644
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.video.spherical;
import android.support.annotation.Nullable;
import com.google.android.exoplayer2.BaseRenderer;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.Renderer;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.Util;
import java.nio.ByteBuffer;
/** A {@link Renderer} that parses the camera motion track. */
public class CameraMotionRenderer extends BaseRenderer {
// The amount of time to read samples ahead of the current time.
private static final int SAMPLE_WINDOW_DURATION_US = 100000;
private final FormatHolder formatHolder;
private final DecoderInputBuffer buffer;
private final ParsableByteArray scratch;
private long offsetUs;
private @Nullable CameraMotionListener listener;
private long lastTimestampUs;
public CameraMotionRenderer() {
super(C.TRACK_TYPE_CAMERA_MOTION);
formatHolder = new FormatHolder();
buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL);
scratch = new ParsableByteArray();
}
@Override
public int supportsFormat(Format format) {
return MimeTypes.APPLICATION_CAMERA_MOTION.equals(format.sampleMimeType)
? FORMAT_HANDLED
: FORMAT_UNSUPPORTED_TYPE;
}
@Override
public void handleMessage(int messageType, Object message) throws ExoPlaybackException {
if (messageType == C.MSG_SET_CAMERA_MOTION_LISTENER) {
listener = (CameraMotionListener) message;
} else {
super.handleMessage(messageType, message);
}
}
@Override
protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
this.offsetUs = offsetUs;
}
@Override
protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
lastTimestampUs = 0;
if (listener != null) {
listener.onCameraMotionReset();
}
}
@Override
protected void onDisabled() {
lastTimestampUs = 0;
}
@Override
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
// Keep reading available samples as long as the sample time is not too far into the future.
while (!hasReadStreamToEnd() && lastTimestampUs < positionUs + SAMPLE_WINDOW_DURATION_US) {
buffer.clear();
int result = readSource(formatHolder, buffer, /* formatRequired= */ false);
if (result != C.RESULT_BUFFER_READ || buffer.isEndOfStream()) {
return;
}
buffer.flip();
lastTimestampUs = buffer.timeUs;
if (listener != null) {
float[] rotation = parseMetadata(buffer.data);
if (rotation != null) {
Util.castNonNull(listener).onCameraMotion(lastTimestampUs - offsetUs, rotation);
}
}
}
}
@Override
public boolean isEnded() {
return hasReadStreamToEnd();
}
@Override
public boolean isReady() {
return true;
}
private @Nullable float[] parseMetadata(ByteBuffer data) {
if (data.remaining() != 16) {
return null;
}
scratch.reset(data.array(), data.limit());
scratch.setPosition(data.arrayOffset() + 4); // skip reserved bytes too.
float[] result = new float[3];
for (int i = 0; i < 3; i++) {
result[i] = Float.intBitsToFloat(scratch.readLittleEndianInt());
}
return result;
}
}
library/core/src/main/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java 0 → 100644
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.video.spherical;
import android.opengl.Matrix;
import com.google.android.exoplayer2.util.TimedValueQueue;
/**
* This class serves multiple purposes:
*
* <ul>
* <li>Queues the rotation metadata extracted from camera motion track.
* <li>Converts the metadata to rotation matrices in OpenGl coordinate system.
* <li>Recenters the rotations to componsate the yaw of the initial rotation.
* </ul>
*/
public final class FrameRotationQueue {
private final float[] recenterMatrix;
private final float[] rotationMatrix;
private final TimedValueQueue<float[]> rotations;
private boolean recenterMatrixComputed;
public FrameRotationQueue() {
recenterMatrix = new float[16];
rotationMatrix = new float[16];
rotations = new TimedValueQueue<>();
}
/**
* Sets a rotation for a given timestamp.
*
* @param timestampUs Timestamp of the rotation.
* @param angleAxis Angle axis orientation in radians representing the rotation from camera
* coordinate system to world coordinate system.
*/
public void setRotation(long timestampUs, float[] angleAxis) {
rotations.add(timestampUs, angleAxis);
}
/** Removes all of the rotations and forces rotations to be recentered. */
public void reset() {
rotations.clear();
recenterMatrixComputed = false;
}
/**
* Copies the rotation matrix with the greatest timestamp which is less than or equal to the given
* timestamp to {@code matrix}. Removes all older rotations and the returned one from the queue.
* Does nothing if there is no such rotation.
*
* @param matrix A float array to hold the rotation matrix.
* @param timestampUs The time in microseconds to query the rotation.
* @return Whether a rotation matrix is copied to {@code matrix}.
*/
public boolean pollRotationMatrix(float[] matrix, long timestampUs) {
float[] rotation = rotations.pollFloor(timestampUs);
if (rotation == null) {
return false;
}
// TODO [Internal: b/113315546]: Slerp between the floor and ceil rotation.
getRotationMatrixFromAngleAxis(rotationMatrix, rotation);
if (!recenterMatrixComputed) {
computeRecenterMatrix(recenterMatrix, rotationMatrix);
recenterMatrixComputed = true;
}
Matrix.multiplyMM(matrix, 0, recenterMatrix, 0, rotationMatrix, 0);
return true;
}
/**
* Computes a recentering matrix from the given angle-axis rotation only accounting for yaw. Roll
* and tilt will not be compensated.
*/
private static void computeRecenterMatrix(float[] recenterMatrix, float[] rotationMatrix) {
// The re-centering matrix is computed as follows:
// recenter.row(2) = temp.col(2).transpose();
// recenter.row(0) = recenter.row(1).cross(recenter.row(2)).normalized();
// recenter.row(2) = recenter.row(0).cross(recenter.row(1)).normalized();
// | temp[10] 0 -temp[8] 0|
// | 0 1 0 0|
// recenter = | temp[8] 0 temp[10] 0|
// | 0 0 0 1|
Matrix.setIdentityM(recenterMatrix, 0);
float normRowSqr =
rotationMatrix[10] * rotationMatrix[10] + rotationMatrix[8] * rotationMatrix[8];
float normRow = (float) Math.sqrt(normRowSqr);
recenterMatrix[0] = rotationMatrix[10] / normRow;
recenterMatrix[2] = rotationMatrix[8] / normRow;
recenterMatrix[8] = -rotationMatrix[8] / normRow;
recenterMatrix[10] = rotationMatrix[10] / normRow;
}
private static void getRotationMatrixFromAngleAxis(float[] matrix, float[] angleAxis) {
// Convert coordinates to OpenGL coordinates.
// CAMM motion metadata: +x right, +y down, and +z forward.
// OpenGL: +x right, +y up, -z forwards
float x = angleAxis[0];
float y = -angleAxis[1];
float z = -angleAxis[2];
float angleRad = Matrix.length(x, y, z);
if (angleRad != 0) {
float angleDeg = (float) Math.toDegrees(angleRad);
Matrix.setRotateM(matrix, 0, angleDeg, x / angleRad, y / angleRad, z / angleRad);
} else {
Matrix.setIdentityM(matrix, 0);
}
}
}
library/core/src/test/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueueTest.java 0 → 100644
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.video.spherical;
import static com.google.common.truth.Truth.assertThat;
import android.opengl.Matrix;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.RobolectricTestRunner;
/** Tests {@link FrameRotationQueue}. */
@RunWith(RobolectricTestRunner.class)
public class FrameRotationQueueTest {
private FrameRotationQueue frameRotationQueue;
private float[] rotationMatrix;
@Before
public void setUp() throws Exception {
frameRotationQueue = new FrameRotationQueue();
rotationMatrix = new float[16];
}
@Test
public void testGetRotationMatrixReturnsNull_whenEmpty() throws Exception {
assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isFalse();
}
@Test
public void testGetRotationMatrixReturnsNotNull_whenNotEmpty() throws Exception {
frameRotationQueue.setRotation(0, new float[] {1, 2, 3});
assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isTrue();
assertThat(rotationMatrix).hasLength(16);
}
@Test
public void testConvertsAngleAxisToRotationMatrix() throws Exception {
doTestAngleAxisToRotationMatrix(/* angleRadian= */ 0, /* x= */ 1, /* y= */ 0, /* z= */ 0);
frameRotationQueue.reset();
doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 0, /* z= */ 0);
frameRotationQueue.reset();
doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 0, /* y= */ 0, /* z= */ 1);
// Don't reset frameRotationQueue as we use recenter matrix from previous calls.
doTestAngleAxisToRotationMatrix(/* angleRadian= */ -1, /* x= */ 0, /* y= */ 1, /* z= */ 0);
doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 1, /* z= */ 1);
}
@Test
public void testRecentering_justYaw() throws Exception {
float[] actualMatrix =
getRotationMatrixFromAngleAxis(
/* angleRadian= */ (float) Math.PI, /* x= */ 0, /* y= */ 1, /* z= */ 0);
float[] expectedMatrix = new float[16];
Matrix.setIdentityM(expectedMatrix, 0);
assertEquals(actualMatrix, expectedMatrix);
}
@Test
public void testRecentering_yawAndPitch() throws Exception {
float[] matrix =
getRotationMatrixFromAngleAxis(
/* angleRadian= */ (float) Math.PI, /* x= */ 1, /* y= */ 1, /* z= */ 0);
assertMultiplication(
/* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1);
}
@Test
public void testRecentering_yawAndPitch2() throws Exception {
float[] matrix =
getRotationMatrixFromAngleAxis(
/* angleRadian= */ (float) Math.PI / 2, /* x= */ 1, /* y= */ 1, /* z= */ 0);
float sqrt2 = (float) Math.sqrt(2);
assertMultiplication(
/* xr= */ sqrt2, /* yr= */ 0, /* zr= */ 0, matrix, /* x= */ 1, /* y= */ -1, /* z= */ 0);
}
@Test
public void testRecentering_yawAndPitchAndRoll() throws Exception {
float[] matrix =
getRotationMatrixFromAngleAxis(
/* angleRadian= */ (float) Math.PI * 2 / 3, /* x= */ 1, /* y= */ 1, /* z= */ 1);
assertMultiplication(
/* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1);
}
private void doTestAngleAxisToRotationMatrix(float angleRadian, int x, int y, int z) {
float[] actualMatrix = getRotationMatrixFromAngleAxis(angleRadian, x, y, z);
float[] expectedMatrix = createRotationMatrix(angleRadian, x, y, z);
assertEquals(actualMatrix, expectedMatrix);
}
private float[] getRotationMatrixFromAngleAxis(float angleRadian, int x, int y, int z) {
float length = Matrix.length(x, y, z);
float factor = angleRadian / length;
// Negate y and z to revert OpenGL coordinate system conversion.
frameRotationQueue.setRotation(0, new float[] {x * factor, -y * factor, -z * factor});
frameRotationQueue.pollRotationMatrix(rotationMatrix, 0);
return rotationMatrix;
}
private static void assertMultiplication(
float xr, float yr, float zr, float[] actualMatrix, float x, float y, float z) {
float[] vector = new float[] {x, y, z, 0};
float[] resultVec = new float[4];
Matrix.multiplyMV(resultVec, 0, actualMatrix, 0, vector, 0);
assertEquals(resultVec, new float[] {xr, yr, zr, 0});
}
private static float[] createRotationMatrix(float angleRadian, int x, int y, int z) {
float[] expectedMatrix = new float[16];
Matrix.setRotateM(expectedMatrix, 0, (float) Math.toDegrees(angleRadian), x, y, z);
return expectedMatrix;
}
private static void assertEquals(float[] actual, float[] expected) {
assertThat(actual).usingTolerance(1.0e-5).containsExactly(expected).inOrder();
}
}
library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java
...@@ -515,6 +515,7 @@ public class PlayerView extends FrameLayout { ...@@ -515,6 +515,7 @@ public class PlayerView extends FrameLayout {
} else if (surfaceView instanceof SphericalSurfaceView) { } else if (surfaceView instanceof SphericalSurfaceView) {
oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView));
} else if (surfaceView instanceof SurfaceView) { } else if (surfaceView instanceof SurfaceView) {
oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView); oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
} }
...@@ -540,8 +541,9 @@ public class PlayerView extends FrameLayout { ...@@ -540,8 +541,9 @@ public class PlayerView extends FrameLayout {
if (surfaceView instanceof TextureView) { if (surfaceView instanceof TextureView) {
newVideoComponent.setVideoTextureView((TextureView) surfaceView); newVideoComponent.setVideoTextureView((TextureView) surfaceView);
} else if (surfaceView instanceof SphericalSurfaceView) { } else if (surfaceView instanceof SphericalSurfaceView) {
newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView));
newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
} else if (surfaceView instanceof SurfaceView) { } else if (surfaceView instanceof SurfaceView) {
newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView); newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
} }
......
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java
...@@ -19,9 +19,12 @@ import static com.google.android.exoplayer2.ui.spherical.GlUtil.checkGlError; ...@@ -19,9 +19,12 @@ import static com.google.android.exoplayer2.ui.spherical.GlUtil.checkGlError;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.Matrix;
import android.support.annotation.Nullable; import android.support.annotation.Nullable;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection; import com.google.android.exoplayer2.video.spherical.Projection;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
...@@ -35,17 +38,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -35,17 +38,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final AtomicBoolean frameAvailable; private final AtomicBoolean frameAvailable;
private final ProjectionRenderer projectionRenderer; private final ProjectionRenderer projectionRenderer;
private final FrameRotationQueue frameRotationQueue;
private final TimedValueQueue<Long> sampleTimestampQueue;
private final float[] rotationMatrix;
private final float[] tempMatrix;
private int textureId; private int textureId;
private @MonotonicNonNull SurfaceTexture surfaceTexture; private @MonotonicNonNull SurfaceTexture surfaceTexture;
private @Nullable Projection pendingProjection; private @Nullable Projection pendingProjection;
private long pendingProjectionTimeNs; private long pendingProjectionTimeNs;
private long lastFrameTimestamp; private long lastFrameTimestamp;
private boolean resetRotationAtNextFrame;
public SceneRenderer(Projection projection) { public SceneRenderer(
Projection projection,
FrameRotationQueue frameRotationQueue,
TimedValueQueue<Long> sampleTimestampQueue) {
this.frameRotationQueue = frameRotationQueue;
this.sampleTimestampQueue = sampleTimestampQueue;
frameAvailable = new AtomicBoolean(); frameAvailable = new AtomicBoolean();
projectionRenderer = new ProjectionRenderer(); projectionRenderer = new ProjectionRenderer();
projectionRenderer.setProjection(projection); projectionRenderer.setProjection(projection);
rotationMatrix = new float[16];
tempMatrix = new float[16];
resetRotation();
} }
/** Initializes the renderer. */ /** Initializes the renderer. */
...@@ -63,6 +79,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -63,6 +79,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return surfaceTexture; return surfaceTexture;
} }
public void resetRotation() {
resetRotationAtNextFrame = true;
}
/** Sets a {@link Projection} to be used to display video. */ /** Sets a {@link Projection} to be used to display video. */
public void setProjection(Projection projection, long timeNs) { public void setProjection(Projection projection, long timeNs) {
pendingProjection = projection; pendingProjection = projection;
...@@ -84,13 +104,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ...@@ -84,13 +104,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (frameAvailable.compareAndSet(true, false)) { if (frameAvailable.compareAndSet(true, false)) {
Assertions.checkNotNull(surfaceTexture).updateTexImage(); Assertions.checkNotNull(surfaceTexture).updateTexImage();
checkGlError(); checkGlError();
if (resetRotationAtNextFrame) {
Matrix.setIdentityM(rotationMatrix, 0);
}
lastFrameTimestamp = surfaceTexture.getTimestamp(); lastFrameTimestamp = surfaceTexture.getTimestamp();
Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp);
if (sampleTimestamp != null) {
frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp);
}
} }
if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) { if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) {
projectionRenderer.setProjection(pendingProjection); projectionRenderer.setProjection(pendingProjection);
pendingProjection = null; pendingProjection = null;
} }
Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
projectionRenderer.draw(textureId, viewProjectionMatrix, eyeType); projectionRenderer.draw(textureId, tempMatrix, eyeType);
} }
} }
library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java
...@@ -40,8 +40,11 @@ import com.google.android.exoplayer2.C; ...@@ -40,8 +40,11 @@ import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.TimedValueQueue;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
import com.google.android.exoplayer2.video.spherical.Projection; import com.google.android.exoplayer2.video.spherical.Projection;
import com.google.android.exoplayer2.video.spherical.ProjectionDecoder; import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
import java.util.Arrays; import java.util.Arrays;
...@@ -60,7 +63,7 @@ import javax.microedition.khronos.opengles.GL10; ...@@ -60,7 +63,7 @@ import javax.microedition.khronos.opengles.GL10;
*/ */
@TargetApi(15) @TargetApi(15)
public final class SphericalSurfaceView extends GLSurfaceView public final class SphericalSurfaceView extends GLSurfaceView
implements VideoFrameMetadataListener { implements VideoFrameMetadataListener, CameraMotionListener {
/** /**
* This listener can be used to be notified when the {@link Surface} associated with this view is * This listener can be used to be notified when the {@link Surface} associated with this view is
...@@ -91,6 +94,8 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -91,6 +94,8 @@ public final class SphericalSurfaceView extends GLSurfaceView
private final PhoneOrientationListener phoneOrientationListener; private final PhoneOrientationListener phoneOrientationListener;
private final Renderer renderer; private final Renderer renderer;
private final Handler mainHandler; private final Handler mainHandler;
private final TimedValueQueue<Long> sampleTimestampQueue;
private final FrameRotationQueue frameRotationQueue;
private final TouchTracker touchTracker; private final TouchTracker touchTracker;
private @Nullable SurfaceListener surfaceListener; private @Nullable SurfaceListener surfaceListener;
private @Nullable SurfaceTexture surfaceTexture; private @Nullable SurfaceTexture surfaceTexture;
...@@ -105,9 +110,6 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -105,9 +110,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
public SphericalSurfaceView(Context context, @Nullable AttributeSet attributeSet) { public SphericalSurfaceView(Context context, @Nullable AttributeSet attributeSet) {
super(context, attributeSet); super(context, attributeSet);
defaultStereoMode = C.STEREO_MODE_MONO;
currentStereoMode = C.STEREO_MODE_MONO;
mainHandler = new Handler(Looper.getMainLooper()); mainHandler = new Handler(Looper.getMainLooper());
// Configure sensors and touch. // Configure sensors and touch.
...@@ -120,7 +122,13 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -120,7 +122,13 @@ public final class SphericalSurfaceView extends GLSurfaceView
int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR; int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR;
orientationSensor = sensorManager.getDefaultSensor(type); orientationSensor = sensorManager.getDefaultSensor(type);
renderer = new Renderer(); defaultStereoMode = C.STEREO_MODE_MONO;
currentStereoMode = defaultStereoMode;
Projection projection = Projection.createEquirectangular(defaultStereoMode);
frameRotationQueue = new FrameRotationQueue();
sampleTimestampQueue = new TimedValueQueue<>();
SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue);
renderer = new Renderer(scene);
touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES); touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES);
WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
...@@ -161,12 +169,29 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -161,12 +169,29 @@ public final class SphericalSurfaceView extends GLSurfaceView
touchTracker.setSingleTapListener(listener); touchTracker.setSingleTapListener(listener);
} }
// VideoFrameMetadataListener implementation.
@Override @Override
public void onVideoFrameAboutToBeRendered( public void onVideoFrameAboutToBeRendered(
long presentationTimeUs, long releaseTimeNs, Format format) { long presentationTimeUs, long releaseTimeNs, Format format) {
sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
setProjection(format.projectionData, format.stereoMode, releaseTimeNs); setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
} }
// CameraMotionListener implementation.
@Override
public void onCameraMotion(long timeUs, float[] rotation) {
frameRotationQueue.setRotation(timeUs, rotation);
}
@Override
public void onCameraMotionReset() {
sampleTimestampQueue.clear();
frameRotationQueue.reset();
queueEvent(renderer.scene::resetRotation);
}
@Override @Override
public void onResume() { public void onResume() {
super.onResume(); super.onResume();
...@@ -354,8 +379,8 @@ public final class SphericalSurfaceView extends GLSurfaceView ...@@ -354,8 +379,8 @@ public final class SphericalSurfaceView extends GLSurfaceView
private final float[] viewMatrix = new float[16]; private final float[] viewMatrix = new float[16];
private final float[] tempMatrix = new float[16]; private final float[] tempMatrix = new float[16];
public Renderer() { public Renderer(SceneRenderer scene) {
scene = new SceneRenderer(Projection.createEquirectangular(C.STEREO_MODE_MONO)); this.scene = scene;
Matrix.setIdentityM(deviceOrientationMatrix, 0); Matrix.setIdentityM(deviceOrientationMatrix, 0);
Matrix.setIdentityM(touchPitchMatrix, 0); Matrix.setIdentityM(touchPitchMatrix, 0);
Matrix.setIdentityM(touchYawMatrix, 0); Matrix.setIdentityM(touchYawMatrix, 0);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment