Skip to content

SDK / exoplayer

Go to a project
Commit bd7be1b5 by eguven Committed by Oliver Woodman
Options

Cache support unbounded requests. 

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=131696858
parent dfad7451
Inline Side-by-side
Showing with 576 additions and 82 deletions
library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/CacheDataSourceTest.java 0 → 100644
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.upstream.cache;
import android.net.Uri;
import android.test.InstrumentationTestCase;
import android.test.MoreAsserts;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.FakeDataSource;
import com.google.android.exoplayer2.testutil.FakeDataSource.Builder;
import com.google.android.exoplayer2.testutil.TestUtil;
import com.google.android.exoplayer2.upstream.DataSpec;
import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
/** Unit tests for {@link CacheDataSource}. */
public class CacheDataSourceTest extends InstrumentationTestCase {
private static final byte[] TEST_DATA = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
private static final int MAX_CACHE_FILE_SIZE = 3;
private static final String KEY_1 = "key 1";
private static final String KEY_2 = "key 2";
private File cacheDir;
private SimpleCache simpleCache;
@Override
protected void setUp() throws Exception {
// Create a temporary folder
cacheDir = File.createTempFile("CacheDataSourceTest", null);
assertTrue(cacheDir.delete());
assertTrue(cacheDir.mkdir());
simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
}
@Override
protected void tearDown() throws Exception {
TestUtil.recursiveDelete(cacheDir);
}
public void testMaxCacheFileSize() throws Exception {
CacheDataSource cacheDataSource = createCacheDataSource(false, false, false);
assertReadDataContentLength(cacheDataSource, false, false);
assertEquals((int) Math.ceil((double) TEST_DATA.length / MAX_CACHE_FILE_SIZE),
cacheDir.listFiles().length);
}
public void testCacheAndRead() throws Exception {
assertCacheAndRead(false, false);
}
public void testCacheAndReadUnboundedRequest() throws Exception {
assertCacheAndRead(true, false);
}
public void testCacheAndReadUnknownLength() throws Exception {
assertCacheAndRead(false, true);
}
// Disabled test as we don't support caching of definitely unknown length content
public void disabledTestCacheAndReadUnboundedRequestUnknownLength() throws Exception {
assertCacheAndRead(true, true);
}
public void testUnsatisfiableRange() throws Exception {
// Bounded request but the content length is unknown. This forces all data to be cached but not
// the length
assertCacheAndRead(false, true);
// Now do an unbounded request. This will read all of the data from cache and then try to read
// more from upstream which will cause to a 416 so CDS will store the length.
CacheDataSource cacheDataSource = createCacheDataSource(true, true, true);
assertReadDataContentLength(cacheDataSource, true, true);
// If the user try to access off range then it should throw an IOException
try {
cacheDataSource = createCacheDataSource(false, false, false);
cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length, 5, KEY_1));
fail();
} catch (TestIOException e) {
// success
}
}
public void testContentLengthEdgeCases() throws Exception {
// Read partial at EOS but don't cross it so length is unknown
CacheDataSource cacheDataSource = createCacheDataSource(false, false, true);
assertReadData(cacheDataSource, true, TEST_DATA.length - 2, 2);
assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));
// Now do an unbounded request for whole data. This will cause a bounded request from upstream.
// End of data from upstream shouldn't be mixed up with EOS and cause length set wrong.
cacheDataSource = createCacheDataSource(true, false, true);
assertReadDataContentLength(cacheDataSource, true, true);
// Now the length set correctly do an unbounded request with offset
assertEquals(2, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
C.LENGTH_UNSET, KEY_1)));
// An unbounded request with offset for not cached content
assertEquals(C.LENGTH_UNSET, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
C.LENGTH_UNSET, KEY_2)));
}
private void assertCacheAndRead(boolean unboundedRequest, boolean simulateUnknownLength)
throws IOException {
// Read all data from upstream and cache
CacheDataSource cacheDataSource = createCacheDataSource(false, false, simulateUnknownLength);
assertReadDataContentLength(cacheDataSource, unboundedRequest, simulateUnknownLength);
// Just read from cache
cacheDataSource = createCacheDataSource(false, true, simulateUnknownLength);
assertReadDataContentLength(cacheDataSource, unboundedRequest,
false /*length is already cached*/);
}
/**
* Reads data until EOI and compares it to {@link #TEST_DATA}. Also checks content length returned
* from open() call and the cached content length.
*/
private void assertReadDataContentLength(CacheDataSource cacheDataSource,
boolean unboundedRequest, boolean unknownLength) throws IOException {
int length = unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length;
assertReadData(cacheDataSource, unknownLength, 0, length);
assertEquals("When the range specified, CacheDataSource doesn't reach EOS so shouldn't cache "
+ "content length", !unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length,
simpleCache.getContentLength(KEY_1));
}
private void assertReadData(CacheDataSource cacheDataSource, boolean unknownLength, int position,
int length) throws IOException {
int actualLength = TEST_DATA.length - position;
if (length != C.LENGTH_UNSET) {
actualLength = Math.min(actualLength, length);
}
assertEquals(unknownLength ? length : actualLength,
cacheDataSource.open(new DataSpec(Uri.EMPTY, position, length, KEY_1)));
byte[] buffer = new byte[100];
int index = 0;
while (true) {
int read = cacheDataSource.read(buffer, index, buffer.length - index);
if (read == C.RESULT_END_OF_INPUT) {
break;
}
index += read;
}
assertEquals(actualLength, index);
MoreAsserts.assertEquals(Arrays.copyOfRange(TEST_DATA, position, position + actualLength),
Arrays.copyOf(buffer, index));
cacheDataSource.close();
}
private CacheDataSource createCacheDataSource(boolean set416exception, boolean setReadException,
boolean simulateUnknownLength) {
Builder builder = new Builder();
if (setReadException) {
builder.appendReadError(new IOException("Shouldn't read from upstream"));
}
builder.setSimulateUnknownLength(simulateUnknownLength);
builder.appendReadData(TEST_DATA);
FakeDataSource upstream = builder.build();
upstream.setUnsatisfiableRangeException(set416exception
? new InvalidResponseCodeException(416, null, null)
: new TestIOException());
return new CacheDataSource(simpleCache, upstream,
CacheDataSource.FLAG_BLOCK_ON_CACHE | CacheDataSource.FLAG_CACHE_UNBOUNDED_REQUESTS,
MAX_CACHE_FILE_SIZE);
}
private static class TestIOException extends IOException {}
}
library/src/androidTest/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheTest.java 0 → 100644
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.upstream.cache;
import android.test.InstrumentationTestCase;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.TestUtil;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.Set;
/**
* Unit tests for {@link SimpleCache}.
*/
public class SimpleCacheTest extends InstrumentationTestCase {
private static final String KEY_1 = "key1";
private File cacheDir;
@Override
protected void setUp() throws Exception {
// Create a temporary folder
cacheDir = File.createTempFile("SimpleCacheTest", null);
assertTrue(cacheDir.delete());
assertTrue(cacheDir.mkdir());
}
@Override
protected void tearDown() throws Exception {
TestUtil.recursiveDelete(cacheDir);
}
public void testCommittingOneFile() throws Exception {
SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
CacheSpan cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
assertFalse(cacheSpan.isCached);
assertTrue(cacheSpan.isOpenEnded());
assertNull(simpleCache.startReadWriteNonBlocking(KEY_1, 0));
assertEquals(0, simpleCache.getKeys().size());
NavigableSet<CacheSpan> cachedSpans = simpleCache.getCachedSpans(KEY_1);
assertTrue(cachedSpans == null || cachedSpans.size() == 0);
assertEquals(0, simpleCache.getCacheSpace());
assertEquals(0, cacheDir.listFiles().length);
addCache(simpleCache, 0, 15);
Set<String> cachedKeys = simpleCache.getKeys();
assertEquals(1, cachedKeys.size());
assertTrue(cachedKeys.contains(KEY_1));
cachedSpans = simpleCache.getCachedSpans(KEY_1);
assertEquals(1, cachedSpans.size());
assertTrue(cachedSpans.contains(cacheSpan));
assertEquals(15, simpleCache.getCacheSpace());
cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
assertTrue(cacheSpan.isCached);
assertFalse(cacheSpan.isOpenEnded());
assertEquals(15, cacheSpan.length);
}
public void testSetGetLength() throws Exception {
SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));
assertTrue(simpleCache.setContentLength(KEY_1, 15));
assertEquals(15, simpleCache.getContentLength(KEY_1));
simpleCache.startReadWrite(KEY_1, 0);
addCache(simpleCache, 0, 15);
assertTrue(simpleCache.setContentLength(KEY_1, 150));
assertEquals(150, simpleCache.getContentLength(KEY_1));
addCache(simpleCache, 140, 10);
// Try to set length shorter then the content
assertFalse(simpleCache.setContentLength(KEY_1, 15));
assertEquals("Content length should be unchanged.",
150, simpleCache.getContentLength(KEY_1));
/* TODO Enable when the length persistance is fixed
// Check if values are kept after cache is reloaded.
simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
assertEquals(150, simpleCache.getContentLength(KEY_1));
CacheSpan lastSpan = simpleCache.startReadWrite(KEY_1, 145);
// Removing the last span shouldn't cause the length be change next time cache loaded
simpleCache.removeSpan(lastSpan);
simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
assertEquals(150, simpleCache.getContentLength(KEY_1));
*/
}
private void addCache(SimpleCache simpleCache, int position, int length) throws IOException {
File file = simpleCache.startFile(KEY_1, position, length);
FileOutputStream fos = new FileOutputStream(file);
fos.write(new byte[length]);
fos.close();
simpleCache.commitFile(file);
}
}
library/src/main/java/com/google/android/exoplayer2/upstream/cache/Cache.java
...@@ -143,11 +143,11 @@ public interface Cache { ...@@ -143,11 +143,11 @@ public interface Cache {
* *
* @param key The cache key for the data. * @param key The cache key for the data.
* @param position The starting position of the data. * @param position The starting position of the data.
* @param length The length of the data to be written. Used only to ensure that there is enough * @param maxLength The maximum length of the data to be written. Used only to ensure that there
* space in the cache. * is enough space in the cache.
* @return The file into which data should be written. * @return The file into which data should be written.
*/ */
File startFile(String key, long position, long length); File startFile(String key, long position, long maxLength);
/** /**
* Commits a file into the cache. Must only be called when holding a corresponding hole * Commits a file into the cache. Must only be called when holding a corresponding hole
...@@ -182,4 +182,22 @@ public interface Cache { ...@@ -182,4 +182,22 @@ public interface Cache {
*/ */
boolean isCached(String key, long position, long length); boolean isCached(String key, long position, long length);
/**
* Sets the content length for the given key.
*
* @param key The cache key for the data.
* @param length The length of the data.
* @return Whether the length was set successfully. Returns false if the length conflicts with the
* existing contents of the cache.
*/
boolean setContentLength(String key, long length);
/**
* Returns the content length for the given key if one set, or {@link
* com.google.android.exoplayer2.C#LENGTH_UNSET} otherwise.
*
* @param key The cache key for the data.
*/
long getContentLength(String key);
} }
library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSink.java
...@@ -64,12 +64,12 @@ public final class CacheDataSink implements DataSink { ...@@ -64,12 +64,12 @@ public final class CacheDataSink implements DataSink {
@Override @Override
public void open(DataSpec dataSpec) throws CacheDataSinkException { public void open(DataSpec dataSpec) throws CacheDataSinkException {
// TODO: Support caching for unbounded requests. See TODO in {@link CacheDataSource} for
// more details.
Assertions.checkState(dataSpec.length != C.LENGTH_UNSET);
try {
this.dataSpec = dataSpec; this.dataSpec = dataSpec;
if (dataSpec.length == C.LENGTH_UNSET) {
return;
}
dataSpecBytesWritten = 0; dataSpecBytesWritten = 0;
try {
openNextOutputStream(); openNextOutputStream();
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
throw new CacheDataSinkException(e); throw new CacheDataSinkException(e);
...@@ -78,6 +78,9 @@ public final class CacheDataSink implements DataSink { ...@@ -78,6 +78,9 @@ public final class CacheDataSink implements DataSink {
@Override @Override
public void write(byte[] buffer, int offset, int length) throws CacheDataSinkException { public void write(byte[] buffer, int offset, int length) throws CacheDataSinkException {
if (dataSpec.length == C.LENGTH_UNSET) {
return;
}
try { try {
int bytesWritten = 0; int bytesWritten = 0;
while (bytesWritten < length) { while (bytesWritten < length) {
...@@ -99,6 +102,9 @@ public final class CacheDataSink implements DataSink { ...@@ -99,6 +102,9 @@ public final class CacheDataSink implements DataSink {
@Override @Override
public void close() throws CacheDataSinkException { public void close() throws CacheDataSinkException {
if (dataSpec == null || dataSpec.length == C.LENGTH_UNSET) {
return;
}
try { try {
closeCurrentOutputStream(); closeCurrentOutputStream();
} catch (IOException e) { } catch (IOException e) {
......
library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java
...@@ -22,6 +22,7 @@ import com.google.android.exoplayer2.upstream.DataSink; ...@@ -22,6 +22,7 @@ import com.google.android.exoplayer2.upstream.DataSink;
import com.google.android.exoplayer2.upstream.DataSource; import com.google.android.exoplayer2.upstream.DataSource;
import com.google.android.exoplayer2.upstream.DataSpec; import com.google.android.exoplayer2.upstream.DataSpec;
import com.google.android.exoplayer2.upstream.FileDataSource; import com.google.android.exoplayer2.upstream.FileDataSource;
import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException;
import com.google.android.exoplayer2.upstream.TeeDataSource; import com.google.android.exoplayer2.upstream.TeeDataSource;
import com.google.android.exoplayer2.upstream.cache.CacheDataSink.CacheDataSinkException; import com.google.android.exoplayer2.upstream.cache.CacheDataSink.CacheDataSinkException;
import java.io.IOException; import java.io.IOException;
...@@ -35,6 +36,34 @@ import java.io.InterruptedIOException; ...@@ -35,6 +36,34 @@ import java.io.InterruptedIOException;
public final class CacheDataSource implements DataSource { public final class CacheDataSource implements DataSource {
/** /**
* Default maximum single cache file size.
*
* @see #CacheDataSource(Cache, DataSource, int)
* @see #CacheDataSource(Cache, DataSource, int, long)
*/
public static final long DEFAULT_MAX_CACHE_FILE_SIZE = 2 * 1024 * 1024;
/**
* A flag indicating whether we will block reads if the cache key is locked. If this flag is
* set, then we will read from upstream if the cache key is locked.
*/
public static final int FLAG_BLOCK_ON_CACHE = 1 << 0;
/**
* A flag indicating whether the cache is bypassed following any cache related error. If set
* then cache related exceptions may be thrown for one cycle of open, read and close calls.
* Subsequent cycles of these calls will then bypass the cache.
*/
public static final int FLAG_IGNORE_CACHE_ON_ERROR = 1 << 1;
/**
* A flag indicating whether the response is cached if the range of the request is unbounded.
* Disabled by default because, as a side effect, this may allow streams with every chunk from a
* separate URL cached which is broken currently.
*/
public static final int FLAG_CACHE_UNBOUNDED_REQUESTS = 1 << 2;
/**
* Listener of {@link CacheDataSource} events. * Listener of {@link CacheDataSource} events.
*/ */
public interface EventListener { public interface EventListener {
...@@ -59,35 +88,44 @@ public final class CacheDataSource implements DataSource { ...@@ -59,35 +88,44 @@ public final class CacheDataSource implements DataSource {
private final boolean blockOnCache; private final boolean blockOnCache;
private final boolean ignoreCacheOnError; private final boolean ignoreCacheOnError;
private final boolean bypassUnboundedRequests;
private DataSource currentDataSource; private DataSource currentDataSource;
private boolean currentRequestUnbounded;
private Uri uri; private Uri uri;
private int flags; private int flags;
private String key; private String key;
private long readPosition; private long readPosition;
private long bytesRemaining; private long bytesRemaining;
private CacheSpan lockedSpan; private CacheSpan lockedSpan;
private boolean ignoreCache; private boolean seenCacheError;
private boolean currentRequestIgnoresCache;
private long totalCachedBytesRead; private long totalCachedBytesRead;
/** /**
* Constructs an instance with default {@link DataSource} and {@link DataSink} instances for * Constructs an instance with default {@link DataSource} and {@link DataSink} instances for
* reading and writing the cache. * reading and writing the cache and with {@link #DEFAULT_MAX_CACHE_FILE_SIZE}.
*/ */
public CacheDataSource(Cache cache, DataSource upstream, boolean blockOnCache, public CacheDataSource(Cache cache, DataSource upstream, int flags) {
boolean ignoreCacheOnError) { this(cache, upstream, flags, DEFAULT_MAX_CACHE_FILE_SIZE);
this(cache, upstream, blockOnCache, ignoreCacheOnError, Long.MAX_VALUE);
} }
/** /**
* Constructs an instance with default {@link DataSource} and {@link DataSink} instances for * Constructs an instance with default {@link DataSource} and {@link DataSink} instances for
* reading and writing the cache. The sink is configured to fragment data such that no single * reading and writing the cache. The sink is configured to fragment data such that no single
* cache file is greater than maxCacheFileSize bytes. * cache file is greater than maxCacheFileSize bytes.
*
* @param cache The cache.
* @param upstream A {@link DataSource} for reading data not in the cache.
* @param flags A combination of {@link #FLAG_BLOCK_ON_CACHE}, {@link #FLAG_IGNORE_CACHE_ON_ERROR}
* and {@link #FLAG_CACHE_UNBOUNDED_REQUESTS} or 0.
* @param maxCacheFileSize The maximum size of a cache file, in bytes. If the cached data size
* exceeds this value, then the data will be fragmented into multiple cache files. The
* finer-grained this is the finer-grained the eviction policy can be.
*/ */
public CacheDataSource(Cache cache, DataSource upstream, boolean blockOnCache, public CacheDataSource(Cache cache, DataSource upstream, int flags, long maxCacheFileSize) {
boolean ignoreCacheOnError, long maxCacheFileSize) {
this(cache, upstream, new FileDataSource(), new CacheDataSink(cache, maxCacheFileSize), this(cache, upstream, new FileDataSource(), new CacheDataSink(cache, maxCacheFileSize),
blockOnCache, ignoreCacheOnError, null); flags, null);
} }
/** /**
...@@ -99,20 +137,17 @@ public final class CacheDataSource implements DataSource { ...@@ -99,20 +137,17 @@ public final class CacheDataSource implements DataSource {
* @param upstream A {@link DataSource} for reading data not in the cache. * @param upstream A {@link DataSource} for reading data not in the cache.
* @param cacheReadDataSource A {@link DataSource} for reading data from the cache. * @param cacheReadDataSource A {@link DataSource} for reading data from the cache.
* @param cacheWriteDataSink A {@link DataSink} for writing data to the cache. * @param cacheWriteDataSink A {@link DataSink} for writing data to the cache.
* @param blockOnCache A flag indicating whether we will block reads if the cache key is locked. * @param flags A combination of {@link #FLAG_BLOCK_ON_CACHE}, {@link #FLAG_IGNORE_CACHE_ON_ERROR}
* If this flag is false, then we will read from upstream if the cache key is locked. * and {@link #FLAG_CACHE_UNBOUNDED_REQUESTS} or 0.
* @param ignoreCacheOnError Whether the cache is bypassed following any cache related error. If
* true, then cache related exceptions may be thrown for one cycle of open, read and close
* calls. Subsequent cycles of these calls will then bypass the cache.
* @param eventListener An optional {@link EventListener} to receive events. * @param eventListener An optional {@link EventListener} to receive events.
*/ */
public CacheDataSource(Cache cache, DataSource upstream, DataSource cacheReadDataSource, public CacheDataSource(Cache cache, DataSource upstream, DataSource cacheReadDataSource,
DataSink cacheWriteDataSink, boolean blockOnCache, boolean ignoreCacheOnError, DataSink cacheWriteDataSink, int flags, EventListener eventListener) {
EventListener eventListener) {
this.cache = cache; this.cache = cache;
this.cacheReadDataSource = cacheReadDataSource; this.cacheReadDataSource = cacheReadDataSource;
this.blockOnCache = blockOnCache; this.blockOnCache = (flags & FLAG_BLOCK_ON_CACHE) != 0;
this.ignoreCacheOnError = ignoreCacheOnError; this.ignoreCacheOnError = (flags & FLAG_IGNORE_CACHE_ON_ERROR) != 0;
this.bypassUnboundedRequests = (flags & FLAG_CACHE_UNBOUNDED_REQUESTS) == 0;
this.upstreamDataSource = upstream; this.upstreamDataSource = upstream;
if (cacheWriteDataSink != null) { if (cacheWriteDataSink != null) {
this.cacheWriteDataSource = new TeeDataSource(upstream, cacheWriteDataSink); this.cacheWriteDataSource = new TeeDataSource(upstream, cacheWriteDataSink);
...@@ -129,9 +164,18 @@ public final class CacheDataSource implements DataSource { ...@@ -129,9 +164,18 @@ public final class CacheDataSource implements DataSource {
flags = dataSpec.flags; flags = dataSpec.flags;
key = dataSpec.key; key = dataSpec.key;
readPosition = dataSpec.position; readPosition = dataSpec.position;
currentRequestIgnoresCache = (ignoreCacheOnError && seenCacheError)
|| (bypassUnboundedRequests && dataSpec.length == C.LENGTH_UNSET);
if (dataSpec.length != C.LENGTH_UNSET || currentRequestIgnoresCache) {
bytesRemaining = dataSpec.length; bytesRemaining = dataSpec.length;
openNextSource(); } else {
return dataSpec.length; bytesRemaining = cache.getContentLength(key);
if (bytesRemaining != C.LENGTH_UNSET) {
bytesRemaining -= dataSpec.position;
}
}
openNextSource(true);
return bytesRemaining;
} catch (IOException e) { } catch (IOException e) {
handleBeforeThrow(e); handleBeforeThrow(e);
throw e; throw e;
...@@ -151,12 +195,19 @@ public final class CacheDataSource implements DataSource { ...@@ -151,12 +195,19 @@ public final class CacheDataSource implements DataSource {
bytesRemaining -= bytesRead; bytesRemaining -= bytesRead;
} }
} else { } else {
if (currentRequestUnbounded) {
// We only do unbounded requests to upstream and only when we don't know the actual stream
// length. So we reached the end of stream.
setContentLength(readPosition);
bytesRemaining = 0;
}
closeCurrentSource(); closeCurrentSource();
if (bytesRemaining > 0 && bytesRemaining != C.LENGTH_UNSET) { if (bytesRemaining > 0 || bytesRemaining == C.LENGTH_UNSET) {
openNextSource(); if (openNextSource(false)) {
return read(buffer, offset, max); return read(buffer, offset, max);
} }
} }
}
return bytesRead; return bytesRead;
} catch (IOException e) { } catch (IOException e) {
handleBeforeThrow(e); handleBeforeThrow(e);
...@@ -185,16 +236,12 @@ public final class CacheDataSource implements DataSource { ...@@ -185,16 +236,12 @@ public final class CacheDataSource implements DataSource {
* Opens the next source. If the cache contains data spanning the current read position then * Opens the next source. If the cache contains data spanning the current read position then
* {@link #cacheReadDataSource} is opened to read from it. Else {@link #upstreamDataSource} is * {@link #cacheReadDataSource} is opened to read from it. Else {@link #upstreamDataSource} is
* opened to read from the upstream source and write into the cache. * opened to read from the upstream source and write into the cache.
* @param initial Whether it is the initial open call.
*/ */
private void openNextSource() throws IOException { private boolean openNextSource(boolean initial) throws IOException {
DataSpec dataSpec; DataSpec dataSpec;
CacheSpan span; CacheSpan span;
if (ignoreCache) { if (currentRequestIgnoresCache) {
span = null;
} else if (bytesRemaining == C.LENGTH_UNSET) {
// TODO: Support caching for unbounded requests. This requires storing the source length
// into the cache (the simplest approach is to incorporate it into each cache file's name).
Log.w(TAG, "Cache bypassed due to unbounded length.");
span = null; span = null;
} else if (blockOnCache) { } else if (blockOnCache) {
try { try {
...@@ -205,6 +252,7 @@ public final class CacheDataSource implements DataSource { ...@@ -205,6 +252,7 @@ public final class CacheDataSource implements DataSource {
} else { } else {
span = cache.startReadWriteNonBlocking(key, readPosition); span = cache.startReadWriteNonBlocking(key, readPosition);
} }
if (span == null) { if (span == null) {
// The data is locked in the cache, or we're ignoring the cache. Bypass the cache and read // The data is locked in the cache, or we're ignoring the cache. Bypass the cache and read
// from upstream. // from upstream.
...@@ -214,18 +262,63 @@ public final class CacheDataSource implements DataSource { ...@@ -214,18 +262,63 @@ public final class CacheDataSource implements DataSource {
// Data is cached, read from cache. // Data is cached, read from cache.
Uri fileUri = Uri.fromFile(span.file); Uri fileUri = Uri.fromFile(span.file);
long filePosition = readPosition - span.position; long filePosition = readPosition - span.position;
long length = Math.min(span.length - filePosition, bytesRemaining); long length = span.length - filePosition;
if (bytesRemaining != C.LENGTH_UNSET) {
length = Math.min(length, bytesRemaining);
}
dataSpec = new DataSpec(fileUri, readPosition, filePosition, length, key, flags); dataSpec = new DataSpec(fileUri, readPosition, filePosition, length, key, flags);
currentDataSource = cacheReadDataSource; currentDataSource = cacheReadDataSource;
} else { } else {
// Data is not cached, and data is not locked, read from upstream with cache backing. // Data is not cached, and data is not locked, read from upstream with cache backing.
lockedSpan = span; lockedSpan = span;
long length = span.isOpenEnded() ? bytesRemaining : Math.min(span.length, bytesRemaining); long length;
if (span.isOpenEnded()) {
length = bytesRemaining;
} else {
length = span.length;
if (bytesRemaining != C.LENGTH_UNSET) {
length = Math.min(length, bytesRemaining);
}
}
dataSpec = new DataSpec(uri, readPosition, length, key, flags); dataSpec = new DataSpec(uri, readPosition, length, key, flags);
currentDataSource = cacheWriteDataSource != null ? cacheWriteDataSource currentDataSource = cacheWriteDataSource != null ? cacheWriteDataSource
: upstreamDataSource; : upstreamDataSource;
} }
currentDataSource.open(dataSpec);
currentRequestUnbounded = dataSpec.length == C.LENGTH_UNSET;
boolean successful = false;
long currentBytesRemaining;
try {
currentBytesRemaining = currentDataSource.open(dataSpec);
successful = true;
} catch (InvalidResponseCodeException e) {
// if this isn't the initial open call (we had read some bytes) and got an 'unsatisfiable
// byte-range' (416) response for an unbounded range request then mute the exception. We are
// trying to find the stream end.
if (!initial && e.responseCode == 416 && currentRequestUnbounded) {
currentBytesRemaining = 0;
} else {
throw e;
}
}
// If we did an unbounded request (which means bytesRemaining == C.LENGTH_UNSET) and got a
// resolved length from open() request
if (currentRequestUnbounded && currentBytesRemaining != C.LENGTH_UNSET) {
bytesRemaining = currentBytesRemaining;
// If writing into cache
if (lockedSpan != null) {
setContentLength(dataSpec.position + bytesRemaining);
}
}
return successful;
}
private void setContentLength(long length) {
if (!cache.setContentLength(key, length)) {
Log.e(TAG, "cache.setContentLength(" + length + ") failed. cache.getContentLength() = "
+ cache.getContentLength(key));
}
} }
private void closeCurrentSource() throws IOException { private void closeCurrentSource() throws IOException {
...@@ -235,6 +328,7 @@ public final class CacheDataSource implements DataSource { ...@@ -235,6 +328,7 @@ public final class CacheDataSource implements DataSource {
try { try {
currentDataSource.close(); currentDataSource.close();
currentDataSource = null; currentDataSource = null;
currentRequestUnbounded = false;
} finally { } finally {
if (lockedSpan != null) { if (lockedSpan != null) {
cache.releaseHoleSpan(lockedSpan); cache.releaseHoleSpan(lockedSpan);
...@@ -244,10 +338,8 @@ public final class CacheDataSource implements DataSource { ...@@ -244,10 +338,8 @@ public final class CacheDataSource implements DataSource {
} }
private void handleBeforeThrow(IOException exception) { private void handleBeforeThrow(IOException exception) {
if (ignoreCacheOnError && (currentDataSource == cacheReadDataSource if (currentDataSource == cacheReadDataSource || exception instanceof CacheDataSinkException) {
|| exception instanceof CacheDataSinkException)) { seenCacheError = true;
// Ignore the cache from now on.
ignoreCache = true;
} }
} }
......
library/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheEvictor.java
...@@ -32,8 +32,8 @@ public interface CacheEvictor extends Cache.Listener { ...@@ -32,8 +32,8 @@ public interface CacheEvictor extends Cache.Listener {
* @param cache The source of the event. * @param cache The source of the event.
* @param key The key being written. * @param key The key being written.
* @param position The starting position of the data being written. * @param position The starting position of the data being written.
* @param length The maximum length of the data being written. * @param maxLength The maximum length of the data being written.
*/ */
void onStartFile(Cache cache, String key, long position, long length); void onStartFile(Cache cache, String key, long position, long maxLength);
} }
library/src/main/java/com/google/android/exoplayer2/upstream/cache/LeastRecentlyUsedCacheEvictor.java
...@@ -39,8 +39,8 @@ public final class LeastRecentlyUsedCacheEvictor implements CacheEvictor, Compar ...@@ -39,8 +39,8 @@ public final class LeastRecentlyUsedCacheEvictor implements CacheEvictor, Compar
} }
@Override @Override
public void onStartFile(Cache cache, String key, long position, long length) { public void onStartFile(Cache cache, String key, long position, long maxLength) {
evictCache(cache, length); evictCache(cache, maxLength);
} }
@Override @Override
......
library/src/main/java/com/google/android/exoplayer2/upstream/cache/NoOpCacheEvictor.java
...@@ -30,7 +30,7 @@ public final class NoOpCacheEvictor implements CacheEvictor { ...@@ -30,7 +30,7 @@ public final class NoOpCacheEvictor implements CacheEvictor {
} }
@Override @Override
public void onStartFile(Cache cache, String key, long position, long length) { public void onStartFile(Cache cache, String key, long position, long maxLength) {
// Do nothing. // Do nothing.
} }
......
library/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java
...@@ -16,6 +16,9 @@ ...@@ -16,6 +16,9 @@
package com.google.android.exoplayer2.upstream.cache; package com.google.android.exoplayer2.upstream.cache;
import android.os.ConditionVariable; import android.os.ConditionVariable;
import android.util.Pair;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
...@@ -35,7 +38,7 @@ public final class SimpleCache implements Cache { ...@@ -35,7 +38,7 @@ public final class SimpleCache implements Cache {
private final File cacheDir; private final File cacheDir;
private final CacheEvictor evictor; private final CacheEvictor evictor;
private final HashMap<String, CacheSpan> lockedSpans; private final HashMap<String, CacheSpan> lockedSpans;
private final HashMap<String, TreeSet<CacheSpan>> cachedSpans; private final HashMap<String, Pair<Long, TreeSet<CacheSpan>>> cachedSpans;
private final HashMap<String, ArrayList<Listener>> listeners; private final HashMap<String, ArrayList<Listener>> listeners;
private long totalSpace = 0; private long totalSpace = 0;
...@@ -89,7 +92,7 @@ public final class SimpleCache implements Cache { ...@@ -89,7 +92,7 @@ public final class SimpleCache implements Cache {
@Override @Override
public synchronized NavigableSet<CacheSpan> getCachedSpans(String key) { public synchronized NavigableSet<CacheSpan> getCachedSpans(String key) {
TreeSet<CacheSpan> spansForKey = cachedSpans.get(key); TreeSet<CacheSpan> spansForKey = getSpansForKey(key);
return spansForKey == null ? null : new TreeSet<>(spansForKey); return spansForKey == null ? null : new TreeSet<>(spansForKey);
} }
...@@ -127,26 +130,21 @@ public final class SimpleCache implements Cache { ...@@ -127,26 +130,21 @@ public final class SimpleCache implements Cache {
} }
private synchronized CacheSpan startReadWriteNonBlocking(CacheSpan lookupSpan) { private synchronized CacheSpan startReadWriteNonBlocking(CacheSpan lookupSpan) {
CacheSpan spanningRegion = getSpan(lookupSpan); CacheSpan cacheSpan = getSpan(lookupSpan);
// Read case. // Read case.
if (spanningRegion.isCached) { if (cacheSpan.isCached) {
CacheSpan oldCacheSpan = spanningRegion;
// Remove the old span from the in-memory representation.
TreeSet<CacheSpan> spansForKey = cachedSpans.get(oldCacheSpan.key);
Assertions.checkState(spansForKey.remove(oldCacheSpan));
// Obtain a new span with updated last access timestamp. // Obtain a new span with updated last access timestamp.
spanningRegion = oldCacheSpan.touch(); CacheSpan newCacheSpan = cacheSpan.touch();
// Add the updated span back into the in-memory representation. replaceSpan(cacheSpan, newCacheSpan);
spansForKey.add(spanningRegion); notifySpanTouched(cacheSpan, newCacheSpan);
notifySpanTouched(oldCacheSpan, spanningRegion); return newCacheSpan;
return spanningRegion;
} }
// Write case, lock available. // Write case, lock available.
if (!lockedSpans.containsKey(lookupSpan.key)) { if (!lockedSpans.containsKey(lookupSpan.key)) {
lockedSpans.put(lookupSpan.key, spanningRegion); lockedSpans.put(lookupSpan.key, cacheSpan);
return spanningRegion; return cacheSpan;
} }
// Write case, lock not available. // Write case, lock not available.
...@@ -154,14 +152,14 @@ public final class SimpleCache implements Cache { ...@@ -154,14 +152,14 @@ public final class SimpleCache implements Cache {
} }
@Override @Override
public synchronized File startFile(String key, long position, long length) { public synchronized File startFile(String key, long position, long maxLength) {
Assertions.checkState(lockedSpans.containsKey(key)); Assertions.checkState(lockedSpans.containsKey(key));
if (!cacheDir.exists()) { if (!cacheDir.exists()) {
// For some reason the cache directory doesn't exist. Make a best effort to create it. // For some reason the cache directory doesn't exist. Make a best effort to create it.
removeStaleSpans(); removeStaleSpans();
cacheDir.mkdirs(); cacheDir.mkdirs();
} }
evictor.onStartFile(this, key, position, length); evictor.onStartFile(this, key, position, maxLength);
return CacheSpan.getCacheFileName(cacheDir, key, position, System.currentTimeMillis(), false); return CacheSpan.getCacheFileName(cacheDir, key, position, System.currentTimeMillis(), false);
} }
...@@ -175,11 +173,15 @@ public final class SimpleCache implements Cache { ...@@ -175,11 +173,15 @@ public final class SimpleCache implements Cache {
return; return;
} }
// If the file has length 0, delete it and don't add it to the in-memory representation. // If the file has length 0, delete it and don't add it to the in-memory representation.
long length = file.length(); if (file.length() == 0) {
if (length == 0) {
file.delete(); file.delete();
return; return;
} }
// Check if the span conflicts with the set content length
Long length = getContentLength(span.key);
if (length != C.LENGTH_UNSET) {
Assertions.checkState((span.position + span.length) <= length);
}
addSpan(span); addSpan(span);
notifyAll(); notifyAll();
} }
...@@ -204,7 +206,7 @@ public final class SimpleCache implements Cache { ...@@ -204,7 +206,7 @@ public final class SimpleCache implements Cache {
private CacheSpan getSpan(CacheSpan lookupSpan) { private CacheSpan getSpan(CacheSpan lookupSpan) {
String key = lookupSpan.key; String key = lookupSpan.key;
long offset = lookupSpan.position; long offset = lookupSpan.position;
TreeSet<CacheSpan> entries = cachedSpans.get(key); TreeSet<CacheSpan> entries = getSpansForKey(key);
if (entries == null) { if (entries == null) {
return CacheSpan.createOpenHole(key, lookupSpan.position); return CacheSpan.createOpenHole(key, lookupSpan.position);
} }
...@@ -260,10 +262,13 @@ public final class SimpleCache implements Cache { ...@@ -260,10 +262,13 @@ public final class SimpleCache implements Cache {
* @param span The span to be added. * @param span The span to be added.
*/ */
private void addSpan(CacheSpan span) { private void addSpan(CacheSpan span) {
TreeSet<CacheSpan> spansForKey = cachedSpans.get(span.key); Pair<Long, TreeSet<CacheSpan>> entryForKey = cachedSpans.get(span.key);
if (spansForKey == null) { TreeSet<CacheSpan> spansForKey;
if (entryForKey == null) {
spansForKey = new TreeSet<>(); spansForKey = new TreeSet<>();
cachedSpans.put(span.key, spansForKey); setKeyValue(span.key, C.LENGTH_UNSET, spansForKey);
} else {
spansForKey = entryForKey.second;
} }
spansForKey.add(span); spansForKey.add(span);
totalSpace += span.length; totalSpace += span.length;
...@@ -272,7 +277,7 @@ public final class SimpleCache implements Cache { ...@@ -272,7 +277,7 @@ public final class SimpleCache implements Cache {
@Override @Override
public synchronized void removeSpan(CacheSpan span) { public synchronized void removeSpan(CacheSpan span) {
TreeSet<CacheSpan> spansForKey = cachedSpans.get(span.key); TreeSet<CacheSpan> spansForKey = getSpansForKey(span.key);
totalSpace -= span.length; totalSpace -= span.length;
Assertions.checkState(spansForKey.remove(span)); Assertions.checkState(spansForKey.remove(span));
span.file.delete(); span.file.delete();
...@@ -287,10 +292,11 @@ public final class SimpleCache implements Cache { ...@@ -287,10 +292,11 @@ public final class SimpleCache implements Cache {
* no longer exist. * no longer exist.
*/ */
private void removeStaleSpans() { private void removeStaleSpans() {
Iterator<Entry<String, TreeSet<CacheSpan>>> iterator = cachedSpans.entrySet().iterator(); Iterator<Entry<String, Pair<Long, TreeSet<CacheSpan>>>> iterator =
cachedSpans.entrySet().iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
Entry<String, TreeSet<CacheSpan>> next = iterator.next(); Entry<String, Pair<Long, TreeSet<CacheSpan>>> next = iterator.next();
Iterator<CacheSpan> spanIterator = next.getValue().iterator(); Iterator<CacheSpan> spanIterator = next.getValue().second.iterator();
boolean isEmpty = true; boolean isEmpty = true;
while (spanIterator.hasNext()) { while (spanIterator.hasNext()) {
CacheSpan span = spanIterator.next(); CacheSpan span = spanIterator.next();
...@@ -342,7 +348,7 @@ public final class SimpleCache implements Cache { ...@@ -342,7 +348,7 @@ public final class SimpleCache implements Cache {
@Override @Override
public synchronized boolean isCached(String key, long position, long length) { public synchronized boolean isCached(String key, long position, long length) {
TreeSet<CacheSpan> entries = cachedSpans.get(key); TreeSet<CacheSpan> entries = getSpansForKey(key);
if (entries == null) { if (entries == null) {
return false; return false;
} }
...@@ -375,4 +381,49 @@ public final class SimpleCache implements Cache { ...@@ -375,4 +381,49 @@ public final class SimpleCache implements Cache {
return false; return false;
} }
@Override
public synchronized boolean setContentLength(String key, long length) {
Pair<Long, TreeSet<CacheSpan>> entryForKey = cachedSpans.get(key);
TreeSet<CacheSpan> entries;
if (entryForKey != null) {
entries = entryForKey.second;
if (entries != null && !entries.isEmpty()) {
CacheSpan last = entries.last();
long end = last.position + last.length;
if (end > length) {
return false;
}
}
} else {
entries = new TreeSet<>();
}
// TODO persist the length value
setKeyValue(key, length, entries);
return true;
}
@Override
public synchronized long getContentLength(String key) {
Pair<Long, TreeSet<CacheSpan>> entryForKey = cachedSpans.get(key);
return entryForKey == null ? C.LENGTH_UNSET : entryForKey.first;
}
private TreeSet<CacheSpan> getSpansForKey(String key) {
Pair<Long, TreeSet<CacheSpan>> entryForKey = cachedSpans.get(key);
return entryForKey != null ? entryForKey.second : null;
}
private void setKeyValue(String key, long length, TreeSet<CacheSpan> entries) {
cachedSpans.put(key, Pair.create(length, entries));
}
private void replaceSpan(CacheSpan oldSpan, CacheSpan newSpan) {
// Remove the old span from the in-memory representation.
TreeSet<CacheSpan> spansForKey = getSpansForKey(oldSpan.key);
Assertions.checkState(spansForKey.remove(oldSpan));
// Add the updated span back into the in-memory representation.
spansForKey.add(newSpan);
}
} }
testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeDataSource.java
...@@ -49,6 +49,7 @@ public final class FakeDataSource implements DataSource { ...@@ -49,6 +49,7 @@ public final class FakeDataSource implements DataSource {
private boolean opened; private boolean opened;
private int currentSegmentIndex; private int currentSegmentIndex;
private long bytesRemaining; private long bytesRemaining;
private IOException unsatisfiableRangeException;
private FakeDataSource(boolean simulateUnknownLength, ArrayList<Segment> segments) { private FakeDataSource(boolean simulateUnknownLength, ArrayList<Segment> segments) {
this.simulateUnknownLength = simulateUnknownLength; this.simulateUnknownLength = simulateUnknownLength;
...@@ -59,6 +60,7 @@ public final class FakeDataSource implements DataSource { ...@@ -59,6 +60,7 @@ public final class FakeDataSource implements DataSource {
} }
this.totalLength = totalLength; this.totalLength = totalLength;
openedDataSpecs = new ArrayList<>(); openedDataSpecs = new ArrayList<>();
unsatisfiableRangeException = new IOException("Unsatisfiable range");
} }
@Override @Override
...@@ -69,11 +71,9 @@ public final class FakeDataSource implements DataSource { ...@@ -69,11 +71,9 @@ public final class FakeDataSource implements DataSource {
uri = dataSpec.uri; uri = dataSpec.uri;
openedDataSpecs.add(dataSpec); openedDataSpecs.add(dataSpec);
// If the source knows that the request is unsatisfiable then fail. // If the source knows that the request is unsatisfiable then fail.
if (dataSpec.position >= totalLength) { if (dataSpec.position >= totalLength || (dataSpec.length != C.LENGTH_UNSET
throw new IOException("Unsatisfiable position"); && (dataSpec.position + dataSpec.length > totalLength))) {
} else if (dataSpec.length != C.LENGTH_UNSET throw (IOException) unsatisfiableRangeException.fillInStackTrace();
&& dataSpec.position + dataSpec.length > totalLength) {
throw new IOException("Unsatisfiable range");
} }
// Scan through the segments, configuring them for the current read. // Scan through the segments, configuring them for the current read.
boolean findingCurrentSegmentIndex = true; boolean findingCurrentSegmentIndex = true;
...@@ -107,10 +107,10 @@ public final class FakeDataSource implements DataSource { ...@@ -107,10 +107,10 @@ public final class FakeDataSource implements DataSource {
return C.RESULT_END_OF_INPUT; return C.RESULT_END_OF_INPUT;
} }
Segment current = segments.get(currentSegmentIndex); Segment current = segments.get(currentSegmentIndex);
if (current.exception != null) { if (current.isErrorSegment()) {
if (!current.exceptionCleared) { if (!current.exceptionCleared) {
current.exceptionThrown = true; current.exceptionThrown = true;
throw current.exception; throw (IOException) current.exception.fillInStackTrace();
} else { } else {
currentSegmentIndex++; currentSegmentIndex++;
} }
...@@ -160,6 +160,10 @@ public final class FakeDataSource implements DataSource { ...@@ -160,6 +160,10 @@ public final class FakeDataSource implements DataSource {
return dataSpecs; return dataSpecs;
} }
public void setUnsatisfiableRangeException(IOException unsatisfiableRangeException) {
this.unsatisfiableRangeException = unsatisfiableRangeException;
}
private static class Segment { private static class Segment {
public final IOException exception; public final IOException exception;
......
testutils/src/main/java/com/google/android/exoplayer2/testutil/TestUtil.java
...@@ -24,6 +24,7 @@ import com.google.android.exoplayer2.extractor.SeekMap; ...@@ -24,6 +24,7 @@ import com.google.android.exoplayer2.extractor.SeekMap;
import com.google.android.exoplayer2.testutil.FakeExtractorInput.SimulatedIOException; import com.google.android.exoplayer2.testutil.FakeExtractorInput.SimulatedIOException;
import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.util.Util;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.Arrays; import java.util.Arrays;
...@@ -303,4 +304,13 @@ public class TestUtil { ...@@ -303,4 +304,13 @@ public class TestUtil {
return extractorOutput; return extractorOutput;
} }
public static void recursiveDelete(File fileOrDirectory) {
if (fileOrDirectory.isDirectory()) {
for (File child : fileOrDirectory.listFiles()) {
recursiveDelete(child);
}
}
fileOrDirectory.delete();
}
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment