Add ImageAssetLoader and ImageConfiguration.

PiperOrigin-RevId: 506619637

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java

@@ -16,18 +16,29 @@
 
 
 package androidx.media3.transformer;
 package androidx.media3.transformer;
 
 
+import static androidx.media3.common.util.Assertions.checkNotNull;
+
 import android.content.Context;
 import android.content.Context;
 import android.os.Looper;
 import android.os.Looper;
+import androidx.annotation.Nullable;
+import androidx.media3.common.MediaItem;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.exoplayer.source.MediaSource;
 import androidx.media3.exoplayer.source.MediaSource;
+import com.google.common.collect.ImmutableList;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 
 /** The default {@link AssetLoader.Factory} implementation. */
 /** The default {@link AssetLoader.Factory} implementation. */
 @UnstableApi
 @UnstableApi
 public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
 public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
 
 
-  private final AssetLoader.Factory assetLoaderFactory;
+  private final Context context;
+  private final Codec.DecoderFactory decoderFactory;
+  private final Clock clock;
+  private final MediaSource.@MonotonicNonNull Factory mediaSourceFactory;
 
 
+  private AssetLoader.@MonotonicNonNull Factory imageAssetLoaderFactory;
+  private AssetLoader.@MonotonicNonNull Factory exoPlayerAssetLoaderFactory;
   /**
   /**
    * Creates an instance.
    * Creates an instance.
    *
    *
@@ -39,7 +50,10 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
    */
    */
   public DefaultAssetLoaderFactory(
   public DefaultAssetLoaderFactory(
       Context context, Codec.DecoderFactory decoderFactory, Clock clock) {
       Context context, Codec.DecoderFactory decoderFactory, Clock clock) {
-    assetLoaderFactory = new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock);
+    this.context = context;
+    this.decoderFactory = decoderFactory;
+    this.clock = clock;
+    this.mediaSourceFactory = null;
   }
   }
 
 
   /**
   /**
@@ -58,13 +72,42 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
       Codec.DecoderFactory decoderFactory,
       Codec.DecoderFactory decoderFactory,
       Clock clock,
       Clock clock,
       MediaSource.Factory mediaSourceFactory) {
       MediaSource.Factory mediaSourceFactory) {
-    assetLoaderFactory =
-        new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock, mediaSourceFactory);
+    this.context = context;
+    this.decoderFactory = decoderFactory;
+    this.clock = clock;
+    this.mediaSourceFactory = mediaSourceFactory;
   }
   }
 
 
   @Override
   @Override
   public AssetLoader createAssetLoader(
   public AssetLoader createAssetLoader(
       EditedMediaItem editedMediaItem, Looper looper, AssetLoader.Listener listener) {
       EditedMediaItem editedMediaItem, Looper looper, AssetLoader.Listener listener) {
-    return assetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener);
+    MediaItem mediaItem = editedMediaItem.mediaItem;
+    if (isImage(mediaItem.localConfiguration)) {
+      if (imageAssetLoaderFactory == null) {
+        imageAssetLoaderFactory = new ImageAssetLoader.Factory();
+      }
+      return imageAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener);
+    }
+    if (exoPlayerAssetLoaderFactory == null) {
+      exoPlayerAssetLoaderFactory =
+          mediaSourceFactory != null
+              ? new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock, mediaSourceFactory)
+              : new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock);
+    }
+    return exoPlayerAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener);
+  }
+
+  private static boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) {
+    if (localConfiguration == null) {
+      return false;
+    }
+    ImmutableList<String> supportedImageTypes = ImmutableList.of(".png", ".webp", ".jpg", ".jpeg");
+    String uriPath = checkNotNull(localConfiguration.uri.getPath());
+    int fileExtensionStart = uriPath.lastIndexOf(".");
+    if (fileExtensionStart < 0) {
+      return false;
+    }
+    String extension = uriPath.substring(fileExtensionStart);
+    return supportedImageTypes.contains(extension);
   }
   }
 }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java

@@ -18,6 +18,8 @@ package androidx.media3.transformer;
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkState;
 
 
+import androidx.annotation.IntRange;
+import androidx.media3.common.C;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.exoplayer.source.MediaSource;
 import androidx.media3.exoplayer.source.MediaSource;
@@ -36,15 +38,22 @@ public final class EditedMediaItem {
     private boolean removeAudio;
     private boolean removeAudio;
     private boolean removeVideo;
     private boolean removeVideo;
     private boolean flattenForSlowMotion;
     private boolean flattenForSlowMotion;
+    private long durationUs;
+    private int frameRate;
     private Effects effects;
     private Effects effects;
 
 
     /**
     /**
      * Creates an instance.
      * Creates an instance.
      *
      *
+     * <p>For image inputs, the values passed into {@link #setRemoveAudio}, {@link #setRemoveVideo}
+     * and {@link #setFlattenForSlowMotion} will be ignored.
+     *
      * @param mediaItem The {@link MediaItem} on which transformations are applied.
      * @param mediaItem The {@link MediaItem} on which transformations are applied.
      */
      */
     public Builder(MediaItem mediaItem) {
     public Builder(MediaItem mediaItem) {
       this.mediaItem = mediaItem;
       this.mediaItem = mediaItem;
+      durationUs = C.TIME_UNSET;
+      frameRate = C.RATE_UNSET_INT;
       effects = Effects.EMPTY;
       effects = Effects.EMPTY;
     }
     }
 
 
@@ -112,6 +121,38 @@ public final class EditedMediaItem {
     }
     }
 
 
     /**
     /**
+     * Sets the duration of the output video in microseconds.
+     *
+     * <p>This should be set for inputs that don't have an implicit duration (e.g. images). It will
+     * be ignored for inputs that do have an implicit duration (e.g. video).
+     *
+     * <p>The default value is {@link C#TIME_UNSET}.
+     */
+    @CanIgnoreReturnValue
+    public Builder setDurationUs(long durationUs) {
+      checkArgument(durationUs > 0);
+      this.durationUs = durationUs;
+      return this;
+    }
+
+    /**
+     * Sets the frame rate of the output video in frames per second.
+     *
+     * <p>This should be set for inputs that don't have an implicit frame rate (e.g. images). It
+     * will be ignored for inputs that do have an implicit frame rate (e.g. video).
+     *
+     * <p>The default value is {@link C#RATE_UNSET_INT}.
+     */
+    // TODO(b/210593170): Remove/deprecate frameRate parameter when frameRate parameter is added to
+    //     transformer.
+    @CanIgnoreReturnValue
+    public Builder setFrameRate(@IntRange(from = 0) int frameRate) {
+      checkArgument(frameRate > 0);
+      this.frameRate = frameRate;
+      return this;
+    }
+
+    /**
      * Sets the {@link Effects} to apply to the {@link MediaItem}.
      * Sets the {@link Effects} to apply to the {@link MediaItem}.
      *
      *
      * <p>The default value is {@link Effects#EMPTY}.
      * <p>The default value is {@link Effects#EMPTY}.
@@ -128,7 +169,13 @@ public final class EditedMediaItem {
     /** Builds an {@link EditedMediaItem} instance. */
     /** Builds an {@link EditedMediaItem} instance. */
     public EditedMediaItem build() {
     public EditedMediaItem build() {
       return new EditedMediaItem(
       return new EditedMediaItem(
-          mediaItem, removeAudio, removeVideo, flattenForSlowMotion, effects);
+          mediaItem,
+          removeAudio,
+          removeVideo,
+          flattenForSlowMotion,
+          durationUs,
+          frameRate,
+          effects);
     }
     }
   }
   }
 
 
@@ -156,6 +203,11 @@ public final class EditedMediaItem {
    * </ul>
    * </ul>
    */
    */
   public final boolean flattenForSlowMotion;
   public final boolean flattenForSlowMotion;
+  /** The duration of the image in the output video, in microseconds. */
+  public final long durationUs;
+  /** The frame rate of the image in the output video, in frames per second. */
+  @IntRange(from = 0)
+  public final int frameRate;
   /** The {@link Effects} to apply to the {@link #mediaItem}. */
   /** The {@link Effects} to apply to the {@link #mediaItem}. */
   public final Effects effects;
   public final Effects effects;
 
 
@@ -164,12 +216,16 @@ public final class EditedMediaItem {
       boolean removeAudio,
       boolean removeAudio,
       boolean removeVideo,
       boolean removeVideo,
       boolean flattenForSlowMotion,
       boolean flattenForSlowMotion,
+      long durationUs,
+      int frameRate,
       Effects effects) {
       Effects effects) {
     checkState(!removeAudio || !removeVideo, "Audio and video cannot both be removed");
     checkState(!removeAudio || !removeVideo, "Audio and video cannot both be removed");
     this.mediaItem = mediaItem;
     this.mediaItem = mediaItem;
     this.removeAudio = removeAudio;
     this.removeAudio = removeAudio;
     this.removeVideo = removeVideo;
     this.removeVideo = removeVideo;
     this.flattenForSlowMotion = flattenForSlowMotion;
     this.flattenForSlowMotion = flattenForSlowMotion;
+    this.durationUs = durationUs;
+    this.frameRate = frameRate;
     this.effects = effects;
     this.effects = effects;
   }
   }
 }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java

+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.transformer.TransformationException.ERROR_CODE_IO_UNSPECIFIED;
+import static androidx.media3.transformer.TransformationException.ERROR_CODE_UNSPECIFIED;
+import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
+import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
+
+import android.graphics.Bitmap;
+import android.os.Looper;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MediaItem;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.BitmapLoader;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.effect.SimpleBitmapLoader;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+
+/** An {@link AssetLoader} implementation that loads images into {@link Bitmap} instances. */
+@UnstableApi
+public final class ImageAssetLoader implements AssetLoader {
+
+  /** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */
+  public static final class Factory implements AssetLoader.Factory {
+
+    @Override
+    public AssetLoader createAssetLoader(
+        EditedMediaItem editedMediaItem, Looper looper, Listener listener) {
+      return new ImageAssetLoader(editedMediaItem, listener);
+    }
+  }
+
+  public static final String MIME_TYPE_IMAGE_ALL = MimeTypes.BASE_TYPE_IMAGE + "/*";
+
+  private final EditedMediaItem editedMediaItem;
+  private final Listener listener;
+
+  private @Transformer.ProgressState int progressState;
+  private int progress;
+
+  private ImageAssetLoader(EditedMediaItem editedMediaItem, Listener listener) {
+    this.editedMediaItem = editedMediaItem;
+    this.listener = listener;
+
+    progressState = PROGRESS_STATE_NOT_STARTED;
+  }
+
+  @Override
+  public void start() {
+    progressState = PROGRESS_STATE_AVAILABLE;
+    listener.onTrackCount(1);
+    BitmapLoader bitmapLoader = new SimpleBitmapLoader();
+    MediaItem.LocalConfiguration localConfiguration =
+        checkNotNull(editedMediaItem.mediaItem.localConfiguration);
+    ListenableFuture<Bitmap> future = bitmapLoader.loadBitmap(localConfiguration.uri);
+    Futures.addCallback(
+        future,
+        new FutureCallback<Bitmap>() {
+          @Override
+          public void onSuccess(Bitmap bitmap) {
+            progress = 50;
+            try {
+              Format format =
+                  new Format.Builder()
+                      .setHeight(bitmap.getHeight())
+                      .setWidth(bitmap.getWidth())
+                      .setSampleMimeType(MIME_TYPE_IMAGE_ALL)
+                      .build();
+              SampleConsumer sampleConsumer =
+                  listener.onTrackAdded(
+                      format,
+                      SUPPORTED_OUTPUT_TYPE_DECODED,
+                      /* streamStartPositionUs= */ 0,
+                      /* streamOffsetUs= */ 0);
+              checkState(editedMediaItem.durationUs != C.TIME_UNSET);
+              checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT);
+              // TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change
+              //    callback (when it's added) rather than setting duration here.
+              sampleConsumer.queueInputBitmap(
+                  bitmap, editedMediaItem.durationUs, editedMediaItem.frameRate);
+              sampleConsumer.signalEndOfVideoInput();
+            } catch (TransformationException e) {
+              listener.onTransformationError(e);
+            } catch (RuntimeException e) {
+              listener.onTransformationError(
+                  TransformationException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
+            }
+            progress = 100;
+          }
+
+          @Override
+          public void onFailure(Throwable t) {
+            listener.onTransformationError(
+                TransformationException.createForAssetLoader(t, ERROR_CODE_IO_UNSPECIFIED));
+          }
+        },
+        MoreExecutors.directExecutor());
+  }
+
+  @Override
+  public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) {
+    if (progressState == PROGRESS_STATE_AVAILABLE) {
+      progressHolder.progress = progress;
+    }
+    return progressState;
+  }
+
+  @Override
+  public ImmutableMap<Integer, String> getDecoderNames() {
+    return ImmutableMap.of();
+  }
+
+  @Override
+  public void release() {
+    progressState = PROGRESS_STATE_NOT_STARTED;
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java

@@ -15,6 +15,7 @@
  */
  */
 package androidx.media3.transformer;
 package androidx.media3.transformer;
 
 
+import android.graphics.Bitmap;
 import android.view.Surface;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.annotation.Nullable;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.ColorInfo;
@@ -65,6 +66,20 @@ public interface SampleConsumer {
     throw new UnsupportedOperationException();
     throw new UnsupportedOperationException();
   }
   }
 
 
+  /**
+   * Provides an input {@link Bitmap} to the consumer.
+   *
+   * <p>Should only be used for image data.
+   *
+   * @param inputBitmap The {@link Bitmap} queued to the consumer.
+   * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
+   * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
+   *     second.
+   */
+  default void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {
+    throw new UnsupportedOperationException();
+  }
+
   // Methods to pass raw video input.
   // Methods to pass raw video input.
 
 
   /**
   /**