Porting over ET MultiModal Demo App (#4455)

Summary:
Pull Request resolved: #4455

Adding ET demo app for multimodal support. This is the first diff that supports Llama3.

This includes major changes to the existing Llama Demo app with having the following features:
1. Llama3 support
2. Settings activity
3. UI/UX improvements to the mainactivity
4. Ability to add multi-images in prep for multimodal support.
5. Metrics

Note: You'll need to build the `executorch-llama.aar` and have it placed in the `app/libs` folder.

Reviewed By: kirklandsign

Differential Revision: D60416605

fbshipit-source-id: 262329c30e1ec28c3905da5c040dc661307f8666

Author: cmodi-meta

examples/demo-apps/android/LlamaDemo/app/build.gradle.kts

@@ -17,7 +17,7 @@ android {
 
   defaultConfig {
     applicationId = "com.example.executorchllamademo"
-    minSdk = 24
+    minSdk = 28
     targetSdk = 33
     versionCode = 1
     versionName = "1.0"
@@ -56,7 +56,10 @@ dependencies {
   implementation("androidx.camera:camera-core:1.3.0-rc02")
   implementation("androidx.constraintlayout:constraintlayout:2.2.0-alpha12")
   implementation("com.facebook.fbjni:fbjni:0.5.1")
+  implementation("com.google.code.gson:gson:2.8.6")
   implementation(files("libs/executorch-llama.aar"))
+  implementation("com.google.android.material:material:1.12.0")
+  implementation("androidx.activity:activity:1.9.0")
   testImplementation("junit:junit:4.13.2")
   androidTestImplementation("androidx.test.ext:junit:1.1.5")
   androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1")

examples/demo-apps/android/LlamaDemo/app/src/main/AndroidManifest.xml

@@ -3,32 +3,44 @@
     xmlns:tools="http://schemas.android.com/tools"
     package="com.example.executorchllamademo">
 
-    <uses-sdk android:minSdkVersion="19"
-          android:targetSdkVersion="34"
-          android:maxSdkVersion="40" />
+    <uses-sdk
+        android:maxSdkVersion="40"
+        android:minSdkVersion="28"
+        android:targetSdkVersion="34" />
 
     <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+    <uses-permission android:name="android.permission.CAMERA" />
+
+    <uses-feature android:name="android.hardware.camera" />
 
     <application
+        android:name=".ETLogging"
         android:allowBackup="true"
         android:dataExtractionRules="@xml/data_extraction_rules"
+        android:extractNativeLibs="true"
         android:fullBackupContent="@xml/backup_rules"
-        android:icon="@mipmap/ic_launcher"
+        android:icon="@drawable/logo"
         android:label="@string/app_name"
-        android:roundIcon="@mipmap/ic_launcher_round"
         android:supportsRtl="true"
-        android:theme="@style/Theme.ExecuTorchLlamaDemo"
-        android:extractNativeLibs="true"
+        android:theme="@style/Theme.AppCompat.Light.NoActionBar"
         tools:targetApi="34">
+        <activity
+            android:name=".LogsActivity"
+            android:exported="false" />
+        <activity
+            android:name=".SettingsActivity"
+            android:exported="false" />
 
-        <uses-native-library android:name="libcdsprpc.so"
-            android:required="false"/>
+        <uses-native-library
+            android:name="libcdsprpc.so"
+            android:required="false" />
 
         <activity
             android:name=".MainActivity"
             android:exported="true"
             android:label="@string/app_name"
-            android:theme="@style/Theme.ExecuTorchLlamaDemo">
+            android:theme="@style/Theme.AppCompat.Light.NoActionBar">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
 

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/AppLog.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.example.executorchllamademo;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Locale;
+
+public class AppLog {
+  private final Long timestamp;
+  private final String message;
+
+  public AppLog(String message) {
+    this.timestamp = getCurrentTimeStamp();
+    this.message = message;
+  }
+
+  public Long getTimestamp() {
+    return timestamp;
+  }
+
+  public String getMessage() {
+    return message;
+  }
+
+  public String getFormattedLog() {
+    return "[" + getFormattedTimeStamp() + "] " + message;
+  }
+
+  private Long getCurrentTimeStamp() {
+    return System.currentTimeMillis();
+  }
+
+  private String getFormattedTimeStamp() {
+    return formatDate(timestamp);
+  }
+
+  private String formatDate(long milliseconds) {
+    SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd  HH:mm:ss", Locale.getDefault());
+    Date date = new Date(milliseconds);
+    return formatter.format(date);
+  }
+}

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/DemoSharedPreferences.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.example.executorchllamademo;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+
+public class DemoSharedPreferences {
+  Context context;
+  SharedPreferences sharedPreferences;
+
+  public DemoSharedPreferences(Context context) {
+    this.context = context;
+    this.sharedPreferences = getSharedPrefs();
+  }
+
+  private SharedPreferences getSharedPrefs() {
+    return context.getSharedPreferences(
+        context.getString(R.string.demo_pref_file_key), Context.MODE_PRIVATE);
+  }
+
+  public String getSavedMessages() {
+    return sharedPreferences.getString(context.getString(R.string.saved_messages_json_key), "");
+  }
+
+  public void addMessages(MessageAdapter messageAdapter) {
+    SharedPreferences.Editor editor = sharedPreferences.edit();
+    Gson gson = new Gson();
+    String msgJSON = gson.toJson(messageAdapter.getSavedMessages());
+    editor.putString(context.getString(R.string.saved_messages_json_key), msgJSON);
+    editor.apply();
+  }
+
+  public void removeExistingMessages() {
+    SharedPreferences.Editor editor = sharedPreferences.edit();
+    editor.remove(context.getString(R.string.saved_messages_json_key));
+    editor.apply();
+  }
+
+  public void addSettings(SettingsFields settingsFields) {
+    SharedPreferences.Editor editor = sharedPreferences.edit();
+    Gson gson = new Gson();
+    String settingsJSON = gson.toJson(settingsFields);
+    editor.putString(context.getString(R.string.settings_json_key), settingsJSON);
+    editor.apply();
+  }
+
+  public String getSettings() {
+    return sharedPreferences.getString(context.getString(R.string.settings_json_key), "");
+  }
+
+  public void saveLogs() {
+    SharedPreferences.Editor editor = sharedPreferences.edit();
+    Gson gson = new Gson();
+    String msgJSON = gson.toJson(ETLogging.getInstance().getLogs());
+    editor.putString(context.getString(R.string.logs_json_key), msgJSON);
+    editor.apply();
+  }
+
+  public void removeExistingLogs() {
+    SharedPreferences.Editor editor = sharedPreferences.edit();
+    editor.remove(context.getString(R.string.logs_json_key));
+    editor.apply();
+  }
+
+  public ArrayList<AppLog> getSavedLogs() {
+    String logsJSONString =
+        sharedPreferences.getString(context.getString(R.string.logs_json_key), null);
+    if (logsJSONString == null || logsJSONString.isEmpty()) {
+      return new ArrayList<>();
+    }
+    Gson gson = new Gson();
+    Type type = new TypeToken<ArrayList<AppLog>>() {}.getType();
+    ArrayList<AppLog> appLogs = gson.fromJson(logsJSONString, type);
+    if (appLogs == null) {
+      return new ArrayList<>();
+    }
+    return appLogs;
+  }
+}

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ETImage.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.example.executorchllamademo;
+
+import android.content.ContentResolver;
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Color;
+import android.net.Uri;
+import androidx.annotation.Nullable;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+
+public class ETImage {
+  private int width;
+  private int height;
+  private final byte[] bytes;
+  private final Uri uri;
+  private final ContentResolver contentResolver;
+
+  ETImage(ContentResolver contentResolver, Uri uri) {
+    this.contentResolver = contentResolver;
+    this.uri = uri;
+    bytes = getBytesFromImageURI(uri);
+  }
+
+  public int getWidth() {
+    return width;
+  }
+
+  public int getHeight() {
+    return height;
+  }
+
+  public Uri getUri() {
+    return uri;
+  }
+
+  public byte[] getBytes() {
+    return bytes;
+  }
+
+  private byte[] getBytesFromImageURI(Uri uri) {
+    try {
+      int RESIZED_IMAGE_WIDTH = 336;
+      Bitmap bitmap = resizeImage(uri, RESIZED_IMAGE_WIDTH);
+
+      if (bitmap == null) {
+        ETLogging.getInstance().log("Unable to get bytes from Image URI. Bitmap is null");
+        return new byte[0];
+      }
+
+      width = bitmap.getWidth();
+      height = bitmap.getHeight();
+
+      byte[] rgbValues = new byte[width * height * 3];
+
+      for (int y = 0; y < height; y++) {
+        for (int x = 0; x < width; x++) {
+          // Get the color of the current pixel
+          int color = bitmap.getPixel(x, y);
+
+          // Extract the RGB values from the color
+          int red = Color.red(color);
+          int green = Color.green(color);
+          int blue = Color.blue(color);
+
+          // Store the RGB values in the byte array
+          rgbValues[(y * width + x) * 3] = (byte) red;
+          rgbValues[(y * width + x) * 3 + 1] = (byte) green;
+          rgbValues[(y * width + x) * 3 + 2] = (byte) blue;
+        }
+      }
+      return rgbValues;
+    } catch (FileNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Nullable
+  private Bitmap resizeImage(Uri uri, int maxLength) throws FileNotFoundException {
+    InputStream inputStream = contentResolver.openInputStream(uri);
+    if (inputStream == null) {
+      ETLogging.getInstance().log("Unable to resize image, input streams is null");
+      return null;
+    }
+    Bitmap bitmap = BitmapFactory.decodeStream(inputStream);
+    if (bitmap == null) {
+      ETLogging.getInstance().log("Unable to resize image, bitmap during decode stream is null");
+      return null;
+    }
+
+    float aspectRatio;
+    int finalWidth, finalHeight;
+
+    if (bitmap.getWidth() > bitmap.getHeight()) {
+      // width > height --> width = maxLength, height scale with aspect ratio
+      aspectRatio = bitmap.getWidth() / (float) bitmap.getHeight();
+      finalWidth = maxLength;
+      finalHeight = Math.round(maxLength / aspectRatio);
+    } else {
+      // height >= width --> height = maxLength, width scale with aspect ratio
+      aspectRatio = bitmap.getHeight() / (float) bitmap.getWidth();
+      finalHeight = maxLength;
+      finalWidth = Math.round(maxLength / aspectRatio);
+    }
+
+    return Bitmap.createScaledBitmap(bitmap, finalWidth, finalHeight, false);
+  }
+}

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/ETLogging.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+package com.example.executorchllamademo;
+
+import android.app.Application;
+import android.util.Log;
+import java.util.ArrayList;
+
+public class ETLogging extends Application {
+  private static ETLogging singleton;
+
+  private ArrayList<AppLog> logs;
+  private DemoSharedPreferences mDemoSharedPreferences;
+
+  @Override
+  public void onCreate() {
+    super.onCreate();
+    singleton = this;
+    mDemoSharedPreferences = new DemoSharedPreferences(this.getApplicationContext());
+    logs = mDemoSharedPreferences.getSavedLogs();
+    if (logs == null) { // We don't have existing sharedPreference stored
+      logs = new ArrayList<>();
+    }
+  }
+
+  public static ETLogging getInstance() {
+    return singleton;
+  }
+
+  public void log(String message) {
+    AppLog appLog = new AppLog(message);
+    logs.add(appLog);
+    Log.d("ETLogging", appLog.getMessage());
+  }
+
+  public ArrayList<AppLog> getLogs() {
+    return logs;
+  }
+
+  public void clearLogs() {
+    logs.clear();
+    mDemoSharedPreferences.removeExistingLogs();
+  }
+
+  public void saveLogs() {
+    mDemoSharedPreferences.saveLogs();
+  }
+}