[pose-detection] Movenet multipose demo (#775)

* Add MoveNet multi-pose option to live demo

* Address reviewer's comments

Author: ardoerlemans

pose-detection/demos/live_video/src/index.js

@@ -53,19 +53,25 @@ async function createDetector() {
         return posedetection.createDetector(STATE.model, {
           runtime,
           modelType: STATE.modelConfig.type,
-          solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]'
+          solutionPath:
+              'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]'
         });
       } else if (runtime === 'tfjs') {
         return posedetection.createDetector(
             STATE.model, {runtime, modelType: STATE.modelConfig.type});
       }
     case posedetection.SupportedModels.MoveNet:
-      const modelType = STATE.modelConfig.type == 'lightning' ?
-          posedetection.movenet.modelType.SINGLEPOSE_LIGHTNING :
-          posedetection.movenet.modelType.SINGLEPOSE_THUNDER;
+      let modelType;
+      if (STATE.modelConfig.type == 'lightning') {
+        modelType = posedetection.movenet.modelType.SINGLEPOSE_LIGHTNING;
+      } else if (STATE.modelConfig.type == 'thunder') {
+        modelType = posedetection.movenet.modelType.SINGLEPOSE_THUNDER;
+      } else if (STATE.modelConfig.type == 'multipose') {
+        modelType = posedetection.movenet.modelType.MULTIPOSE;
+      }
       if (STATE.modelConfig.customModel !== '') {
-        return posedetection.createDetector(STATE.model, {modelType,
-          modelUrl: STATE.modelConfig.customModel});
+        return posedetection.createDetector(
+            STATE.model, {modelType, modelUrl: STATE.modelConfig.customModel});
       }
       return posedetection.createDetector(STATE.model, {modelType});
   }
@@ -83,13 +89,21 @@ async function checkGuiUpdate() {
 
     window.cancelAnimationFrame(rafId);
 
-    detector.dispose();
+    if (detector != null) {
+      detector.dispose();
+    }
 
     if (STATE.isFlagChanged || STATE.isBackendChanged) {
       await setBackendAndEnvFlags(STATE.flags, STATE.backend);
     }
 
-    detector = await createDetector(STATE.model);
+    try {
+      detector = await createDetector(STATE.model);
+    } catch (error) {
+      detector = null;
+      alert(error);
+    }
+
     STATE.isFlagChanged = false;
     STATE.isBackendChanged = false;
     STATE.isModelChanged = false;
@@ -125,21 +139,35 @@ async function renderResult() {
     });
   }
 
-  // FPS only counts the time it takes to finish estimatePoses.
-  beginEstimatePosesStats();
-
-  const poses = await detector.estimatePoses(
-      camera.video,
-      {maxPoses: STATE.modelConfig.maxPoses, flipHorizontal: false});
+  let poses = null;
+
+  // Detector can be null if initialization failed (for example when loading
+  // from a URL that does not exist).
+  if (detector != null) {
+    // FPS only counts the time it takes to finish estimatePoses.
+    beginEstimatePosesStats();
+
+    // Detectors can throw errors, for example when using custom URLs that
+    // contain a model that doesn't provide the expected output.
+    try {
+      poses = await detector.estimatePoses(
+          camera.video,
+          {maxPoses: STATE.modelConfig.maxPoses, flipHorizontal: false});
+    } catch (error) {
+      detector.dispose();
+      detector = null;
+      alert(error);
+    }
 
-  endEstimatePosesStats();
+    endEstimatePosesStats();
+  }
 
   camera.drawCtx();
 
   // The null check makes sure the UI is not in the middle of changing to a
   // different model. If during model change, the result is from an old model,
   // which shouldn't be rendered.
-  if (poses.length > 0 && !STATE.isModelChanged) {
+  if (poses && poses.length > 0 && !STATE.isModelChanged) {
     camera.drawResults(poses);
   }
 }

pose-detection/demos/live_video/src/option_panel.js

@@ -56,7 +56,7 @@ export async function setupDatGui(urlParams) {
       break;
     case 'movenet':
       params.STATE.model = posedetection.SupportedModels.MoveNet;
-      if (type !== 'lightning' && type !== 'thunder') {
+      if (type !== 'lightning' && type !== 'thunder' && type !== 'multipose') {
         // Nulify invalid value.
         type = null;
       }
@@ -158,14 +158,15 @@ function addMoveNetControllers(modelConfigFolder, type) {
   params.STATE.modelConfig.type = type != null ? type : 'lightning';
 
   const typeController = modelConfigFolder.add(
-      params.STATE.modelConfig, 'type', ['lightning', 'thunder']);
+      params.STATE.modelConfig, 'type', ['lightning', 'thunder', 'multipose']);
   typeController.onChange(_ => {
     // Set isModelChanged to true, so that we don't render any result during
     // changing models.
     params.STATE.isModelChanged = true;
   });
 
-  const customModelController = modelConfigFolder.add(params.STATE.modelConfig, 'customModel');
+  const customModelController =
+      modelConfigFolder.add(params.STATE.modelConfig, 'customModel');
   customModelController.onFinishChange(_ => {
     params.STATE.isModelChanged = true;
   })