[pose-detection]Update BlazePose OneEuro filter config. (#740)

FEATURE

Author: lina128

pose-detection/src/blazepose_mediapipe/mediapipe_test.ts

@@ -30,7 +30,9 @@ const MEDIAPIPE_MODEL_CONFIG: BlazePoseMediaPipeModelConfig = {
 };
 
 const EPSILON_IMAGE = 10;
-const EPSILON_VIDEO = 18;
+// TODO(lina128): Reduce video tolerance once MP Web Solution publishes new
+// version.
+const EPSILON_VIDEO = 42;
 // ref:
 // https://github.com/google/mediapipe/blob/7c331ad58b2cca0dca468e342768900041d65adc/mediapipe/python/solutions/pose_test.py#L31-L51
 const EXPECTED_LANDMARKS = [

pose-detection/src/blazepose_tfjs/blazepose_test.ts

@@ -25,7 +25,7 @@ import * as poseDetection from '../index';
 import {getXYPerFrame, KARMA_SERVER, loadImage, loadVideo} from '../test_util';
 
 const EPSILON_IMAGE = 10;
-const EPSILON_VIDEO = 15;
+const EPSILON_VIDEO = 18;
 
 // ref:
 // https://github.com/google/mediapipe/blob/7c331ad58b2cca0dca468e342768900041d65adc/mediapipe/python/solutions/pose_test.py#L31-L51

pose-detection/src/blazepose_tfjs/constants.ts

@@ -109,9 +109,9 @@ export const BLAZEPOSE_VISIBILITY_SMOOTHING_CONFIG = {
 export const BLAZEPOSE_LANDMARKS_SMOOTHING_CONFIG_ACTUAL = {
   oneEuroFilter: {
     frequency: 30,
-    minCutOff: 0.1,  // minCutOff 0.1 results into ~0.02 alpha in landmark EMA
-                     // filter when landmark is static.
-    beta: 40,  // beta 40 in combination with minCutOff 0.1 results into ~0.8
+    minCutOff: 0.05,  // minCutOff 0.05 results into ~0.01 alpha in landmark EMA
+                      // filter when landmark is static.
+    beta: 80,  // beta 80 in combination with minCutOff 0.05 results into ~0.94
                // alpha in landmark EMA filter when landmark is moving fast.
     derivateCutOff: 1.0,  // derivativeCutOff 1.0 results into ~0.17 alpha in
                           // landmark velocity EMA filter.,
@@ -126,8 +126,9 @@ export const BLAZEPOSE_LANDMARKS_SMOOTHING_CONFIG_AUXILIARY = {
     frequency: 30,
     minCutOff: 0.01,  // minCutOff 0.01 results into ~0.002 alpha in landmark
                       // EMA filter when landmark is static.
-    beta: 1.0,  // beta 1.0 in combination with minCutOff 0.01 results into ~0.2
-                // alpha in landmark EMA filter when landmark is moving fast.
+    beta: 10.0,  // beta 10.0 in combination with minCutOff 0.01 results into
+                 // ~0.68 alpha in landmark EMA filter when landmark is moving
+                 // fast.
     derivateCutOff: 1.0,  // derivateCutOff 1.0 results into ~0.17 alpha in
                           // landmark velocity EMA filter.
     minAllowedObjectScale: 1e-6

pose-detection/src/blazepose_tfjs/detector.ts

@@ -439,6 +439,14 @@ class BlazePoseTfjsDetector implements PoseDetector {
       actualLandmarksFiltered = actualLandmarks;
       auxiliaryLandmarksFiltered = auxiliaryLandmarks;
     } else {
+      const auxDetection = landmarksToDetection(auxiliaryLandmarks);
+      const objectScaleROI =
+          calculateAlignmentPointsRects(auxDetection, imageSize, {
+            rotationVectorEndKeypointIndex: 0,
+            rotationVectorStartKeypointIndex: 1,
+            rotationVectorTargetAngleDegree: 90
+          });
+
       // Smoothes pose landmark visibilities to reduce jitter.
       if (this.visibilitySmoothingFilterActual == null) {
         this.visibilitySmoothingFilterActual = new LowPassVisibilityFilter(
@@ -461,15 +469,15 @@ class BlazePoseTfjsDetector implements PoseDetector {
       }
       actualLandmarksFiltered = this.landmarksSmoothingFilterActual.apply(
           actualLandmarksFiltered, this.timestamp, imageSize,
-          true /* normalized */);
+          true /* normalized */, objectScaleROI);
 
       if (this.landmarksSmoothingFilterAuxiliary == null) {
         this.landmarksSmoothingFilterAuxiliary = new KeypointsSmoothingFilter(
             constants.BLAZEPOSE_LANDMARKS_SMOOTHING_CONFIG_AUXILIARY);
       }
       auxiliaryLandmarksFiltered = this.landmarksSmoothingFilterAuxiliary.apply(
           auxiliaryLandmarksFiltered, this.timestamp, imageSize,
-          true /* normalized */);
+          true /* normalized */, objectScaleROI);
     }
 
     return {actualLandmarksFiltered, auxiliaryLandmarksFiltered};

pose-detection/src/calculators/get_object_scale.ts

@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright 2021 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+import {ImageSize} from './interfaces/common_interfaces';
+import {Rect} from './interfaces/shape_interfaces';
+
+/**
+ * Estimate object scale to allow filter work similarly on nearer or futher
+ * objects.
+ * @param roi Normalized rectangle.
+ * @param imageSize An object that contains width and height.
+ * @returns A number representing the object scale.
+ */
+export function getObjectScale(roi: Rect, imageSize: ImageSize): number {
+  const objectWidth = roi.width * imageSize.width;
+  const objectHeight = roi.height * imageSize.height;
+
+  return (objectWidth + objectHeight) / 2;
+}

pose-detection/src/calculators/interfaces/common_interfaces.ts

@@ -39,6 +39,7 @@ export interface WindowElement {
 }
 
 export interface KeypointsFilter {
-  apply(landmarks: Keypoint[], microSeconds: number): Keypoint[];
+  apply(landmarks: Keypoint[], microSeconds: number, objectScale: number):
+      Keypoint[];
   reset(): void;
 }

pose-detection/src/calculators/keypoints_one_euro_filter.ts

@@ -20,7 +20,6 @@ import {Keypoint} from '../types';
 import {KeypointsFilter} from './interfaces/common_interfaces';
 import {OneEuroFilterConfig} from './interfaces/config_interfaces';
 import {OneEuroFilter} from './one_euro_filter';
-import {getObjectScale} from './velocity_filter_utils';
 
 /**
  * A stateful filter that smoothes keypoints values overtime.
@@ -38,7 +37,8 @@ export class KeypointsOneEuroFilter implements KeypointsFilter {
 
   constructor(private readonly config: OneEuroFilterConfig) {}
 
-  apply(keypoints: Keypoint[], microSeconds: number): Keypoint[] {
+  apply(keypoints: Keypoint[], microSeconds: number, objectScale: number):
+      Keypoint[] {
     if (keypoints == null) {
       this.reset();
       return null;
@@ -52,7 +52,6 @@ export class KeypointsOneEuroFilter implements KeypointsFilter {
     // returned as is.
     let valueScale = 1;
     if (this.config.minAllowedObjectScale != null) {
-      const objectScale = getObjectScale(keypoints);
       if (objectScale < this.config.minAllowedObjectScale) {
         return [...keypoints];
       }

pose-detection/src/calculators/keypoints_smoothing.ts

@@ -17,8 +17,10 @@
 
 import {Keypoint} from '../types';
 
+import {getObjectScale} from './get_object_scale';
 import {ImageSize, KeypointsFilter} from './interfaces/common_interfaces';
 import {KeypointsSmoothingConfig} from './interfaces/config_interfaces';
+import {Rect} from './interfaces/shape_interfaces';
 import {KeypointsOneEuroFilter} from './keypoints_one_euro_filter';
 import {keypointsToNormalizedKeypoints} from './keypoints_to_normalized_keypoints';
 import {KeypointsVelocityFilter} from './keypoints_velocity_filter';
@@ -53,20 +55,25 @@ export class KeypointsSmoothingFilter {
    *     normalized.
    * @param normalized Optional. Whether the keypoints are normalized. Default
    *     to false.
+   * @param objectScaleROI Optional. The auxiliary ROI to calculate object
+   *     scale. If not set, objectScale defaults to 1.
    */
   apply(
       keypoints: Keypoint[], timestamp: number, imageSize?: ImageSize,
-      normalized = false): Keypoint[] {
+      normalized = false, objectScaleROI?: Rect): Keypoint[] {
     if (keypoints == null) {
       this.keypointsFilter.reset();
       return null;
     }
 
+    const objectScale =
+        objectScaleROI != null ? getObjectScale(objectScaleROI, imageSize) : 1;
+
     const scaledKeypoints = normalized ?
         normalizedKeypointsToKeypoints(keypoints, imageSize) :
         keypoints;
     const scaledKeypointsFiltered =
-        this.keypointsFilter.apply(scaledKeypoints, timestamp);
+        this.keypointsFilter.apply(scaledKeypoints, timestamp, objectScale);
 
     return normalized ?
         keypointsToNormalizedKeypoints(scaledKeypointsFiltered, imageSize) :

pose-detection/src/calculators/keypoints_velocity_filter.ts

@@ -19,7 +19,6 @@ import {Keypoint} from '../types';
 import {KeypointsFilter} from './interfaces/common_interfaces';
 import {VelocityFilterConfig} from './interfaces/config_interfaces';
 import {RelativeVelocityFilter} from './relative_velocity_filter';
-import {getObjectScale} from './velocity_filter_utils';
 
 /**
  * A stateful filter that smoothes landmark values overtime.
@@ -37,7 +36,8 @@ export class KeypointsVelocityFilter implements KeypointsFilter {
 
   constructor(private readonly config: VelocityFilterConfig) {}
 
-  apply(keypoints: Keypoint[], microSeconds: number): Keypoint[] {
+  apply(keypoints: Keypoint[], microSeconds: number, objectScale: number):
+      Keypoint[] {
     if (keypoints == null) {
       this.reset();
       return null;
@@ -47,7 +47,6 @@ export class KeypointsVelocityFilter implements KeypointsFilter {
     // returned as is.
     let valueScale = 1;
     if (!this.config.disableValueScaling) {
-      const objectScale = getObjectScale(keypoints);
       if (objectScale < this.config.minAllowedObjectScale) {
         return [...keypoints];
       }