Skip to content

Commit 250b679

Browse files
ahmedsabieahmedsabie
authored
Add 3D keypoints to TFJS hand detection (#850)
* Add 3D keypoints to TFJS hand detection * Fix CSS typo * Add maxNumHands controller
1 parent f8a7f92 commit 250b679

File tree

7 files changed

+140
-44
lines changed

7 files changed

+140
-44
lines changed

‎hand-detection/demos/live_video/index.html‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,13 @@
2727
position: relative;
2828
margin: 0;
2929
}
30-
#canvas-wrapper,
31-
#scatter-gl-container {
30+
#canvas-wrapper {
3231
position: relative;
3332
}
33+
#scatter-gl-container-left, #scatter-gl-container-right {
34+
position: relative;
35+
float: left;
36+
}
3437
</style>
3538
</head>
3639
<body>
@@ -48,7 +51,9 @@
4851
">
4952
</video>
5053
</div>
51-
<div id="scatter-gl-container"></div>
54+
<div id="scatter-gl-container-left"></div>
55+
</div>
56+
<div id="scatter-gl-container-right"></div>
5257
</div>
5358
</div>
5459
</body>

‎hand-detection/demos/live_video/src/camera.js‎

Lines changed: 61 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {isMobile} from './util';
2121

2222
// These anchor points allow the hand pointcloud to resize according to its
2323
// position in the input.
24-
const ANCHOR_POINTS = [[0, 0, 0], [0, 1, 0], [-1, 0, 0], [-1, -1, 0]];
24+
const ANCHOR_POINTS = [[0, 0, 0], [0, 0.1, 0], [-0.1, 0, 0], [-0.1, -0.1, 0]];
2525

2626
const fingerLookupIndices = {
2727
thumb: [0, 1, 2, 3, 4],
@@ -39,18 +39,27 @@ const connections = [
3939
[0, 17], [17, 18],[18, 19], [19,20]
4040
];
4141

42+
function createScatterGLContext(selectors) {
43+
const scatterGLEl = document.querySelector(selectors);
44+
return {
45+
scatterGLEl,
46+
scatterGL: new scatter.ScatterGL(scatterGLEl, {
47+
'rotateOnStart': true,
48+
'selectEnabled': false,
49+
'styles': {polyline: {defaultOpacity: 1, deselectedOpacity: 1}}
50+
}),
51+
scatterGLHasInitialized: false,
52+
};
53+
}
54+
55+
const scatterGLCtxtLeftHand = createScatterGLContext('#scatter-gl-container-left');
56+
const scatterGLCtxtRightHand = createScatterGLContext('#scatter-gl-container-right');
57+
4258
export class Camera {
4359
constructor() {
4460
this.video = document.getElementById('video');
4561
this.canvas = document.getElementById('output');
4662
this.ctx = this.canvas.getContext('2d');
47-
this.scatterGLEl = document.querySelector('#scatter-gl-container');
48-
this.scatterGL = new scatter.ScatterGL(this.scatterGLEl, {
49-
'rotateOnStart': true,
50-
'selectEnabled': false,
51-
'styles': {polyline: {defaultOpacity: 1, deselectedOpacity: 1}}
52-
});
53-
this.scatterGLHasInitialized = false;
5463
}
5564

5665
/**
@@ -108,12 +117,14 @@ export class Camera {
108117
camera.ctx.translate(camera.video.videoWidth, 0);
109118
camera.ctx.scale(-1, 1);
110119

111-
camera.scatterGLEl.style =
112-
`width: ${videoWidth}px; height: ${videoHeight}px;`;
113-
camera.scatterGL.resize();
120+
for (const ctxt of [scatterGLCtxtLeftHand, scatterGLCtxtRightHand]) {
121+
ctxt.scatterGLEl.style =
122+
`width: ${videoWidth / 2}px; height: ${videoHeight / 2}px;`;
123+
ctxt.scatterGL.resize();
114124

115-
camera.scatterGLEl.style.display =
116-
params.STATE.modelConfig.render3D ? 'inline-block' : 'none';
125+
ctxt.scatterGLEl.style.display =
126+
params.STATE.modelConfig.render3D ? 'inline-block' : 'none';
127+
}
117128

118129
return camera;
119130
}
@@ -132,31 +143,53 @@ export class Camera {
132143
* @param hands A list of hands to render.
133144
*/
134145
drawResults(hands) {
135-
for (const hand of hands) {
136-
this.drawResult(hand);
146+
// Sort by right to left hands.
147+
hands.sort((hand1, hand2) => {
148+
if (hand1.handedness < hand2.handedness) return 1;
149+
if (hand1.handedness > hand2.handedness) return -1;
150+
return 0;
151+
});
152+
153+
// Pad hands to clear empty scatter GL plots.
154+
while (hands.length < 2) hands.push({});
155+
156+
for (let i = 0; i < hands.length; ++i) {
157+
// Third hand and onwards scatterGL context is set to null since we
158+
// don't render them.
159+
const ctxt = [scatterGLCtxtLeftHand, scatterGLCtxtRightHand][i];
160+
this.drawResult(hands[i], ctxt);
137161
}
138162
}
139163

140164
/**
141165
* Draw the keypoints on the video.
142166
* @param hand A hand with keypoints to render.
167+
* @param ctxt Scatter GL context to render 3D keypoints to.
143168
*/
144-
drawResult(hand) {
169+
drawResult(hand, ctxt) {
145170
if (hand.keypoints != null) {
146-
this.drawKeypoints(hand.keypoints);
171+
this.drawKeypoints(hand.keypoints, hand.handedness);
172+
}
173+
// Don't render 3D hands after first two.
174+
if (ctxt == null) {
175+
return;
147176
}
148177
if (hand.keypoints3D != null && params.STATE.modelConfig.render3D) {
149-
this.drawKeypoints3D(hand.keypoints3D);
178+
this.drawKeypoints3D(hand.keypoints3D, hand.handedness, ctxt);
179+
} else {
180+
// Clear scatter plot.
181+
this.drawKeypoints3D([], '', ctxt);
150182
}
151183
}
152184

153185
/**
154186
* Draw the keypoints on the video.
155187
* @param keypoints A list of keypoints.
188+
* @param handedness Label of hand (either Left or Right).
156189
*/
157-
drawKeypoints(keypoints) {
190+
drawKeypoints(keypoints, handedness) {
158191
const keypointsArray = keypoints;
159-
this.ctx.fillStyle = 'Red';
192+
this.ctx.fillStyle = handedness === 'Left' ? 'Red' : 'Blue';
160193
this.ctx.strokeStyle = 'White';
161194
this.ctx.lineWidth = params.DEFAULT_LINE_WIDTH;
162195

@@ -194,29 +227,29 @@ export class Camera {
194227
this.ctx.fill();
195228
}
196229

197-
drawKeypoints3D(keypoints) {
230+
drawKeypoints3D(keypoints, handedness, ctxt) {
198231
const scoreThreshold = params.STATE.modelConfig.scoreThreshold || 0;
199232
const pointsData =
200233
keypoints.map(keypoint => ([-keypoint.x, -keypoint.y, -keypoint.z]));
201234

202235
const dataset =
203236
new scatter.ScatterGL.Dataset([...pointsData, ...ANCHOR_POINTS]);
204237

205-
this.scatterGL.setPointColorer((i) => {
238+
ctxt.scatterGL.setPointColorer((i) => {
206239
if (keypoints[i] == null || keypoints[i].score < scoreThreshold) {
207240
// hide anchor points and low-confident points.
208241
return '#ffffff';
209242
}
210-
return '#ff0000' /* Red */;
243+
return handedness === 'Left' ? '#ff0000' : '#0000ff';
211244
});
212245

213-
if (!this.scatterGLHasInitialized) {
214-
this.scatterGL.render(dataset);
246+
if (!ctxt.scatterGLHasInitialized) {
247+
ctxt.scatterGL.render(dataset);
215248
} else {
216-
this.scatterGL.updateDataset(dataset);
249+
ctxt.scatterGL.updateDataset(dataset);
217250
}
218251
const sequences = connections.map(pair => ({indices: pair}));
219-
this.scatterGL.setSequences(sequences);
220-
this.scatterGLHasInitialized = true;
252+
ctxt.scatterGL.setSequences(sequences);
253+
ctxt.scatterGLHasInitialized = true;
221254
}
222255
}

‎hand-detection/demos/live_video/src/index.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ async function createDetector() {
4545
return handdetection.createDetector(STATE.model, {
4646
runtime,
4747
modelType: STATE.modelConfig.type,
48-
maxHands: STATE.modelConfig.maxHands,
48+
maxHands: STATE.modelConfig.maxNumHands,
4949
solutionPath: `https://cdn.jsdelivr.net/npm/@mediapipe/hands@${mpHands.VERSION}`
5050
});
5151
} else if (runtime === 'tfjs') {
5252
return handdetection.createDetector(STATE.model, {
5353
runtime,
5454
modelType: STATE.modelConfig.type,
55-
maxHands: STATE.modelConfig.maxHands
55+
maxHands: STATE.modelConfig.maxNumHands
5656
});
5757
}
5858
}

‎hand-detection/demos/live_video/src/option_panel.js‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ export async function setupDatGui(urlParams) {
4949

5050
const model = urlParams.get('model');
5151
let type = urlParams.get('type');
52+
let maxNumHands = urlParams.get('maxNumHands');
5253

5354
switch (model) {
5455
case 'mediapipe_hands':
@@ -57,6 +58,10 @@ export async function setupDatGui(urlParams) {
5758
// Nulify invalid value.
5859
type = null;
5960
}
61+
if (maxNumHands == null || maxNumHands < 1 ) {
62+
// Nulify invalid value.
63+
maxNumHands = 2;
64+
}
6065
break;
6166
default:
6267
alert(`${urlParams.get('model')}`);
@@ -72,7 +77,7 @@ export async function setupDatGui(urlParams) {
7277
showBackendConfigs(backendFolder);
7378
});
7479

75-
showModelConfigs(modelFolder, type);
80+
showModelConfigs(modelFolder, type, maxNumHands);
7681

7782
modelFolder.open();
7883

@@ -106,7 +111,7 @@ async function showBackendConfigs(folderController) {
106111
await showFlagSettings(folderController, params.STATE.backend);
107112
}
108113

109-
function showModelConfigs(folderController, type) {
114+
function showModelConfigs(folderController, type, maxNumHands) {
110115
// Clean up model configs for the previous model.
111116
// The first constroller under the `folderController` is the model
112117
// selection.
@@ -119,7 +124,7 @@ function showModelConfigs(folderController, type) {
119124

120125
switch (params.STATE.model) {
121126
case handdetection.SupportedModels.MediaPipeHands:
122-
addMediaPipeHandsControllers(folderController, type);
127+
addMediaPipeHandsControllers(folderController, type, maxNumHands);
123128
break;
124129
default:
125130
alert(`Model ${params.STATE.model} is not supported.`);
@@ -128,9 +133,10 @@ function showModelConfigs(folderController, type) {
128133

129134
// The MediaPipeHands model config folder contains options for MediaPipeHands config
130135
// settings.
131-
function addMediaPipeHandsControllers(modelConfigFolder, type) {
136+
function addMediaPipeHandsControllers(modelConfigFolder, type, maxNumHands) {
132137
params.STATE.modelConfig = {...params.MEDIAPIPE_HANDS_CONFIG};
133138
params.STATE.modelConfig.type = type != null ? type : 'full';
139+
params.STATE.modelConfig.maxNumHands = maxNumHands != null ? maxNumHands : 2;
134140

135141
const typeController = modelConfigFolder.add(
136142
params.STATE.modelConfig, 'type', ['lite', 'full']);
@@ -140,10 +146,20 @@ function addMediaPipeHandsControllers(modelConfigFolder, type) {
140146
params.STATE.isModelChanged = true;
141147
});
142148

149+
const maxNumHandsController = modelConfigFolder.add(
150+
params.STATE.modelConfig, 'maxNumHands', 1, 10).step(1);
151+
maxNumHandsController.onChange(_ => {
152+
// Set isModelChanged to true, so that we don't render any result during
153+
// changing models.
154+
params.STATE.isModelChanged = true;
155+
});
156+
143157
const render3DController =
144158
modelConfigFolder.add(params.STATE.modelConfig, 'render3D');
145159
render3DController.onChange(render3D => {
146-
document.querySelector('#scatter-gl-container').style.display =
160+
document.querySelector('#scatter-gl-container-left').style.display =
161+
render3D ? 'inline-block' : 'none';
162+
document.querySelector('#scatter-gl-container-right').style.display =
147163
render3D ? 'inline-block' : 'none';
148164
});
149165
}

‎hand-detection/demos/live_video/src/params.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ export const STATE = {
3131
modelConfig: {}
3232
};
3333
export const MEDIAPIPE_HANDS_CONFIG = {
34-
maxHands: 2,
3534
type: 'full',
3635
render3D: true
3736
};

‎hand-detection/src/tfjs/constants.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,12 @@ export const MPHANDS_TENSORS_TO_LANDMARKS_CONFIG: TensorsToLandmarksConfig = {
112112
flipHorizontally: false,
113113
flipVertically: false
114114
};
115+
export const MPHANDS_TENSORS_TO_WORLD_LANDMARKS_CONFIG:
116+
TensorsToLandmarksConfig = {
117+
numLandmarks: 21,
118+
inputImageWidth: 1,
119+
inputImageHeight: 1,
120+
visibilityActivation: 'none',
121+
flipHorizontally: false,
122+
flipVertically: false
123+
};

0 commit comments

Comments
 (0)