Real-time face mesh point cloud with Three.JS, Tensorflow.js and Typescript

  1. Get Three Js setup
  2. Generate video data from webcam
  3. Create a face mesh detector
  4. Create empty point cloud
  5. Feed the tracking information to the point cloud
  1. Get Three.js setup:
getSetUp(){
return {
camera: this.camera,
scene: this.scene,
renderer : this.renderer,
sizes: this.sizes,
}
}
applyOrbitControls(){
const controls = new OrbitControls(
this.camera, this.renderer.domElement!
)
controls.enableDamping = true
return ()=> controls.update();
}
private init(){
navigator.mediaDevices.getUserMedia(this.videoConstraints)
.then((mediaStream)=>{
this.videoTarget.srcObject = mediaStream
this.videoTarget.onloadedmetadata = () => this.onLoadMetadata()
}
).catch(function (err) {
alert(err.name + ': ' + err.message)
}
)
}
private onLoadMetadata(){
this.videoTarget.setAttribute('autoplay', 'true')
this.videoTarget.setAttribute('playsinline', 'true')
this.videoTarget.play()
this.onReceivingData()
}
updateFromWebCam(){
this.canvasCtx.drawImage(
this.webcamVideo.videoTarget,
0,
0,
this.canvas.width,
this.canvas.height
)
}
npm add @tensorflow/tfjs-core, @tensorflow/tfjs-converter
npm add @tensorflow/tfjs-backend-webgl
npm add @tensorflow-models/face-detection
npm add @tensorflow-models/face-landmarks-detection
import '@mediapipe/face_mesh'
import '@tensorflow/tfjs-core'
import '@tensorflow/tfjs-backend-webgl'
import * as faceLandmarksDetection from '@tensorflow-models/face-landmarks-detection'
this.model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
this.detectorConfig = {
runtime: 'mediapipe',
refineLandmarks: true,
solutionPath: 'https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh',
}
async detectFace(source){
const data = await this.detector!.estimateFaces(source)
const keypoints = (data as FaceLandmark[])[0]?.keypoints
if(keypoints) return keypoints;
return [];
}
faceMeshDetector.detectFace(this.webcamCanvas.canvas)
[
{
box: {
xMin: 304.6476503248806,
xMax: 502.5079975897382,
yMin: 102.16298762367356,
yMax: 349.035215984403,
width: 197.86034726485758,
height: 246.87222836072945
},
keypoints: [
{x: 406.53152857172876, y: 256.8054528661723, z: 10.2, name:
"lips"},
{x: 406.544237446397, y: 230.06933367750395, z: 8},
...
],
}
]
export default class PointCloud {
bufferGeometry: THREE.BufferGeometry;
material: THREE.PointsMaterial;
cloud: THREE.Points<THREE.BufferGeometry, THREE.PointsMaterial>;

constructor() {
this.bufferGeometry = new THREE.BufferGeometry();
this.material = new THREE.PointsMaterial({
color: 0x888888,
size: 0.0151,
sizeAttenuation: true,
});
this.cloud = new THREE.Points(this.bufferGeometry, this.material);
}
updateProperty(attribute: THREE.BufferAttribute, name: string){
this.bufferGeometry.setAttribute(
name,
attribute
);
this.bufferGeometry.attributes[name].needsUpdate = true;
}
}
constructor() {
this.threeSetUp = new ThreeSetUp()
this.setUpElements = this.threeSetUp.getSetUp()
this.webcamCanvas = new WebcamCanvas();
this.faceMeshDetector = new faceLandMark()
this.pointCloud = new PointCloud()
}
async bindFaceDataToPointCloud(){
const keypoints = await
this.faceMeshDetector.detectFace(this.webcamCanvas.canvas)
const flatData = flattenFacialLandMarkArray(keypoints)
const facePositions = createBufferAttribute(flatData)
this.pointCloud.updateProperty(facePositions, 'position')
}
keypoints: [
{x: 0.542, y: 0.967, z: 0.037},
...
]
number[] or [0.542, 0.967, 0.037, .....]
a picture explaining html canvas coordinates system starting from top left corner
3D space coordinates system with xyz and origin in the center
function flattenFacialLandMarkArray(data: vector[]){
let array: number[] = [];
data.forEach((el)=>{
el.x = mapRangetoRange(500 / videoAspectRatio, el.x,
screenRange.height) - 1

el.y = mapRangetoRange(500 / videoAspectRatio, el.y,
screenRange.height, true)+1
el.z = (el.z / 100 * -1) + 0.5;

array = [
...array,
...Object.values(el),
]
})
return array.filter((el)=> typeof el === 'number');
}
function mapRangetoRange(from: number, point: number, range: range, invert: boolean = false): number{
let pointMagnitude: number = point/from;
if(invert) pointMagnitude = 1-pointMagnitude;
const targetMagnitude = range.to - range.from;
const pointInRange = targetMagnitude * pointMagnitude +
range.from;

return pointInRange
}
async initWork() {
const { camera, scene, renderer } = this.setUpElements
camera.position.z = 3
camera.position.y = 1
camera.lookAt(0,0,0)
const orbitControlsUpdate = this.threeSetUp.applyOrbitControls()
const gridHelper = new THREE.GridHelper(10, 10)
scene.add(gridHelper)
scene.add(this.pointCloud.cloud)

await this.faceMeshDetector.loadDetector()

const animate = () => {
requestAnimationFrame(animate)
if (this.webcamCanvas.receivingStreem){
this.bindFaceDataToPointCloud()
}
this.webcamCanvas.updateFromWebCam()
orbitControlsUpdate()
renderer.render(scene, camera)
}

animate()
}

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store