功能介绍

*funpose是一个可以捕捉全身的动作，面部表情、手势等追踪库，根据真实的人物肢体、面部、手的动作来驱动虚拟形象。

*对于人体骨骼关节点进行检测，定位人体的姿态。

*可以驱动 Live2D 形象，又可以驱动 3D VRM 形象。

*接受Facemesh、Blazepose、Handpose和Obsolental等模型输出

Install

NPM

npm install funpose

import * as funpose from "funpose";

// 导入所需类

import { Face, Pose, Hand } from "funpose";

方法

FUNPOSE由3个类组成，面部、姿势和手势计算。接受Facemesh、Blazepose、Handpose和Obsolental等模型输出。

// 向量数组（468或478带有虹膜跟踪）
funpose.Face.solve(facelandmarkArray, {
    runtime: "tfjs", // mediapipe或tfjs
    video: HTMLVideoElement,
    imageSize: { height: 0, width: 0 },
    smoothBlink: false, // 眨眼延迟
    blinkSettings: [0.25, 0.75], //调整闪烁灵敏度
});

// 使用姿势关键点和3D姿势关键点的阵列（33）
funpose.Pose.solve(poseWorld3DArray, poseLandmarkArray, {
    runtime: "tfjs", // mediapipe或tfjs
    video: HTMLVideoElement,
    imageSize: { height: 0, width: 0 },
    enableLegs: true,
});

// 使用手部地标向量的阵列（21）；指定“右侧”或“左侧”
funpose.Hand.solve(handLandmarkArray, "Right");

// 使用导出类
Face.solve(facelandmarkArray);
Pose.solve(poseWorld3DArray, poseLandmarkArray);
Hand.solve(handLandmarkArray, "Right");

其他方法

// 形状和头部旋转，稳定左/右眨眼延迟+眨眼
funpose.Face.stabilizeBlink(
    { r: 0, l: 1 }, // 左眼和右眼混合形状值
    headRotationY, // 头部旋转弧度
    {
        noWink = false, // 禁止眨眼 
        maxRot = 0.5 // 插值最大头部旋转弧度
    });


funpose.Vector();

例子

根据你使用的姿势和人脸检测模型，使用Mediapipe一起使用

import * as funpose from 'funpose'
import '@mediapipe/holistic/holistic';
import '@mediapipe/camera_utils/camera_utils';

let holistic = new Holistic({locateFile: (file) => {
    return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
}});

holistic.onResults(results=>{
    // 预测结果
    // landmark名称可能会根据TFJS/Mediapipe模型版本而变化
    let facelm = results.faceLandmarks;
    let poselm = results.poseLandmarks;
    let poselm3D = results.ea;
    let rightHandlm = results.rightHandLandmarks;
    let leftHandlm = results.leftHandLandmarks;

    let faceRig = funpose.Face.solve(facelm,{runtime:'mediapipe',video:HTMLVideoElement})
    let poseRig = funpose.Pose.solve(poselm3d,poselm,{runtime:'mediapipe',video:HTMLVideoElement})
    let rightHandRig = funpose.Hand.solve(rightHandlm,"Right")
    let leftHandRig = funpose.Hand.solve(leftHandlm,"Left")

    };
});

// 使用Mediapipe的网络摄像头utils将视频发送到每一帧
const camera = new Camera(HTMLVideoElement, {
  onFrame: async () => {
    await holistic.send({image: HTMLVideoElement});
  },
  width: 640,
  height: 480
});
camera.start();

由于Mediapipe和Tensorflow.js有所不同

建议指定您使用的运行时版本以及视频输入/图像大小作为参考。

funpose.Pose.solve(poselm3D,poselm,{
    runtime:'tfjs', // 默认是 'mediapipe'
    video: HTMLVideoElement,// 设置视频或图片大小
    imageSize:{
        width: 640,
        height: 480,
    };
})

funpose.Face.solve(facelm,{
    runtime:'mediapipe', // 默认是 'tfjs'
    video: HTMLVideoElement, // 设置视频或图片大小
    imageSize:{
        width: 640,
        height: 480,
    };
})

输出

funpose解算器

// funpose.Face.solve()
// 头部旋转弧度
// 度和标准化旋转也可用
{
    eye: {l: 1,r: 1},
    mouth: {
        x: 0,
        y: 0,
        shape: {A:0, E:0, I:0, O:0, U:0}
    },
    head: {
        x: 0,
        y: 0,
        z: 0,
        width: 0.3,
        height: 0.6,
        position: {x: 0.5, y: 0.5, z: 0}
    },
    brow: 0,
    pupil: {x: 0, y: 0}
}

// funpose.Pose.solve()
// 关节旋转弧度，腿部计算器是WIP
{
    RightUpperArm: {x: 0, y: 0, z: -1.25},
    LeftUpperArm: {x: 0, y: 0, z: 1.25},
    RightLowerArm: {x: 0, y: 0, z: 0},
    LeftLowerArm: {x: 0, y: 0, z: 0},
    LeftUpperLeg: {x: 0, y: 0, z: 0},
    RightUpperLeg: {x: 0, y: 0, z: 0},
    RightLowerLeg: {x: 0, y: 0, z: 0},
    LeftLowerLeg: {x: 0, y: 0, z: 0},
    LeftHand: {x: 0, y: 0, z: 0},
    RightHand: {x: 0, y: 0, z: 0},
    Spine: {x: 0, y: 0, z: 0},
    Hips: {
        worldPosition: {x: 0, y: 0, z: 0},
        position: {x: 0, y: 0, z: 0},
        rotation: {x: 0, y: 0, z: 0},
    }
}

// funpose.Hand.solve()
// 弧度为单位的关节旋转
// 只有手腕和拇指有3个自由度
// 其他手指关节仅在Z轴上移动
{
    RightWrist: {x: -0.13, y: -0.07, z: -1.04},
    RightRingProximal: {x: 0, y: 0, z: -0.13},
    RightRingIntermediate: {x: 0, y: 0, z: -0.4},
    RightRingDistal: {x: 0, y: 0, z: -0.04},
    RightIndexProximal: {x: 0, y: 0, z: -0.24},
    RightIndexIntermediate: {x: 0, y: 0, z: -0.25},
    RightIndexDistal: {x: 0, y: 0, z: -0.06},
    RightMiddleProximal: {x: 0, y: 0, z: -0.09},
    RightMiddleIntermediate: {x: 0, y: 0, z: -0.44},
    RightMiddleDistal: {x: 0, y: 0, z: -0.06},
    RightThumbProximal: {x: -0.23, y: -0.33, z: -0.12},
    RightThumbIntermediate: {x: -0.2, y: -0.19, z: -0.01},
    RightThumbDistal: {x: -0.2, y: 0.002, z: 0.15},
    RightLittleProximal: {x: 0, y: 0, z: -0.09},
    RightLittleIntermediate: {x: 0, y: 0, z: -0.22},
    RightLittleDistal: {x: 0, y: 0, z: -0.1}
}