feat(three-mmd-b): ik

examples/playground/src/hooks/use-mmd-animations.ts

@@ -0,0 +1,91 @@
+import type { Grant } from '@moeru/three-mmd'
+import type { AnimationAction, AnimationClip, SkinnedMesh } from 'three'
+import type { IK } from 'three/examples/jsm/animation/CCDIKSolver.js'
+
+import { GrantSolver } from '@moeru/three-mmd'
+import { useFrame } from '@react-three/fiber'
+import { useEffect, useMemo, useRef } from 'react'
+import { AnimationMixer, Vector3 } from 'three'
+import { CCDIKSolver } from 'three/examples/jsm/animation/CCDIKSolver.js'
+
+import { processBones } from '../utils/process-bones'
+
+interface Api<T extends AnimationClip> {
+  actions: { [key in T['name']]: AnimationAction | null }
+  clips: AnimationClip[]
+  grantSolver: GrantSolver
+  ikSolver: CCDIKSolver
+  mixer: AnimationMixer
+  names: T['name'][]
+}
+
+export const useMMDAnimations = <T extends AnimationClip>(
+  clips: T[],
+  root: SkinnedMesh,
+  iks?: IK[],
+  grants?: Grant[],
+): Api<T> => {
+  const { restoreBones, saveBones } = processBones()
+
+  const mixer = useMemo(() => new AnimationMixer(root), [root])
+
+  // const ikSolver = useMemo(() => new CCDIKSolver(root, iks ?? (root.geometry.userData.MMD as { iks: IK[] }).iks), [root, iks])
+  const ikSolver = useMemo(() => new CCDIKSolver(root, iks), [root, iks])
+  const grantSolver = useMemo(() => new GrantSolver(root, grants ?? (root.geometry.userData.MMD as { grants: Grant[] }).grants), [root, grants])
+  // const grantSolver = useMemo(() => new GrantSolver(root, grants), [root, grants])
+
+  const lazyActions = useRef<Api<AnimationClip>['actions']>({})
+
+  const api = useMemo<Api<T>>(() => {
+    const actions = {} as { [key in T['name']]: AnimationAction | null }
+    clips.forEach(clip =>
+      Object.defineProperty(actions, clip.name, {
+        configurable: true,
+        enumerable: true,
+        get: () => (
+          lazyActions.current[clip.name]
+          || (lazyActions.current[clip.name] = mixer.clipAction(clip, root))
+        ),
+      }),
+    )
+    return { actions, clips, grantSolver, ikSolver, mixer, names: clips.map(c => c.name) }
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [clips])
+
+  useFrame((_, delta) => {
+    restoreBones(root)
+
+    mixer.update(delta)
+
+    saveBones(root)
+
+    root.updateMatrixWorld(true)
+
+    // const bones = root.skeleton.bones
+    // const ik = ikSolver.iks.find(i => bones[i.target].name.includes('左足ＩＫ'))
+    // const w1 = bones[ik!.effector].getWorldPosition(new Vector3())
+    // const w2 = bones[ik!.target].getWorldPosition(new Vector3())
+    // console.log('before', w1.toArray(), 'target', w2.toArray())
+
+    ikSolver.update(delta)
+
+    // const w3 = bones[ik!.effector].getWorldPosition(new Vector3())
+    // console.log('after', w3.toArray())
+
+    grantSolver.update()
+  })
+
+  useEffect(() => {
+    return () => {
+      // Clean up only when clips change, wipe out lazy actions and uncache clips
+      lazyActions.current = {}
+      mixer.stopAllAction()
+      Object.values(api.actions).forEach((action) => {
+        mixer.uncacheAction(action as AnimationClip, root)
+      })
+    }
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [clips])
+
+  return api
+}

examples/playground/src/pages/_app.tsx

@@ -1,11 +1,12 @@
 import { Ammo } from '@moeru/three-mmd-r3f'
-import { Environment, Loader, OrbitControls } from '@react-three/drei'
+import { Environment, Loader, OrbitControls, Stats } from '@react-three/drei'
 import { Canvas } from '@react-three/fiber'
 import { Suspense } from 'react'
 import { Outlet } from 'react-router'
 
 const App = () => (
   <>
+    <Stats />
     <Loader />
     <Canvas
       gl={{ localClippingEnabled: true }}

examples/playground/src/pages/b/animation.tsx

@@ -1,14 +1,26 @@
-import { buildAnimation, MMDLoader, VMDLoader } from '@moeru/three-mmd-b'
-import { useAnimations } from '@react-three/drei'
+import { buildAnimation, ExperimentalMMDLoader, VMDLoader } from '@moeru/three-mmd-b'
 import { useLoader } from '@react-three/fiber'
+import { useControls } from 'leva'
 import { useEffect, useMemo } from 'react'
 
+import vmdUrl from '../../../../assets/Telephone/モーションデータ(forMMD)/telephone_motion.vmd?url'
 import pmxUrl from '../../../../assets/げのげ式初音ミク/げのげ式初音ミク.pmx?url'
-import vmdUrl from '../../../../basic/src/assets/vmds/wavefile_v2.vmd?url'
+import { useMMDAnimations } from '../../hooks/use-mmd-animations'
 
 const BAnimation = () => {
-  const mesh = useLoader(MMDLoader, pmxUrl)
-
+  const { showIK, showSkeleton } = useControls({ showIK: false, showSkeleton: false })
+
+  const { iks, mesh, grants } = useLoader(ExperimentalMMDLoader, pmxUrl)
+  // check iks
+  // const { bones } = mesh.skeleton
+  // iks.forEach((ik) => {
+  //   console.log(
+  //     'target', ik.target, bones[ik.target].name,
+  //     'effector', ik.effector, bones[ik.effector].name,
+  //     'links', ik.links.map(l => `${l.index}:${bones[l.index].name}`)
+  //   )
+  // })
+
   const vmd = useLoader(VMDLoader, vmdUrl)
 
   const animation = useMemo(() => {
@@ -17,14 +29,28 @@
     return animation
   }, [vmd, mesh])
 
-  const { actions, ref } = useAnimations([animation])
+  const { actions, ikSolver } = useMMDAnimations([animation], mesh, iks, grants)
+
+  // console.log(
+  //   animation.tracks
+  //     .map(t => t.name)
+  //     .filter(n => n.includes('ＩＫ') || n.toLowerCase().includes('ik'))
+  // )
+
+  const ikHelper = useMemo(() => ikSolver.createHelper(), [ikSolver])
 
   useEffect(() => {
+    // console.log(mesh.skeleton.bones)
     actions?.dance?.play()
   })
 
   return (
-    <primitive object={mesh} ref={ref} rotation={[0, Math.PI, 0]} scale={0.1} />
+    <>
+      <primitive object={mesh} rotation={[0, Math.PI, 0]} scale={0.1} />
+      {showIK && <primitive object={ikHelper} />}
+      {showSkeleton && <skeletonHelper args={[mesh]} />}
+    </>
+
   )
 }
 

examples/playground/src/pages/debug/animation.tsx

@@ -1,20 +1,19 @@
-import {
-  // createMMDAnimationClip,
-  MMDAnimationHelper,
-  MMDLoader,
-  // VMDLoader
-} from '@moeru/three-mmd'
+import type { IK } from 'three/examples/jsm/animation/CCDIKSolver.js'
+
+import { MMDAnimationHelper, MMDLoader } from '@moeru/three-mmd'
 import { buildAnimation, VMDLoader } from '@moeru/three-mmd-b'
-// import { useAnimations } from '@react-three/drei'
 import { useFrame, useLoader } from '@react-three/fiber'
+import { useControls } from 'leva'
 import { useEffect, useMemo } from 'react'
+import { CCDIKHelper } from 'three/examples/jsm/animation/CCDIKSolver.js'
 
 import vmdUrl from '../../../../assets/Telephone/モーションデータ(forMMD)/telephone_motion.vmd?url'
 import pmxUrl from '../../../../assets/げのげ式初音ミク/げのげ式初音ミク.pmx?url'
 
 const DebugAnimation = () => {
-  const object = useLoader(MMDLoader, pmxUrl)
+  const { showIK, showSkeleton } = useControls({ showIK: false, showSkeleton: false })
 
+  const object = useLoader(MMDLoader, pmxUrl)
   const vmd = useLoader(VMDLoader, vmdUrl)
 
   const animation = useMemo(() => {
@@ -24,7 +23,8 @@ const DebugAnimation = () => {
     return animation
   }, [vmd, object])
 
-  const helper = new MMDAnimationHelper({ afterglow: 2 })
+  const helper = useMemo(() => new MMDAnimationHelper(), [])
+  const ikHelper = useMemo(() => new CCDIKHelper(object, (object.geometry.userData.MMD as { iks: IK[] }).iks), [object])
 
   useEffect(() => {
     helper.add(object, {
@@ -35,22 +35,19 @@ const DebugAnimation = () => {
     return () => {
       helper.remove(object)
     }
-  })
+  }, [object, animation, helper])
 
   useFrame((_, delta) => helper.update(delta))
 
-  // const { actions, ref } = useAnimations([animation])
-
-  // useEffect(() => {
-  //   actions?.dance?.play()
-  // })
-
   return (
-    <primitive
-      object={object}
-      // ref={ref}
-      scale={0.1}
-    />
+    <>
+      <primitive
+        object={object}
+        scale={0.1}
+      />
+      {showIK && <primitive object={ikHelper} />}
+      {showSkeleton && <skeletonHelper args={[object]} />}
+    </>
   )
 }
 

examples/playground/src/pages/debug/animation2.tsx

@@ -0,0 +1,49 @@
+import { MMDLoader } from '@moeru/three-mmd'
+import { buildAnimation, VMDLoader } from '@moeru/three-mmd-b'
+import { useLoader } from '@react-three/fiber'
+import { useControls } from 'leva'
+import { useEffect, useMemo } from 'react'
+
+import vmdUrl from '../../../../assets/Telephone/モーションデータ(forMMD)/telephone_motion.vmd?url'
+import pmxUrl from '../../../../assets/げのげ式初音ミク/げのげ式初音ミク.pmx?url'
+import { useMMDAnimations } from '../../hooks/use-mmd-animations'
+
+const DebugAnimation2 = () => {
+  const { showIK, showSkeleton } = useControls({ showIK: false, showSkeleton: false })
+
+  const object = useLoader(MMDLoader, pmxUrl)
+  const vmd = useLoader(VMDLoader, vmdUrl)
+
+  const animation = useMemo(() => {
+    const animation = buildAnimation(vmd, object)
+    animation.name = 'dance'
+    return animation
+  }, [vmd, object])
+
+  const { actions, ikSolver } = useMMDAnimations([animation], object)
+
+  const ikHelper = useMemo(() => ikSolver.createHelper(), [ikSolver])
+
+  useEffect(() => {
+    // console.log(object.skeleton.bones)
+    actions?.dance?.play()
+
+    return () => {
+      object.pose()
+    }
+  })
+
+  return (
+    <>
+      <primitive
+        object={object}
+        // ref={ref}
+        scale={0.1}
+      />
+      {showIK && <primitive object={ikHelper} />}
+      {showSkeleton && <skeletonHelper args={[object]} />}
+    </>
+  )
+}
+
+export default DebugAnimation2

examples/playground/src/router.ts

@@ -9,6 +9,7 @@ export type Path =
   | `/b/mesh`
   | `/b/skeleton`
   | `/debug/animation`
+  | `/debug/animation2`
   | `/debug/mesh`
   | `/debug/physics-correct`
   | `/debug/physics-incorrect`

examples/playground/src/utils/process-bones.ts

@@ -0,0 +1,38 @@
+import type { SkinnedMesh } from 'three'
+
+export const processBones = () => {
+  let backupBones: Float32Array | undefined
+
+  const restoreBones = (mesh: SkinnedMesh) => {
+    if (backupBones === undefined)
+      return
+
+    mesh.skeleton.bones.forEach((bone, i) => {
+      bone.position.fromArray(backupBones!, i * 7)
+      bone.quaternion.fromArray(backupBones!, i * 7 + 3)
+    })
+  }
+
+  /*
+   * Avoiding these two issues by restore/save bones before/after mixer animation.
+   *
+   * 1. PropertyMixer used by AnimationMixer holds cache value in .buffer.
+   *    Calculating IK, Grant, and Physics after mixer animation can break
+   *    the cache coherency.
+   *
+   * 2. Applying Grant two or more times without reset the posing breaks model.
+   */
+  const saveBones = (mesh: SkinnedMesh) => {
+    const bones = mesh.skeleton.bones
+
+    if (backupBones === undefined)
+      backupBones = new Float32Array(bones.length * 7)
+
+    mesh.skeleton.bones.forEach((bone, i) => {
+      bone.position.toArray(backupBones!, i * 7)
+      bone.quaternion.toArray(backupBones!, i * 7 + 3)
+    })
+  }
+
+  return { restoreBones, saveBones }
+}

packages/three-mmd-b/src/index.ts

@@ -1,9 +1,11 @@
+export { ExperimentalMMDLoader, type MMD } from './loaders/experimental-mmd-loader'
 export { MMDLoader } from './loaders/mmd-loader'
 export { PMDLoader } from './loaders/pmd-loader'
 export { PMXLoader } from './loaders/pmx-loader'
 export { VMDLoader } from './loaders/vmd-loader'
 
 export { buildAnimation, buildCameraAnimation } from './utils/build-animation'
 export { buildGeometry } from './utils/build-geometry'
+export { buildIK } from './utils/build-ik'
 export { buildMaterial } from './utils/build-material'
 export { buildMesh } from './utils/build-mesh'