pmndrs · mikeskydev · Jul 24, 2025 · Jul 24, 2025 · Jul 24, 2025
diff --git a/examples/camera-access/.gitignore b/examples/camera-access/.gitignore
@@ -0,0 +1 @@
+dist
diff --git a/examples/camera-access/index.html b/examples/camera-access/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Document</title>
+    <script async type="module" src="./index.tsx"></script>
+</head>
+<body style="touch-action: none; margin: 0; position: relative; width: 100dvw; height: 100dvh; overflow: hidden;">
+    <div id="root" style="position: absolute; inset: 0; display: flex; flex-direction: column;"></div>
+</body>
+</html>
diff --git a/examples/camera-access/index.tsx b/examples/camera-access/index.tsx
@@ -0,0 +1,9 @@
+import { StrictMode } from 'react'
+import { createRoot } from 'react-dom/client'
+import { App } from './src/app.js'
+
+createRoot(document.getElementById('root')!).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)
diff --git a/examples/camera-access/package.json b/examples/camera-access/package.json
@@ -0,0 +1,10 @@
+{
+    "dependencies": {
+        "@react-three/xr": "workspace:~"
+    },
+    "scripts": {
+        "dev": "vite --host",
+        "check:eslint": "eslint \"*.{ts,tsx}\"",
+        "fix:eslint": "eslint \"*.{ts,tsx}\" --fix"
+    }
+}
diff --git a/examples/camera-access/src/app.tsx b/examples/camera-access/src/app.tsx
@@ -0,0 +1,20 @@
+import { Canvas, useFrame } from '@react-three/fiber'
+import { createXRStore, XR, XROrigin } from '@react-three/xr'
+import { RotatingBox } from './box'
+
+const store = createXRStore({ cameraAccess: true })
+
+export function App() {
+  return (
+    <>
+      <button onClick={() => store.enterAR()}>Enter AR</button>
+      <Canvas style={{ width: '100%', flexGrow: 1 }}>
+        <XR store={store}>
+          <XROrigin />
+          <hemisphereLight position={[0.5, 1, 0.25]} color={0xffffbb} groundColor={0x080820} intensity={3}/>
+          <RotatingBox/>
+        </XR>
+      </Canvas>
+    </>
+  )
+}
diff --git a/examples/camera-access/src/box.tsx b/examples/camera-access/src/box.tsx
@@ -0,0 +1,41 @@
+import { useState, useRef, useEffect } from 'react'
+import { MeshStandardMaterial, Texture } from 'three'
+import { useFrame } from '@react-three/fiber'
+import { Box } from '@react-three/drei'
+import { createXRStore, useXR } from '@react-three/xr'
+
+const store = createXRStore()
+
+export function RotatingBox() {
+  const boxRef = useRef<Box>(null)
+  const matRef = useRef<MeshStandardMaterial>(null);
+
+  const cameraTextures = useXR((xr) => xr.cameraImages)
+  const [camTexture, setCamTexutre] = useState<Texture>()
+
+  useEffect(() => {
+    if (cameraTextures && cameraTextures[0]) {
+      setCamTexutre(cameraTextures[0])
+    }
+  }, [cameraTextures])
+
+  useEffect(() => {
+    if (camTexture) {
+      matRef.current.map = camTexture
+      matRef.current.needsUpdate = true
+    }
+  }, [camTexture]);
+
+  useFrame((state, delta, xrFrame) => {
+    boxRef.current.rotation.x += delta
+    boxRef.current.rotation.y += delta
+  })
+
+  return (
+    <>
+      <Box ref={boxRef} position={[0, 0, -3]}>
+        <meshStandardMaterial ref={matRef} color={'white'}/>
+      </Box>
+    </>
+  )
+}
diff --git a/examples/camera-access/vite.config.ts b/examples/camera-access/vite.config.ts
@@ -0,0 +1,11 @@
+import basicSsl from '@vitejs/plugin-basic-ssl'
+import react from '@vitejs/plugin-react'
+import { defineConfig } from 'vite'
+
+// https://vitejs.dev/config/
+export default defineConfig({
+  plugins: [react(), basicSsl()],
+  resolve: {
+    dedupe: ['@react-three/fiber', 'three'],
+  },
+})
diff --git a/packages/xr/src/init.ts b/packages/xr/src/init.ts
@@ -39,6 +39,10 @@ export type XRSessionInitOptions = {
    * @default false
    */
   depthSensing?: XRSessionFeatureRequest
+  /**
+   * @default false
+   */
+  cameraAccess?: XRSessionFeatureRequest
   /**
    * overrides the session init object
    * use with caution
@@ -64,8 +68,9 @@ export function buildXRSessionInit(
     layers = true,
     meshDetection = true,
     planeDetection = true,
-    customSessionInit,
     depthSensing = false,
+    cameraAccess = false,
+    customSessionInit,
     hitTest = true,
     domOverlay = true,
     bodyTracking = false, //until 6.7 since breaking change
@@ -91,6 +96,7 @@ export function buildXRSessionInit(
   addXRSessionFeature(meshDetection, 'mesh-detection', requiredFeatures, optionalFeatures)
   addXRSessionFeature(planeDetection, 'plane-detection', requiredFeatures, optionalFeatures)
   addXRSessionFeature(depthSensing, 'depth-sensing', requiredFeatures, optionalFeatures)
+  addXRSessionFeature(cameraAccess, 'camera-access', requiredFeatures, optionalFeatures)
   addXRSessionFeature(domOverlay, 'dom-overlay', requiredFeatures, optionalFeatures)
   addXRSessionFeature(hitTest, 'hit-test', requiredFeatures, optionalFeatures)
   addXRSessionFeature(bodyTracking, 'body-tracking', requiredFeatures, optionalFeatures)

diff --git a/packages/xr/src/store.ts b/packages/xr/src/store.ts
@@ -1,5 +1,5 @@
 import { XRDevice } from 'iwer'
-import { Camera, Object3D, WebXRManager, Vector3 } from 'three'
+import { Camera, Object3D, WebXRManager, Vector3, Texture } from 'three'
 import { StoreApi, createStore } from 'zustand/vanilla'
 import { XRControllerLayoutLoaderOptions, updateXRControllerState } from './controller/index.js'
 import { XRHandLoaderOptions } from './hand/index.js'
@@ -165,6 +165,10 @@ export type XRState<T extends XRElementImplementations> = Readonly<
      * active additional webxr layers
      */
     layerEntries: ReadonlyArray<XRLayerEntry>
+    /**
+     * `Texture`s representing the current camera view 
+     */
+    cameraImages: ReadonlyArray<Texture>
     /**
      * access to the emulator values to change the emulated input device imperatively
      */
@@ -387,6 +391,7 @@ const baseInitialState: Omit<
   inputSourceStates: [],
   detectedMeshes: [],
   detectedPlanes: [],
+  cameraImages: [],
   layerEntries: [],
 }
 
@@ -923,6 +928,7 @@ function createBindToSession(
       visibilityState: session.visibilityState,
       detectedMeshes: [],
       detectedPlanes: [],
+      cameraImages: [],
       mode: session.environmentBlendMode === 'opaque' ? 'immersive-vr' : 'immersive-ar',
       session,
       mediaBinding: typeof XRMediaBinding == 'undefined' ? undefined : new XRMediaBinding(session),
@@ -936,7 +942,7 @@ type Mutable<T> = {
 
 function updateSession(store: StoreApi<XRState<XRElementImplementations>>, frame: XRFrame, manager: WebXRManager) {
   const referenceSpace = manager.getReferenceSpace()
-  const { detectedMeshes: prevMeshes, detectedPlanes: prevPlanes, session, inputSourceStates } = store.getState()
+  const { detectedMeshes: prevMeshes, detectedPlanes: prevPlanes, cameraImages: prevCamImages, session, inputSourceStates } = store.getState()
   if (referenceSpace == null || session == null) {
     //not in a XR session
     return
@@ -946,10 +952,30 @@ function updateSession(store: StoreApi<XRState<XRElementImplementations>>, frame
   const detectedPlanes = updateDetectedEntities(prevPlanes, frame.detectedPlanes)
   const detectedMeshes = updateDetectedEntities(prevMeshes, frame.detectedMeshes)
 
-  if (prevPlanes != detectedPlanes || prevMeshes != detectedMeshes) {
-    store.setState({ detectedPlanes, detectedMeshes })
+
+  const viewerPose = frame.getViewerPose(referenceSpace)
+
+  const camImages = new Set<Texture>();
+
+  if (viewerPose) {
+    for (let i = 0; i < viewerPose.views.length; i++) {
+      if (viewerPose.views[i].camera) {
+        const camImage = manager.getCameraTexture( viewerPose.views[i].camera );
+        if (camImage) {
+          camImages.add(camImage);
+        }
+      }
+    }
+  }
+
+  const cameraImages = updateDetectedEntities(prevCamImages, camImages);
+
+  if (prevPlanes != detectedPlanes || prevMeshes != detectedMeshes || prevCamImages != cameraImages) {
+    console.log("updating");
+    store.setState({ detectedPlanes, detectedMeshes, cameraImages })
   }
 
+
   //update input sources
   const inputSourceStatesLength = inputSourceStates.length
   for (let i = 0; i < inputSourceStatesLength; i++) {

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml