Thiết kế website giá rẻ

Question

In developing my app using ImmersiveSpace’s CompositorLayer feature in VisionPro, I utilize the layerRenderer.onSpatialEvent to obtain the gesture pose3d of one hand. I’m wondering how I can acquire the gestures of both hands. Alternatively, are there any better methods for gestural manipulation of the rendered scene after using CompositorLayer?

https://github.com/scier/MetalSplatter (Swift package for rendering splat files)

import SwiftUI

import CompositorServices


@main
struct splatting_testzjhApp: App {
    var body: some Scene {
        WindowGroup("MetalSplatter Sample App", id: "main")  {
            ContentView()
        }
        
        ImmersiveSpace(id: "ImmersiveSpace") {
            ImmersiveView()
        }.immersionStyle(selection: .constant(.mixed), in: .mixed)
        

        ImmersiveSpace(for: ModelIdentifier.self) { modelIdentifier in
            CompositorLayer(configuration: ContentStageConfiguration()) { layerRenderer in
                let renderer = VisionSceneRenderer(layerRenderer)
                do {
                    try renderer.load(modelIdentifier.wrappedValue)
                } catch {
                    print("Error loading model: (error.localizedDescription)")
                }
                renderer.startRenderLoop()
                // Set up spatial event handling
                layerRenderer.onSpatialEvent = { events in
                    for event in events {
                        switch event.phase {
                        case .active:
                            //Update UI or state to reflect the ongoing interaction
                            print("Active event at (String(describing: event.inputDevicePose))")
                            if let pose = event.inputDevicePose {
                                let currentPosition = SIMD3<Float>(
                                    Float(pose.pose3D.position.x),
                                    Float(pose.pose3D.position.y),
                                    Float(pose.pose3D.position.z)
                                )
                                renderer.updatePositionBasedOnMovement(currentPosition: currentPosition)
                            }
                            
                        case .cancelled:
                            // Perform any necessary cleanup or rollback
                            print("Event cancelled")
                        case .ended:
                            renderer.lastInputDevicePosition = nil
                            //print("ended event at (String(describing: event.inputDevicePose))")
                            // Finalize the interaction
                            print("Event ended normally")
                        default:
                            break
                        }
                    }
                }
                
            }
        }
        .immersionStyle(selection: .constant(.mixed), in: .mixed)
        
    }
}

import CompositorServices
import Metal
import MetalSplatter
import os
import SampleBoxRenderer
import simd
import Spatial
import SwiftUI
import simd

func matrix4x4_scale(sx: Float, sy: Float, sz: Float) -> matrix_float4x4 {
    var matrix = matrix_identity_float4x4
    matrix.columns.0.x = sx
    matrix.columns.1.y = sy
    matrix.columns.2.z = sz
    return matrix
}

extension LayerRenderer.Clock.Instant.Duration {
    var timeInterval: TimeInterval {
        let nanoseconds = TimeInterval(components.attoseconds / 1_000_000_000)
        return TimeInterval(components.seconds) + (nanoseconds / TimeInterval(NSEC_PER_SEC))
    }
}

class VisionSceneRenderer {
    private static let log =
    Logger(subsystem: Bundle.main.bundleIdentifier!,
           category: "CompsitorServicesSceneRenderer")
    
    let layerRenderer: LayerRenderer
    let device: MTLDevice
    let commandQueue: MTLCommandQueue
    
    var model: ModelIdentifier?
    var modelRenderer: (any ModelRenderer)?
    
    let inFlightSemaphore = DispatchSemaphore(value: Constants.maxSimultaneousRenders)
    
    var lastRotationUpdateTimestamp: Date? = nil
    var rotation: Angle = .zero
    
    var lastInputDevicePosition: SIMD3<Float>? = nil
    var position: SIMD3<Float> = SIMD3<Float>(0.0, 0.0, 0.0)
    
    // Add scale and timestamp variables
    var scale: Float = 1.0
    var lastScaleUpdateTimestamp: Date? = nil
    let minScale: Float = 0.5
    let maxScale: Float = 1.5
    var scaleIncreasing: Bool = true
    
    // Add horizontal movement variables
    var positionX: Float = 0.0
    var lastPositionUpdateTimestamp: Date? = nil
    let minPositionX: Float = -100.0
    let maxPositionX: Float = 100.0
    let minPositionY: Float = -100.0
    let maxPositionY: Float = 100.0
    let minPositionZ: Float = -100.0  // 示例最小值
    let maxPositionZ: Float = 100.0   // 示例最大值
    
    var positionIncreasing: Bool = true
    
    let arSession: ARKitSession
    let worldTracking: WorldTrackingProvider
    
    init(_ layerRenderer: LayerRenderer) {
        self.layerRenderer = layerRenderer
        self.device = layerRenderer.device
        self.commandQueue = self.device.makeCommandQueue()!
        
        worldTracking = WorldTrackingProvider()
        arSession = ARKitSession()
    }
    
    func load(_ model: ModelIdentifier?) throws {
        guard model != self.model else { return }
        self.model = model
        
        modelRenderer = nil
        switch model {
        case .gaussianSplat(let url):
            let splat = try SplatRenderer(device: device,
                                          colorFormat: layerRenderer.configuration.colorFormat,
                                          depthFormat: layerRenderer.configuration.depthFormat,
                                          stencilFormat: .invalid,
                                          sampleCount: 1,
                                          maxViewCount: layerRenderer.properties.viewCount,
                                          maxSimultaneousRenders: Constants.maxSimultaneousRenders)
            try splat.readPLY(from: url)
            modelRenderer = splat
        case .sampleBox:
            modelRenderer = try! SampleBoxRenderer(device: device,
                                                   colorFormat: layerRenderer.configuration.colorFormat,
                                                   depthFormat: layerRenderer.configuration.depthFormat,
                                                   stencilFormat: .invalid,
                                                   sampleCount: 1,
                                                   maxViewCount: layerRenderer.properties.viewCount,
                                                   maxSimultaneousRenders: Constants.maxSimultaneousRenders)
        case .none:
            break
        }
    }
    
    func startRenderLoop() {
        Task {
            do {
                try await arSession.run([worldTracking])
            } catch {
                fatalError("Failed to initialize ARSession")
            }
            
            let renderThread = Thread {
                self.renderLoop()
            }
            renderThread.name = "Render Thread"
            renderThread.start()
        }
    }
    

    
    func setScale(newScale: Float) {
        scale = min(max(newScale, minScale), maxScale)
    }
    
    func updateScaleBasedOnMovement(currentPosition: SIMD3<Float>) {
        guard let lastPosition = lastInputDevicePosition else {
            lastInputDevicePosition = currentPosition
            return
        }
        
        // Calculate the difference in Y position (you can choose any axis or a combination)
        let deltaY = currentPosition.y - lastPosition.y
        
        // Determine the direction and adjust scale accordingly
        let scaleChange = deltaY * 0.1 // Adjust this factor based on how sensitive you want the scaling to be
        setScale(newScale: scale + scaleChange)
        
        // Update the last position
        lastInputDevicePosition = currentPosition
    }
    
    func updatePositionBasedOnMovement(currentPosition: SIMD3<Float>) {
        guard let lastPosition = lastInputDevicePosition else {
            lastInputDevicePosition = currentPosition
            return
        }
        
        let deltaX = currentPosition.x - lastPosition.x
        let deltaY = currentPosition.y - lastPosition.y
        let deltaZ = currentPosition.z - lastPosition.z  // 新增对Z轴的处理
        
        position.x += deltaX
        position.y += deltaY
        position.z += deltaZ  // 更新Z轴位置
        lastInputDevicePosition = currentPosition
    }

    private func viewportCameras(drawable: LayerRenderer.Drawable, deviceAnchor: DeviceAnchor?) -> [ModelRenderer.CameraMatrices] {

        let rotationMatrix = matrix4x4_rotation(radians: Float(rotation.radians),
                                                axis: Constants.rotationAxis)
        //        let translationMatrix = matrix4x4_translation(positionX, 0.0, Constants.modelCenterZ)
        let translationMatrix = matrix4x4_translation(position.x, position.y, position.z)
        
        let scaleMatrix = matrix4x4_scale(sx: scale, sy: scale, sz: scale) // 使用自定义的缩放函数
        
        let commonUpCalibration = matrix4x4_rotation(radians: .pi, axis: SIMD3<Float>(0, 0, 1))
        
        let simdDeviceAnchor = deviceAnchor?.originFromAnchorTransform ?? matrix_identity_float4x4
        
        return drawable.views.map { view in
            let userViewpointMatrix = (simdDeviceAnchor * view.transform).inverse
            let projectionMatrix = ProjectiveTransform3D(leftTangent: Double(view.tangents[0]),
                                                         rightTangent: Double(view.tangents[1]),
                                                         topTangent: Double(view.tangents[2]),
                                                         bottomTangent: Double(view.tangents[3]),
                                                         nearZ: Double(drawable.depthRange.y),
                                                         farZ: Double(drawable.depthRange.x),
                                                         reverseZ: true)
            let screenSize = SIMD2(x: Int(view.textureMap.viewport.width),
                                   y: Int(view.textureMap.viewport.height))
            return (projection: .init(projectionMatrix),
                    view: userViewpointMatrix * translationMatrix * rotationMatrix * scaleMatrix * commonUpCalibration,
                    screenSize: screenSize)
        }
    }
    
    func renderFrame() {
        guard let frame = layerRenderer.queryNextFrame() else { return }
        
        frame.startUpdate()
        frame.endUpdate()
        
        guard let timing = frame.predictTiming() else { return }
        LayerRenderer.Clock().wait(until: timing.optimalInputTime)
        
        guard let commandBuffer = commandQueue.makeCommandBuffer() else {
            fatalError("Failed to create command buffer")
        }
        
        guard let drawable = frame.queryDrawable() else { return }
        
        _ = inFlightSemaphore.wait(timeout: DispatchTime.distantFuture)
        
        frame.startSubmission()
        
        let time = LayerRenderer.Clock.Instant.epoch.duration(to: drawable.frameTiming.presentationTime).timeInterval
        let deviceAnchor = worldTracking.queryDeviceAnchor(atTimestamp: time)
        
        drawable.deviceAnchor = deviceAnchor
        
        let semaphore = inFlightSemaphore
        commandBuffer.addCompletedHandler { (_ commandBuffer)-> Swift.Void in
            semaphore.signal()
        }
        
        let viewportCameras = self.viewportCameras(drawable: drawable, deviceAnchor: deviceAnchor)
        modelRenderer?.willRender(viewportCameras: viewportCameras)
        
        let renderPassDescriptor = MTLRenderPassDescriptor()
        renderPassDescriptor.colorAttachments[0].texture = drawable.colorTextures[0]
        renderPassDescriptor.colorAttachments[0].loadAction = .clear
        renderPassDescriptor.colorAttachments[0].storeAction = .store
        renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
        renderPassDescriptor.depthAttachment.texture = drawable.depthTextures[0]
        renderPassDescriptor.depthAttachment.loadAction = .clear
        renderPassDescriptor.depthAttachment.storeAction = .store
        renderPassDescriptor.depthAttachment.clearDepth = 1.0
        renderPassDescriptor.rasterizationRateMap = drawable.rasterizationRateMaps.first
        if layerRenderer.configuration.layout == .layered {
            renderPassDescriptor.renderTargetArrayLength = drawable.views.count
        }
        
        guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
            fatalError("Failed to create render encoder")
        }
        
        renderEncoder.label = "Primary Render Encoder"
        
        renderEncoder.setViewports(drawable.views.map { $0.textureMap.viewport })
        
        if drawable.views.count > 1 {
            var viewMappings = (0..<drawable.views.count).map {
                MTLVertexAmplificationViewMapping(viewportArrayIndexOffset: UInt32($0),
                                                  renderTargetArrayIndexOffset: UInt32($0))
            }
            renderEncoder.setVertexAmplificationCount(viewportCameras.count, viewMappings: &viewMappings)
        }
        
        modelRenderer?.render(viewportCameras: viewportCameras, to: renderEncoder)
        
        renderEncoder.endEncoding()
        
        drawable.encodePresent(commandBuffer: commandBuffer)
        
        commandBuffer.commit()
        
        frame.endSubmission()
    }
    
    func renderLoop() {
        while true {
            if layerRenderer.state == .invalidated {
                Self.log.warning("Layer is invalidated")
                return
            } else if layerRenderer.state == .paused {
                layerRenderer.waitUntilRunning()
                continue
            } else {
                autoreleasepool {
                    self.renderFrame()
                }
            }
        }
    }
}

Danh mục