I am currently developing a local multiplayer MR (Mixed Reality) application for Meta Quest 3 using Unity. The application uses PUN2 for network communication, and I aim to synchronize the player’s head and hands (left and right) across the network.

Problem

When running the application on two HMDs (Meta Quest 3) and engaging in multiplayer, the positions of the Head and Hands become significantly misaligned with the actual position of the user’s body. Specifically, the Head and Hand objects are not correctly aligned with the other player’s body and are displayed at incorrect locations.

What I Want to Achieve

I want the Head and Hands to accurately align with the actual positions of the other user’s body. However, I am currently struggling with this issue and would appreciate any advice on how to resolve it.

Current Setup

• I am using OVRCameraRig for camera setup.
• OVRSceneManager and Passthrough features are also utilized.

Code Samples

NetworkManager.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Photon.Pun;
using Photon.Realtime;

public class NetworkManager : MonoBehaviourPunCallbacks
{
    void Start()
    {
        ConnectToServer();
    }

    void ConnectToServer()
    {
        PhotonNetwork.ConnectUsingSettings();
        Debug.Log("Connecting to server...");
    }

    public override void OnConnectedToMaster()
    {
        Debug.Log("Connected to server.");
        base.OnConnectedToMaster();
        RoomOptions roomOptions = new RoomOptions();
        roomOptions.MaxPlayers = 4;
        roomOptions.IsVisible = true;
        roomOptions.IsOpen = true;

        PhotonNetwork.JoinOrCreateRoom("Room 1", roomOptions, TypedLobby.Default);
    }

    public override void OnJoinedRoom()
    {
        Debug.Log("Joined room.");
        base.OnJoinedRoom();
    }

    public override void OnPlayerEnteredRoom(Player newPlayer)
    {
        Debug.Log("New player joined.");
        base.OnPlayerEnteredRoom(newPlayer);
    }
}

NetworkPlayerSpawner.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Photon.Pun;

public class NetworkPlayerSpawner : MonoBehaviourPunCallbacks
{
    public GameObject spawnedPlayerPrefab;
    public override void OnJoinedRoom()
    {
        base.OnJoinedRoom();
        spawnedPlayerPrefab = PhotonNetwork.Instantiate("Network Player", transform.position, transform.rotation);
    }

    public override void OnLeftRoom()
    {
        base.OnLeftRoom();
        PhotonNetwork.Destroy(spawnedPlayerPrefab);
    }
}

NetworkPlayer.cs

using UnityEngine;
using Photon.Pun;

public class NetworkPlayer : MonoBehaviour
{
    public Transform head;
    public Transform leftHand;
    public Transform rightHand;
    private PhotonView photonView;

    public Animator leftHandAnimator;
    public Animator rightHandAnimator;

    private Transform headRig;
    private Transform leftHandRig;
    private Transform rightHandRig;

    void Start()
    {
        photonView = GetComponent<PhotonView>();
        OVRCameraRig rig = FindObjectOfType<OVRCameraRig>();
        headRig = rig.centerEyeAnchor;
        leftHandRig = rig.leftHandAnchor;
        rightHandRig = rig.rightHandAnchor;

        if (photonView.IsMine)
        {
            foreach (var item in GetComponentsInChildren<Renderer>())
            {
                item.enabled = false;
            }
        }
    }

    void Update()
    {
        if (photonView.IsMine)
        {
            MapPosition(head, headRig);
            MapPosition(leftHand, leftHandRig);
            MapPosition(rightHand, rightHandRig);

            UpdateHandAnimation(OVRInput.Controller.LTouch, leftHandAnimator);
            UpdateHandAnimation(OVRInput.Controller.RTouch, rightHandAnimator);
        }
    }

    void UpdateHandAnimation(OVRInput.Controller controller, Animator handAnimator)
    {
        float triggerValue = OVRInput.Get(OVRInput.Axis1D.PrimaryIndexTrigger, controller);
        handAnimator.SetFloat("Trigger", triggerValue);

        float gripValue = OVRInput.Get(OVRInput.Axis1D.PrimaryHandTrigger, controller);
        handAnimator.SetFloat("Grip", gripValue);
    }

    void MapPosition(Transform target, Transform rigTransform)
    {
        target.position = rigTransform.position;
        target.rotation = rigTransform.rotation;
    }
}

What I Have Tried

So far, the objects are being synchronized correctly using PUN2, but the Head and Hand positions do not align with the player’s body, resulting in a significant misalignment. I am unsure of the cause and would appreciate any suggestions or solutions to resolve this issue.

If you have any approaches to ensure that the Head and Hands are accurately aligned with the player’s body, please share your insights.

Thank you in advance for your assistance.