From a7c488a36b509496faec5a71a84530f6c70b2eab Mon Sep 17 00:00:00 2001
From: Christian Stuff <christian@daily.co>
Date: Wed, 17 Sep 2025 13:14:02 +0200
Subject: [PATCH 1/2] implement multi-participant handling & state

---
 client-react/src/PipecatClientAudio.tsx       | 102 +++++++--
 .../src/PipecatClientParticipantManager.tsx   | 203 ++++++++++++++++++
 client-react/src/PipecatClientProvider.tsx    |   6 +-
 client-react/src/PipecatClientVideo.tsx       |  24 ++-
 client-react/src/index.ts                     |   4 +
 .../src/usePipecatClientMediaTrack.ts         | 119 +++-------
 .../src/usePipecatClientParticipant.ts        |  16 ++
 .../src/usePipecatClientParticipantIds.ts     |  28 +++
 8 files changed, 390 insertions(+), 112 deletions(-)
 create mode 100644 client-react/src/PipecatClientParticipantManager.tsx
 create mode 100644 client-react/src/usePipecatClientParticipant.ts
 create mode 100644 client-react/src/usePipecatClientParticipantIds.ts

diff --git a/client-react/src/PipecatClientAudio.tsx b/client-react/src/PipecatClientAudio.tsx
index d1f061b..68dc35a 100644
--- a/client-react/src/PipecatClientAudio.tsx
+++ b/client-react/src/PipecatClientAudio.tsx
@@ -5,38 +5,112 @@
  */
 
 import { RTVIEvent } from "@pipecat-ai/client-js";
-import { useCallback, useEffect, useRef } from "react";
+import React, { useCallback, useEffect, useRef } from "react";
 
 import { usePipecatClientMediaTrack } from "./usePipecatClientMediaTrack";
+import { usePipecatClientParticipantIds } from "./usePipecatClientParticipantIds";
 import { useRTVIClientEvent } from "./useRTVIClientEvent";
 
-export const PipecatClientAudio = () => {
-  const botAudioRef = useRef<HTMLAudioElement>(null);
-  const botAudioTrack = usePipecatClientMediaTrack("audio", "bot");
+interface AudioElementProps
+  extends React.AudioHTMLAttributes<HTMLAudioElement> {
+  participantId: string;
+  track: MediaStreamTrack | null;
+}
+
+const AudioElement = ({
+  participantId,
+  track,
+  ...props
+}: AudioElementProps) => {
+  const audioRef = useRef<HTMLAudioElement>(null);
 
   useEffect(() => {
-    if (!botAudioRef.current || !botAudioTrack) return;
-    if (botAudioRef.current.srcObject) {
+    if (!audioRef.current || !track) return;
+    if (audioRef.current.srcObject) {
       const oldTrack = (
-        botAudioRef.current.srcObject as MediaStream
+        audioRef.current.srcObject as MediaStream
       ).getAudioTracks()[0];
-      if (oldTrack.id === botAudioTrack.id) return;
+      if (oldTrack.id === track.id) return;
     }
-    botAudioRef.current.srcObject = new MediaStream([botAudioTrack]);
-  }, [botAudioTrack]);
+    audioRef.current.srcObject = new MediaStream([track]);
+  }, [track]);
 
   useRTVIClientEvent(
     RTVIEvent.SpeakerUpdated,
     useCallback((speaker: MediaDeviceInfo) => {
-      if (!botAudioRef.current) return;
-      if (typeof botAudioRef.current.setSinkId !== "function") return;
-      botAudioRef.current.setSinkId(speaker.deviceId);
+      if (!audioRef.current) return;
+      if (typeof audioRef.current.setSinkId !== "function") return;
+      audioRef.current.setSinkId(speaker.deviceId);
     }, [])
   );
 
+  return (
+    <audio
+      ref={audioRef}
+      autoPlay
+      data-participant-id={participantId}
+      {...props}
+    />
+  );
+};
+
+/**
+ * Component for individual participant audio
+ */
+const ParticipantAudio = ({ participantId }: { participantId: string }) => {
+  // Determine participant type and get appropriate tracks
+  const isLocal = participantId === "local";
+  const isBot = participantId === "bot";
+  const isRemote = !isLocal && !isBot;
+
+  const audioTrack = usePipecatClientMediaTrack(
+    "audio",
+    isLocal ? "local" : isBot ? "bot" : "remote",
+    isRemote ? participantId : undefined
+  );
+
+  const screenAudioTrack = usePipecatClientMediaTrack(
+    "screenAudio",
+    isLocal ? "local" : "remote",
+    isRemote ? participantId : undefined
+  );
+
+  return (
+    <>
+      <AudioElement
+        data-bot={isBot}
+        data-local={isLocal}
+        data-remote={isRemote}
+        data-track-type="audio"
+        participantId={participantId}
+        track={audioTrack}
+      />
+      {screenAudioTrack && (
+        <AudioElement
+          data-bot={isBot}
+          data-local={isLocal}
+          data-remote={isRemote}
+          data-track-type="screenAudio"
+          participantId={`${participantId}-screen`}
+          track={screenAudioTrack}
+        />
+      )}
+    </>
+  );
+};
+
+/**
+ * Component that renders all participant audio.
+ */
+export const PipecatClientAudio = () => {
+  const { participantIds } = usePipecatClientParticipantIds(false, true);
+
   return (
     <>
-      <audio ref={botAudioRef} autoPlay />
+      {/* All participant audio */}
+      {participantIds.map((participantId) => (
+        <ParticipantAudio key={participantId} participantId={participantId} />
+      ))}
     </>
   );
 };
diff --git a/client-react/src/PipecatClientParticipantManager.tsx b/client-react/src/PipecatClientParticipantManager.tsx
new file mode 100644
index 0000000..60900bf
--- /dev/null
+++ b/client-react/src/PipecatClientParticipantManager.tsx
@@ -0,0 +1,203 @@
+/**
+ * Copyright (c) 2025, Daily.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+import { Participant, RTVIEvent, Tracks } from "@pipecat-ai/client-js";
+import { atom } from "jotai";
+import { atomFamily, useAtomCallback } from "jotai/utils";
+import { memo, useCallback } from "react";
+
+import { useRTVIClientEvent } from "./useRTVIClientEvent";
+
+// Remote participants tracking
+export const remoteParticipantIdsAtom = atom<string[]>([]);
+
+// Bot ID mapping - maps the bot's UUID to our internal "bot" identifier
+const botIdAtom = atom<string | null>(null);
+
+const localParticipantAtom = atom<Participant | null>(null);
+const botParticipantAtom = atom<Participant | null>(null);
+
+export const participantAtom = atomFamily((participantId: string) =>
+  atom<Participant | null>((get) => {
+    // Handle special cases for local and bot participants
+    if (participantId === "local") {
+      return get(localParticipantAtom);
+    }
+
+    if (participantId === "bot") {
+      return get(botParticipantAtom);
+    }
+
+    // For remote participants, return the stored value
+    return get(remoteParticipantAtom(participantId));
+  })
+);
+
+// Keep the original remoteParticipantAtom for internal use
+const remoteParticipantAtom = atomFamily(() => atom<Participant | null>(null));
+
+// Unified track atom family for all participants and track types
+type TrackType = keyof Tracks["local"];
+type TrackKey = `${string}:${TrackType}`;
+
+export const trackAtom = atomFamily((key: TrackKey) => {
+  // Create a unique atom for each participant/track combination
+  // Key format: "participantId:trackType"
+  void key; // Acknowledge the key parameter
+  return atom<MediaStreamTrack | null>(null);
+});
+
+/**
+ * Component that manages participant events globally.
+ */
+export const PipecatClientParticipantManager = memo(() => {
+  const addParticipant = useAtomCallback(
+    useCallback((get, set, participant: Participant) => {
+      if (participant.local) {
+        set(localParticipantAtom, participant);
+        return;
+      }
+
+      if (participant.id === get(botIdAtom)) {
+        set(botParticipantAtom, participant);
+        return;
+      }
+
+      const currentIds = get(remoteParticipantIdsAtom);
+      if (!currentIds.includes(participant.id)) {
+        set(remoteParticipantIdsAtom, [...currentIds, participant.id]);
+      }
+      set(remoteParticipantAtom(participant.id), participant);
+    }, [])
+  );
+
+  const removeParticipant = useAtomCallback(
+    useCallback((get, set, participant: Participant) => {
+      if (participant.local) {
+        set(localParticipantAtom, null);
+        return;
+      }
+
+      if (participant.id === get(botIdAtom)) {
+        set(botParticipantAtom, null);
+        return;
+      }
+
+      const currentIds = get(remoteParticipantIdsAtom);
+      set(
+        remoteParticipantIdsAtom,
+        currentIds.filter((id) => id !== participant.id)
+      );
+
+      // Clean up participant data
+      set(remoteParticipantAtom(participant.id), null);
+
+      // Clean up all track types for this participant
+      const trackTypes: TrackType[] = [
+        "audio",
+        "video",
+        "screenAudio",
+        "screenVideo",
+      ];
+      trackTypes.forEach((trackType) => {
+        const atom = trackAtom(`${participant.id}:${trackType}`);
+        set(atom, null);
+      });
+    }, [])
+  );
+
+  // Set up event listeners for participant events
+  useRTVIClientEvent(
+    RTVIEvent.ParticipantConnected,
+    useCallback(
+      (participant: Participant) => {
+        addParticipant(participant);
+      },
+      [addParticipant]
+    )
+  );
+
+  useRTVIClientEvent(
+    RTVIEvent.ParticipantLeft,
+    useCallback(
+      (participant: Participant) => {
+        removeParticipant(participant);
+      },
+      [removeParticipant]
+    )
+  );
+
+  // Handle bot connection to map bot UUID to our internal "bot" identifier
+  useRTVIClientEvent(
+    RTVIEvent.BotConnected,
+    useAtomCallback(
+      useCallback((_get, set, participant: Participant) => {
+        set(botIdAtom, participant.id);
+      }, [])
+    )
+  );
+
+  // Set up event listeners for media track events
+  const handleTrackStarted = useAtomCallback(
+    useCallback(
+      (get, set, track: MediaStreamTrack, participant?: Participant) => {
+        if (!participant) return;
+
+        const trackType = track.kind as TrackType;
+        // Map participant to our internal ID system
+        let internalId: string;
+        if (participant.local) {
+          internalId = "local";
+        } else {
+          // Check if this is the bot by comparing with stored bot ID
+          const botId = get(botIdAtom);
+          internalId =
+            botId && participant.id === botId ? "bot" : participant.id;
+        }
+        // Update track directly
+        const atom = trackAtom(`${internalId}:${trackType}`);
+        const oldTrack = get(atom);
+        if (oldTrack?.id === track.id) return;
+        set(atom, track);
+      },
+      []
+    )
+  );
+
+  const handleScreenTrackStarted = useAtomCallback(
+    useCallback(
+      (get, set, track: MediaStreamTrack, participant?: Participant) => {
+        if (!participant) return;
+
+        const trackType =
+          track.kind === "audio" ? "screenAudio" : "screenVideo";
+        // Map participant to our internal ID system
+        let internalId: string;
+        if (participant.local) {
+          internalId = "local";
+        } else {
+          // Check if this is the bot by comparing with stored bot ID
+          const botId = get(botIdAtom);
+          internalId =
+            botId && participant.id === botId ? "bot" : participant.id;
+        }
+        // Update track directly
+        const atom = trackAtom(`${internalId}:${trackType}`);
+        const oldTrack = get(atom);
+        if (oldTrack?.id === track.id) return;
+        set(atom, track);
+      },
+      []
+    )
+  );
+
+  useRTVIClientEvent(RTVIEvent.TrackStarted, handleTrackStarted);
+  useRTVIClientEvent(RTVIEvent.ScreenTrackStarted, handleScreenTrackStarted);
+
+  return null;
+});
+
+PipecatClientParticipantManager.displayName = "PipecatClientParticipantManager";
diff --git a/client-react/src/PipecatClientProvider.tsx b/client-react/src/PipecatClientProvider.tsx
index 92da9a7..d0a4c14 100644
--- a/client-react/src/PipecatClientProvider.tsx
+++ b/client-react/src/PipecatClientProvider.tsx
@@ -18,6 +18,7 @@ import {
   name as packageName,
   version as packageVersion,
 } from "../package.json";
+import { PipecatClientParticipantManager } from "./PipecatClientParticipantManager";
 import { PipecatClientStateProvider } from "./PipecatClientState";
 import { RTVIEventContext } from "./RTVIEventContext";
 
@@ -116,7 +117,10 @@ export const PipecatClientProvider: React.FC<
     <JotaiProvider store={jotaiStore}>
       <PipecatClientContext.Provider value={{ client }}>
         <RTVIEventContext.Provider value={{ on, off }}>
-          <PipecatClientStateProvider>{children}</PipecatClientStateProvider>
+          <PipecatClientStateProvider>
+            {children}
+            <PipecatClientParticipantManager />
+          </PipecatClientStateProvider>
         </RTVIEventContext.Provider>
       </PipecatClientContext.Provider>
     </JotaiProvider>
diff --git a/client-react/src/PipecatClientVideo.tsx b/client-react/src/PipecatClientVideo.tsx
index a51b618..9986787 100644
--- a/client-react/src/PipecatClientVideo.tsx
+++ b/client-react/src/PipecatClientVideo.tsx
@@ -15,9 +15,12 @@ interface PipecatClientVideoInterface {
   width: number;
 }
 
-export interface Props
-  extends Omit<React.VideoHTMLAttributes<HTMLVideoElement>, "onResize"> {
-  participant: "local" | "bot";
+type BaseProps = Omit<
+  React.VideoHTMLAttributes<HTMLVideoElement>,
+  "onResize"
+> & {
+  participant: "local" | "bot" | "remote";
+  participantId?: string;
 
   /**
    * Defines the video track type to display. Default: 'video'.
@@ -38,12 +41,22 @@ export interface Props
    * Returns the video's native width, height and aspectRatio.
    */
   onResize?(dimensions: PipecatClientVideoInterface): void;
-}
+};
+
+export type Props = BaseProps extends { participant: infer P }
+  ? P extends "remote"
+    ? BaseProps & { participantId: string }
+    : BaseProps
+  : BaseProps;
 
+/**
+ * Component that renders a video track
+ */
 export const PipecatClientVideo = forwardRef<HTMLVideoElement, Props>(
   function VoiceClientVideo(
     {
       participant = "local",
+      participantId,
       fit = "contain",
       mirror,
       onResize,
@@ -55,7 +68,8 @@ export const PipecatClientVideo = forwardRef<HTMLVideoElement, Props>(
   ) {
     const videoTrack: MediaStreamTrack | null = usePipecatClientMediaTrack(
       trackType,
-      participant
+      participant,
+      participantId
     );
 
     const videoEl = useRef<HTMLVideoElement>(null);
diff --git a/client-react/src/index.ts b/client-react/src/index.ts
index dc7c3a0..c9d54f4 100644
--- a/client-react/src/index.ts
+++ b/client-react/src/index.ts
@@ -14,6 +14,8 @@ import { usePipecatClientCamControl } from "./usePipecatClientCamControl";
 import { usePipecatClientMediaDevices } from "./usePipecatClientMediaDevices";
 import { usePipecatClientMediaTrack } from "./usePipecatClientMediaTrack";
 import { usePipecatClientMicControl } from "./usePipecatClientMicControl";
+import { usePipecatClientParticipant } from "./usePipecatClientParticipant";
+import { usePipecatClientParticipantIds } from "./usePipecatClientParticipantIds";
 import { usePipecatClientTransportState } from "./usePipecatClientTransportState";
 import { useRTVIClientEvent } from "./useRTVIClientEvent";
 import { VoiceVisualizer } from "./VoiceVisualizer";
@@ -29,6 +31,8 @@ export {
   usePipecatClientMediaDevices,
   usePipecatClientMediaTrack,
   usePipecatClientMicControl,
+  usePipecatClientParticipant,
+  usePipecatClientParticipantIds,
   usePipecatClientTransportState,
   useRTVIClientEvent,
   VoiceVisualizer,
diff --git a/client-react/src/usePipecatClientMediaTrack.ts b/client-react/src/usePipecatClientMediaTrack.ts
index 1ecd69e..1238329 100644
--- a/client-react/src/usePipecatClientMediaTrack.ts
+++ b/client-react/src/usePipecatClientMediaTrack.ts
@@ -4,107 +4,42 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
-import { Participant, RTVIEvent, Tracks } from "@pipecat-ai/client-js";
-import { atom, useAtomValue } from "jotai";
-import { atomFamily, useAtomCallback } from "jotai/utils";
-import { PrimitiveAtom } from "jotai/vanilla";
-import { useCallback, useEffect } from "react";
+import { Tracks } from "@pipecat-ai/client-js";
+import { useAtomValue } from "jotai";
+import { useMemo } from "react";
 
-import { usePipecatClient } from "./usePipecatClient";
-import { useRTVIClientEvent } from "./useRTVIClientEvent";
+import { trackAtom } from "./PipecatClientParticipantManager";
 
-type ParticipantType = keyof Tracks;
+type ParticipantType = keyof Tracks | "remote";
 type TrackType = keyof Tracks["local"];
 
-const localAudioTrackAtom = atom<MediaStreamTrack | null>(null);
-const localVideoTrackAtom = atom<MediaStreamTrack | null>(null);
-const localScreenAudioTrackAtom = atom<MediaStreamTrack | null>(null);
-const localScreenVideoTrackAtom = atom<MediaStreamTrack | null>(null);
-const botAudioTrackAtom = atom<MediaStreamTrack | null>(null);
-const botVideoTrackAtom = atom<MediaStreamTrack | null>(null);
-
-const trackAtom = atomFamily<
-  { local: boolean; trackType: TrackType },
-  PrimitiveAtom<MediaStreamTrack | null>
->(({ local, trackType }) => {
-  if (local) {
-    switch (trackType) {
-      case "audio":
-        return localAudioTrackAtom;
-      case "screenAudio":
-        return localScreenAudioTrackAtom;
-      case "screenVideo":
-        return localScreenVideoTrackAtom;
-      case "video":
-        return localVideoTrackAtom;
-    }
-  }
-  return trackType === "audio" ? botAudioTrackAtom : botVideoTrackAtom;
-});
-
+/**
+ * Hook to get individual participant media track
+ */
 export const usePipecatClientMediaTrack = (
   trackType: TrackType,
-  participantType: ParticipantType
+  participantType: ParticipantType,
+  participantId?: string
 ) => {
-  const client = usePipecatClient();
-  const track = useAtomValue(
-    trackAtom({ local: participantType === "local", trackType })
-  );
-
-  const updateTrack = useAtomCallback(
-    useCallback(
-      (
-        get,
-        set,
-        track: MediaStreamTrack,
-        trackType: TrackType,
-        local: boolean
-      ) => {
-        const atom = trackAtom({
-          local,
-          trackType,
-        });
-        const oldTrack = get(atom);
-        if (oldTrack?.id === track.id) return;
-        set(atom, track);
-      },
-      []
-    )
-  );
-
-  useRTVIClientEvent(
-    RTVIEvent.TrackStarted,
-    useCallback(
-      (track: MediaStreamTrack, participant?: Participant) => {
-        updateTrack(
-          track,
-          track.kind as TrackType,
-          Boolean(participant?.local)
-        );
-      },
-      [updateTrack]
-    )
-  );
+  // Memoize the track key to prevent infinite re-renders
+  const trackKey = useMemo((): `${string}:${TrackType}` => {
+    let actualParticipantId: string;
+
+    if (participantType === "local") {
+      actualParticipantId = "local";
+    } else if (participantType === "bot") {
+      actualParticipantId = "bot";
+    } else if (participantType === "remote" && participantId) {
+      actualParticipantId = participantId;
+    } else {
+      // Fallback for invalid combinations
+      actualParticipantId = "local";
+    }
 
-  useRTVIClientEvent(
-    RTVIEvent.ScreenTrackStarted,
-    useCallback(
-      (track: MediaStreamTrack, participant?: Participant) => {
-        const trackType =
-          track.kind === "audio" ? "screenAudio" : "screenVideo";
-        updateTrack(track, trackType, Boolean(participant?.local));
-      },
-      [updateTrack]
-    )
-  );
+    return `${actualParticipantId}:${trackType}`;
+  }, [trackType, participantType, participantId]);
 
-  useEffect(() => {
-    if (!client) return;
-    const tracks = client.tracks();
-    const track = tracks?.[participantType]?.[trackType];
-    if (!track) return;
-    updateTrack(track, trackType, participantType === "local");
-  }, [participantType, trackType, updateTrack, client]);
+  const track = useAtomValue(trackAtom(trackKey));
 
   return track;
 };
diff --git a/client-react/src/usePipecatClientParticipant.ts b/client-react/src/usePipecatClientParticipant.ts
new file mode 100644
index 0000000..602f308
--- /dev/null
+++ b/client-react/src/usePipecatClientParticipant.ts
@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) 2025, Daily.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+import { useAtomValue } from "jotai";
+
+import { participantAtom } from "./PipecatClientParticipantManager";
+
+/**
+ * Hook to get individual participant data
+ */
+export const usePipecatClientParticipant = (participantId: string) => {
+  return useAtomValue(participantAtom(participantId));
+};
diff --git a/client-react/src/usePipecatClientParticipantIds.ts b/client-react/src/usePipecatClientParticipantIds.ts
new file mode 100644
index 0000000..a7cea22
--- /dev/null
+++ b/client-react/src/usePipecatClientParticipantIds.ts
@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) 2025, Daily.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+import { useAtomValue } from "jotai";
+
+import { remoteParticipantIdsAtom } from "./PipecatClientParticipantManager";
+
+/**
+ * Hook to get all participant IDs
+ */
+export const usePipecatClientParticipantIds = (
+  includeLocal = true,
+  includeBot = true
+) => {
+  const remoteParticipantIds = useAtomValue(remoteParticipantIdsAtom);
+
+  return {
+    participantIds: [
+      ...(includeLocal ? ["local"] : []),
+      ...(includeBot ? ["bot"] : []),
+      ...remoteParticipantIds,
+    ],
+    remoteParticipantIds,
+  };
+};

From 5b9d2cc2069ed12edb240d94be1040ace8602600 Mon Sep 17 00:00:00 2001
From: Christian Stuff <christian@daily.co>
Date: Wed, 17 Sep 2025 13:19:16 +0200
Subject: [PATCH 2/2] add changelog entries

---
 client-react/CHANGELOG.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/client-react/CHANGELOG.md b/client-react/CHANGELOG.md
index e62f116..d7a0b89 100644
--- a/client-react/CHANGELOG.md
+++ b/client-react/CHANGELOG.md
@@ -5,6 +5,22 @@ All notable changes to **Pipecat Client React** will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- **Multi-participant support**: Full support for handling multiple participants in sessions
+  - New `usePipecatClientParticipant()` hook to access individual participant data
+  - New `usePipecatClientParticipantIds()` hook to get all participant IDs with filtering options
+  - Support for both regular and screen share media tracks per participant
+
+### Changed
+
+- `PipecatClientVideo` component now requires `participantId` prop when `participant` is set to `"remote"`
+- Enhanced `PipecatClientAudio` component now renders audio for all participants automatically
+- Enhanced `PipecatClientVideo` component now supports remote participants
+- `usePipecatClientMediaTrack()` hook signature updated to support remote participants
+
 ## [1.0.1]
 
 - Fixed state synchronization between different instances of `usePipecatClientCamControl()`, `usePipecatClientMicControl()` and `usePipecatClientTransportState()` ([#125](https://github.com/pipecat-ai/pipecat-client-web/pull/125))