Discussions

Ask a Question
Back to all

agents streaming

agents, i am unable to see my generated video when i use clip type of agents , api give success response but i can't see the generated video , but when i use talk type agent then it's working fine .

import React, { useEffect, useRef, useState } from "react";

const DID_API_KEY = "My api key";
const BASE_URL = "https://api.d-id.com";
const AGENT_ID = "agent_id";
function App() {
const videoRef = useRef(null);
const [chatId, setChatId] = useState("");
const [streamId, setStreamId] = useState("");
const [sessionId, setSessionId] = useState("");
const [peerConnection, setPeerConnection] = useState(null);
const [message, setMessage] = useState("");
const dataChannelRef = useRef(null);

const [chatLoading, setChatLoading] = useState(false);
const [streamLoading, setStreamLoading] = useState(false);
const [sendLoading, setSendLoading] = useState(false);
const [streamConnected, setStreamConnected] = useState(false);

const [toast, setToast] = useState({ show: false, msg: "", type: "info" });

const showToast = (msg, type= "info") => {
setToast({ show: true, msg, type });
setTimeout(() => setToast({ show: false, msg: "", type: "" }), 3000);
};

const headers = {
Authorization: DID_API_KEY,
"Content-Type": "application/json",
};

const startWebRTCStream = async () => {
setStreamLoading(true);
try {
const res = await fetch(${BASE_URL}/agents/${AGENT_ID}/streams, {
method: "POST",
headers,
});

  const { id, offer, ice_servers, session_id } = await res.json();
  setStreamId(id);
  setSessionId(session_id);

  const pc = new RTCPeerConnection({ iceServers: ice_servers });
  setPeerConnection(pc);

  const dataChannel = pc.createDataChannel("JanusDataChannel", {
    ordered: true,
    maxRetransmits: 3,
  });
  dataChannelRef.current = dataChannel;

  pc.ontrack = (evt) => {
    const remoteStream = evt.streams[0];
    console.log("Track received:", remoteStream);
    if (videoRef.current && remoteStream) {
      videoRef.current.srcObject = remoteStream;
      videoRef.current.play().catch((err) => {
        console.error("Play error:", err);
      });
    }
  };

  pc.onicecandidate = (event) => {
    if (event.candidate) {
      fetch(`${BASE_URL}/agents/${AGENT_ID}/streams/${id}/ice`, {
        method: "POST",
        headers,
        body: JSON.stringify({
          candidate: event.candidate.candidate,
          sdpMid: event.candidate.sdpMid,
          sdpMLineIndex: event.candidate.sdpMLineIndex,
          session_id,
        }),
      });
    }
  };

  pc.oniceconnectionstatechange = () => {
    const state = pc.iceConnectionState;
    if (state === "connected" || state === "completed") {
      setStreamConnected(true);
      showToast(" Stream Connected", "success");
    } else if (["disconnected", "failed", "closed"].includes(state)) {
      setStreamConnected(false);
      showToast(" Stream Disconnected", "warning");
    }
  };

  dataChannel.onmessage = (event) => {
    console.log(event,401)
    if (event.data === "stream/start") {

      showToast(" Stream is starting...", "info");
    } else if (event.data === "stream/done") {
      showToast("Stream finished", "info");
    }
  };

  await pc.setRemoteDescription(offer);
  const answer = await pc.createAnswer();
  await pc.setLocalDescription(answer);

  await fetch(`${BASE_URL}/agents/${AGENT_ID}/streams/${id}/sdp`, {
    method: "POST",
    headers,
    body: JSON.stringify({ answer, session_id }),
  });
} catch (err) {
  console.error("Stream error:", err);
  showToast(" Stream connection failed", "error");
} finally {
  setStreamLoading(false);
}

};

const sendMessage = async () => {
if (!message || !peerConnection) {
showToast(" Chat or stream not ready", "warning");
return;
}

setSendLoading(true);
try {
  await fetch(`${BASE_URL}/agents/${AGENT_ID}/streams/${streamId}`, {
    method: "POST",
    headers,
    body: JSON.stringify({
       session_id:sessionId,
      script: { type: "text", input: message},
    }),
  });
  setMessage("");
} catch {
  showToast(" Message failed", "error");
} finally {
  setSendLoading(false);
}

};

const destroyStream = async () => {
if (streamId && sessionId) {
await fetch(${BASE_URL}/talks/streams/${streamId}, {
method: "DELETE",
headers,
body: JSON.stringify({ session_id: sessionId }),
});
}

peerConnection?.close();
videoRef.current && (videoRef.current.srcObject = null);
setStreamConnected(false);
setStreamId("");
setSessionId("");
showToast(" Stream destroyed", "warning");

};

useEffect(() => {
return () => {
destroyStream();
};
}, []);

const toastColors = {
success: "bg-green-600",
error: "bg-red-600",
info: "bg-blue-600",
warning: "bg-yellow-400 text-black",
};

return (



🎥 D-ID Streaming Agent

  <video
    ref={videoRef}
    playsInline
    autoPlay
    className="rounded-xl border shadow-lg mb-4  max-w-md w-96 h-96 object-contain"
  />

  <div className="flex gap-4 mb-4">
  
    <button
      onClick={startWebRTCStream}
      disabled={streamConnected || streamLoading}
      className={`px-4 py-2 rounded-lg font-semibold ${
        streamConnected
          ? "bg-green-600 text-white cursor-not-allowed"
          : streamLoading
          ? "bg-blue-300 text-white"
          : "bg-blue-600 text-white hover:bg-blue-700"
      }`}
    >
      {streamConnected
        ? "Connected"
        : streamLoading
        ? "Connecting..."
        : "Connect"}
    </button>

    <button
      onClick={destroyStream}
      className="px-4 py-2 bg-red-500 text-white rounded-lg hover:bg-red-600"
    >
      Destroy
    </button>
  </div>

  <div className="w-full max-w-md">
    <textarea
      rows={3}
      value={message}
      onChange={(e) => setMessage(e.target.value)}
      placeholder="Type a message..."
      className="w-full p-3 rounded-lg border border-gray-300"
    />
    <button
      onClick={sendMessage}
      disabled={sendLoading}
      className={`mt-3 w-full py-2 rounded-lg font-semibold ${
        sendLoading
          ? "bg-green-300 text-green-900"
          : "bg-green-600 text-white hover:bg-green-700"
      }`}
    >
      {sendLoading ? "Sending..." : "Send"}
    </button>
  </div>

  {toast.show && (
    <div
      className={`fixed top-5 left-5 z-50 px-6 py-3 rounded-xl shadow-xl text-white ${toastColors[toast.type]} animate-bounce`}
    >
      {toast.msg}
    </div>
  )}
</div>

);
}

export default App;