Discussions

I'm using Stream API for POC and while it changes to idle video multiple times when WebRTC stream isn't being received, I'm experiencing a visible jerk/white splash when avatar changes. Here is the code snippet.

Is there any standard way to do this?

'use strict';

const DID_API = {
    "key":"<Key>",
    "url": "https://api.d-id.com",
    "service": "talks"
}

const RTCPeerConnection = (
    window.RTCPeerConnection ||
        window.webkitRTCPeerConnection ||
        window.mozRTCPeerConnection
).bind(window);

let peerConnection;
let streamId;
let sessionId;
let sessionClientAnswer;

let statsIntervalId;
let intervalId;
let videoIsPlaying;
let lastBytesReceived;

const videoElement = document.getElementById('video-element');
videoElement.setAttribute('playsinline', '');
const peerStatusLabel = document.getElementById('peer-status-label');
const iceStatusLabel = document.getElementById('ice-status-label');
const iceGatheringStatusLabel = document.getElementById('ice-gathering-status-label');
const signalingStatusLabel = document.getElementById('signaling-status-label');
const streamingStatusLabel = document.getElementById('streaming-status-label');

// List of sentences with associated emotions
const sentencesWithEmotions = [
    { sentence: "Hello there! I'm excited to chat with you today.", emotion: "happy" },
    { sentence: "Did you know that the average human can produce a quart of saliva a day?", emotion: "surprise" },
    { sentence: "Life is a journey, not a destination. Enjoy every moment!", emotion: "happy" },
    { sentence: "I find joy in helping you discover new things.", emotion: "happy" },
    { sentence: "Unexpected challenges can lead to remarkable opportunities.", emotion: "serious" },
    // Add more sentences with emotions as needed
];

const presenterInputByService = {
    talks: {
				source_url: 'https://d-id-public-bucket.s3.amazonaws.com/or-roman.jpg',
    },
    clips: {
        presenter_id: 'rian-lZC6MmWfC1',
        driver_id: 'mXra4jY38i'
    }
}

const connectButton = document.getElementById('connect-button');
connectButton.onclick = async () => {
    if (peerConnection && peerConnection.connectionState === 'connected') {
        return;
    }

    stopAllStreams();
    closePC();

    const sessionResponse = await fetchWithRetries(`${DID_API.url}/${DID_API.service}/streams`, {
        method: 'POST',
        headers: {
            Authorization: `Basic ${DID_API.key}`,
            'Content-Type': 'application/json',
        },
        body: JSON.stringify(presenterInputByService[DID_API.service]),
    });

    const { id: newStreamId, offer, ice_servers: iceServers, session_id: newSessionId } = await sessionResponse.json();
    streamId = newStreamId;
    sessionId = newSessionId;

    try {
        sessionClientAnswer = await createPeerConnection(offer, iceServers);
    } catch (e) {
        console.log('error during streaming setup', e);
        stopAllStreams();
        closePC();
        return;
    }

    const sdpResponse = await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}/sdp`, {
        method: 'POST',
        headers: {
            Authorization: `Basic ${DID_API.key}`,
            'Content-Type': 'application/json',
        },
        body: JSON.stringify({
            answer: sessionClientAnswer,
            session_id: sessionId,
        }),
    });
};

async function sendText(input_text, emotion) {

    // connectionState not supported in firefox
    if (peerConnection?.signalingState === 'stable' || peerConnection?.iceConnectionState === 'connected') {
        console.log('we are under sendText')
        const playResponse = await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}`, {
            method: 'POST',
            headers: {
                Authorization: `Basic ${DID_API.key}`,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
        	           script: {
                    type: 'text',
                    subtitles: 'false',
                    provider: {
                        type: 'microsoft',
                        voice_id: 'en-US-JennyNeural'
                    },
                    ssml: 'false',
                    input: input_text
                },
                config: {
                    stitch: false,
                    driver_expressions: {
                        expressions: [
                            {
                             start_frame: 0,
                             expression: emotion,
                             intensity: 1,
                            },
                        ],
                        transition_frames: 150,
                    }
                },
                session_id: sessionId,
            }),
        });
        console.log('Response from /talk/streams is: ', playResponse)
    }
}

const startButton = document.getElementById('start-button');
startButton.onclick = async () => {
    // connectionState not supported in firefox
    if (peerConnection?.signalingState === 'stable' || peerConnection?.iceConnectionState === 'connected') {
        console.log('we are under sendText')
        const playResponse = await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}`, {
            method: 'POST',
            headers: {
                Authorization: `Basic ${DID_API.key}`,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
        	             script: {
                    type: 'text',
                    subtitles: 'false',
                    provider: {
                        type: 'microsoft',
                        voice_id: 'en-US-JennyNeural'
                    },
                    ssml: 'false',
                    input: 'Hey Welcome, I am here to help you out!' 
                },
                config: {
                    stitch: false,
                    driver_expressions: {
                        expressions: [
                            {
                             start_frame: 0,
                             expression: 'happy',
                             intensity: 1,
                            },
                        ],
                        transition_frames: 150,
                    }
                },
                session_id: sessionId,
            }),
        });
    intervalId = setInterval(async () => {
     const randomIndex = Math.floor(Math.random() * sentencesWithEmotions.length);
            const { sentence, emotion } = sentencesWithEmotions[randomIndex];
            console.log(sentence, emotion);
            sendText(sentence, emotion);
    }, 10000);
}};

const destroyButton = document.getElementById('destroy-button');
destroyButton.onclick = async () => {
    await fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}`, {
        method: 'DELETE',
        headers: {
            Authorization: `Basic ${DID_API.key}`,
            'Content-Type': 'application/json',
        },
        body: JSON.stringify({ session_id: sessionId }),
    });

    stopAllStreams();
    closePC();
};

function onIceGatheringStateChange() {
    iceGatheringStatusLabel.innerText = peerConnection.iceGatheringState;
    iceGatheringStatusLabel.className = 'iceGatheringState-' + peerConnection.iceGatheringState;
}
function onIceCandidate(event) {
    console.log('onIceCandidate', event);
    if (event.candidate) {
        const { candidate, sdpMid, sdpMLineIndex } = event.candidate;

        fetch(`${DID_API.url}/${DID_API.service}/streams/${streamId}/ice`, {
            method: 'POST',
            headers: {
                Authorization: `Basic ${DID_API.key}`,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
                candidate,
                sdpMid,
                sdpMLineIndex,
                session_id: sessionId,
            }),
        });
    }
}
function onIceConnectionStateChange() {
    iceStatusLabel.innerText = peerConnection.iceConnectionState;
    iceStatusLabel.className = 'iceConnectionState-' + peerConnection.iceConnectionState;
    if (peerConnection.iceConnectionState === 'failed' || peerConnection.iceConnectionState === 'closed') {
        stopAllStreams();
        closePC();
    }
}
function onConnectionStateChange() {
    // not supported in firefox
    peerStatusLabel.innerText = peerConnection.connectionState;
    peerStatusLabel.className = 'peerConnectionState-' + peerConnection.connectionState;
}
function onSignalingStateChange() {
    signalingStatusLabel.innerText = peerConnection.signalingState;
    signalingStatusLabel.className = 'signalingState-' + peerConnection.signalingState;
}

function onVideoStatusChange(videoIsPlaying, stream) {
    let status;
    if (videoIsPlaying) {
        status = 'streaming';
        console.log(stream)
        const remoteStream = stream;
        setVideoElement(remoteStream);
    } else {
        status = 'empty';
        playIdleVideo();
    }
    streamingStatusLabel.innerText = status;
    streamingStatusLabel.className = 'streamingState-' + status;
}

function onTrack(event) {
    /**
   * The following code is designed to provide information about wether currently there is data
   * that's being streamed - It does so by periodically looking for changes in total stream data size
   *
   * This information in our case is used in order to show idle video while no video is streaming.
   * To create this idle video use the POST https://api.d-id.com/talks (or clips) endpoint with a silent audio file or a text script with only ssml breaks
   * https://docs.aws.amazon.com/polly/latest/dg/supportedtags.html#break-tag
   * for seamless results use `config.fluent: true` and provide the same configuration as the streaming video
   */
    console.log('we are under onTrack')

    if (!event.track) return;

    statsIntervalId = setInterval(async () => {
        const stats = await peerConnection.getStats(event.track);
        stats.forEach((report) => {
            if (report.type === 'inbound-rtp' && report.mediaType === 'video') {
                const videoStatusChanged = videoIsPlaying !== report.bytesReceived > lastBytesReceived;

                if (videoStatusChanged) {
                    videoIsPlaying = report.bytesReceived > lastBytesReceived;
                    onVideoStatusChange(videoIsPlaying, event.streams[0]);
                }
                lastBytesReceived = report.bytesReceived;
            }
        });
    }, 500);
}

async function createPeerConnection(offer, iceServers) {
    if (!peerConnection) {
        peerConnection = new RTCPeerConnection({ iceServers });
        peerConnection.addEventListener('icegatheringstatechange', onIceGatheringStateChange, true);
        peerConnection.addEventListener('icecandidate', onIceCandidate, true);
        peerConnection.addEventListener('iceconnectionstatechange', onIceConnectionStateChange, true);
        peerConnection.addEventListener('connectionstatechange', onConnectionStateChange, true);
        peerConnection.addEventListener('signalingstatechange', onSignalingStateChange, true);
        peerConnection.addEventListener('track', onTrack, true);
    }

    await peerConnection.setRemoteDescription(offer);
    console.log('set remote sdp OK');

    const sessionClientAnswer = await peerConnection.createAnswer();
    console.log('create local sdp OK');

    await peerConnection.setLocalDescription(sessionClientAnswer);
    console.log('set local sdp OK');

    return sessionClientAnswer;
}

function setVideoElement(stream) {
    if (!stream) return;
    videoElement.srcObject = stream;
    videoElement.loop = false;

    // safari hotfix
    if (videoElement.paused) {
        videoElement
            .play()
            .then((_) => {})
            .catch((e) => {});
    }
}

function playIdleVideo() {
    videoElement.srcObject = undefined;
    videoElement.src = 'test_idle.mp4';
    videoElement.loop = true;
}

function stopAllStreams() {
    if (videoElement.srcObject) {
        console.log('stopping video streams');
        videoElement.srcObject.getTracks().forEach((track) => track.stop());
        videoElement.srcObject = null;
    }
}

function closePC(pc = peerConnection) {
    if (!pc) return;
    console.log('stopping peer connection');
    pc.close();
    pc.removeEventListener('icegatheringstatechange', onIceGatheringStateChange, true);
    pc.removeEventListener('icecandidate', onIceCandidate, true);
    pc.removeEventListener('iceconnectionstatechange', onIceConnectionStateChange, true);
    pc.removeEventListener('connectionstatechange', onConnectionStateChange, true);
    pc.removeEventListener('signalingstatechange', onSignalingStateChange, true);
    pc.removeEventListener('track', onTrack, true);
    clearInterval(statsIntervalId);
    clearInterval(intervalId);
    iceGatheringStatusLabel.innerText = '';
    signalingStatusLabel.innerText = '';
    iceStatusLabel.innerText = '';
    peerStatusLabel.innerText = '';
    console.log('stopped peer connection');
    if (pc === peerConnection) {
        peerConnection = null;
    }
}

const maxRetryCount = 3;
const maxDelaySec = 4;

async function fetchWithRetries(url, options, retries = 1) {
    try {
        return await fetch(url, options);
    } catch (err) {
        if (retries <= maxRetryCount) {
            const delay = Math.min(Math.pow(2, retries) / 4 + Math.random(), maxDelaySec) * 1000;

            await new Promise((resolve) => setTimeout(resolve, delay));

            console.log(`Request failed, retrying ${retries}/${maxRetryCount}. Error ${err}`);
            return fetchWithRetries(url, options, retries + 1);
        } else {
            throw new Error(`Max retries exceeded. error: ${err}`);
        }
    }
}

Looking forward to hearing back from you soon!