From 1423b86623b6245b28e5aa0f5efbc4b573216262 Mon Sep 17 00:00:00 2001 From: Miroslav Pejic Date: Thu, 13 Jan 2022 19:07:49 +0100 Subject: [PATCH] [mirotalk] - refactoring live caption feature --- app/src/server.js | 11 +- public/css/client.css | 12 +- public/js/client.js | 336 +++++++++++------- public/js/speechRecognition.js | 79 +++++ public/js/speech_recognition.js | 38 --- public/js/volume_meter.js | 96 ------ public/sounds/speech.mp3 | Bin 0 -> 2595 bytes public/view/client.html | 588 +++++++++++++++++--------------- 8 files changed, 601 insertions(+), 559 deletions(-) create mode 100644 public/js/speechRecognition.js delete mode 100644 public/js/speech_recognition.js delete mode 100644 public/js/volume_meter.js create mode 100644 public/sounds/speech.mp3 diff --git a/app/src/server.js b/app/src/server.js index 7591d8e6..612dfe14 100755 --- a/app/src/server.js +++ b/app/src/server.js @@ -221,7 +221,7 @@ function getMeetingURL(host) { // end of MiroTalk API v1 // not match any of page before, so 404 not found -app.get('*', function(req, res) { +app.get('*', function (req, res) { res.sendFile(path.join(__dirname, '../../', 'public/view/404.html')); }); @@ -695,13 +695,6 @@ io.sockets.on('connect', (socket) => { let room_id = config.room_id; sendToRoom(room_id, socket.id, 'whiteboardAction', config); }); - - // for live subtitle or transcripts - socket.on('speech_transcript', (config) => { - log.debug(config); - let room_id = config.room_id; - sendToRoom(room_id, socket.id, 'speech_transcript', config); - }); }); // end [sockets.on-connect] /** @@ -731,4 +724,4 @@ async function sendToPeer(peer_id, sockets, msg, config = {}) { if (peer_id in sockets) { await sockets[peer_id].emit(msg, config); } -} \ No newline at end of file +} diff --git a/public/css/client.css b/public/css/client.css index a6ebd22c..38747b5b 100755 --- a/public/css/client.css +++ b/public/css/client.css @@ -982,7 +982,9 @@ video:fullscreen { #myPeerNameSetBtn, #muteEveryoneBtn, #hideEveryoneBtn, -#lockUnlockRoomBtn { +#lockUnlockRoomBtn, +#speechRecognitionStart, +#speechRecognitionStop { padding: 5px; border-radius: 5px; color: white; @@ -990,7 +992,9 @@ video:fullscreen { } #myPeerNameSetBtn:hover, -#lockUnlockRoomBtn:hover { +#lockUnlockRoomBtn:hover, +#speechRecognitionStart:hover, +#speechRecognitionStop:hover { color: var(--hover-color); transform: var(--btns-hover-scale); transition: all 0.3s ease-in-out; @@ -1007,7 +1011,7 @@ video:fullscreen { .tab { overflow: hidden; border: 1px solid rgb(0, 0, 0); - background-color: rgba(0, 0, 0, 0.7); + background-color: rgba(0, 0, 0); } /* Style the buttons inside the tab */ @@ -1031,7 +1035,7 @@ video:fullscreen { /* Create an active/current tablink class */ .tab button.active { - background-color: rgba(0, 0, 0, 0.7); + background-color: rgb(30 29 29); } /* Style the tab content */ diff --git a/public/js/client.js b/public/js/client.js index 19f6e9c4..7bbddecb 100644 --- a/public/js/client.js +++ b/public/js/client.js @@ -85,7 +85,7 @@ let myVideoStatus = true; let myAudioStatus = true; let isScreenStreaming = false; let isChatRoomVisible = false; -let isCaptionBoxVisible = false; //added for caption button +let isCaptionBoxVisible = false; let isChatEmojiVisible = false; let isButtonsVisible = false; let isMySettingsVisible = false; @@ -134,7 +134,7 @@ let screenShareBtn; let recordStreamBtn; let fullScreenBtn; let chatRoomBtn; -let captionBtn; //for text transcript +let captionBtn; let myHandBtn; let whiteboardBtn; let fileShareBtn; @@ -188,6 +188,8 @@ let screenFpsSelect; let themeSelect; let btnsBarSelect; let selectors; +let speechRecognitionStart; +let speechRecognitionStop; // my video element let myVideo; let myVideoWrap; @@ -270,7 +272,7 @@ function getHtmlElementsById() { screenShareBtn = getId('screenShareBtn'); recordStreamBtn = getId('recordStreamBtn'); fullScreenBtn = getId('fullScreenBtn'); - captionBtn = getId('captionBtn'); //for getting caption buttons + captionBtn = getId('captionBtn'); chatRoomBtn = getId('chatRoomBtn'); whiteboardBtn = getId('whiteboardBtn'); fileShareBtn = getId('fileShareBtn'); @@ -324,6 +326,8 @@ function getHtmlElementsById() { screenFpsSelect = getId('screenFps'); themeSelect = getId('mirotalkTheme'); btnsBarSelect = getId('mirotalkBtnsBar'); + speechRecognitionStart = getId('speechRecognitionStart'); + speechRecognitionStop = getId('speechRecognitionStop'); // my conference name, hand, video - audio status myVideoParagraph = getId('myVideoParagraph'); myHandStatusIcon = getId('myHandStatusIcon'); @@ -401,9 +405,8 @@ function setButtonsTitle() { content: 'OPEN the chat', placement: 'right-start', }); - //for hover pop-over tippy(captionBtn, { - content: 'See Caption', + content: 'OPEN the caption', placement: 'right-start', }); tippy(myHandBtn, { @@ -445,7 +448,7 @@ function setButtonsTitle() { content: 'Save messages', }); tippy(msgerClose, { - content: 'Close the chat', + content: 'Close', }); tippy(msgerEmojiBtn, { content: 'Emoji', @@ -454,6 +457,20 @@ function setButtonsTitle() { content: 'Send', }); + // caption buttons + tippy(captionTheme, { + content: 'Ghost theme', + }); + tippy(captionClean, { + content: 'Clean messages', + }); + tippy(captionSaveBtn, { + content: 'Save messages', + }); + tippy(msgerClose, { + content: 'Close', + }); + // settings tippy(mySettingsCloseBtn, { content: 'Close settings', @@ -462,6 +479,20 @@ function setButtonsTitle() { content: 'Change name', }); + // tab btns + tippy(tabDevicesBtn, { + content: 'Devices', + }); + tippy(tabBandwidthBtn, { + content: 'Bandwidth', + }); + tippy(tabRoomBtn, { + content: 'Room', + }); + tippy(tabStylingBtn, { + content: 'Styling', + }); + // whiteboard btns tippy(wbDrawingColorEl, { content: 'DRAWING color', @@ -687,7 +718,6 @@ function initClientPeer() { signalingSocket.on('videoPlayer', handleVideoPlayer); signalingSocket.on('disconnect', handleDisconnect); signalingSocket.on('removePeer', handleRemovePeer); - signalingSocket.on('speech_transcript', handleSpeechTranscript); } // end [initClientPeer] /** @@ -703,7 +733,7 @@ async function sendToServer(msg, config = {}) { * Connected to Signaling Server. Once the user has given us access to their * microphone/cam, join the channel and start peering up */ -function handleConnect(socket) { +function handleConnect() { console.log('Connected to signaling server'); if (localMediaStream) joinToChannel(); else @@ -828,7 +858,8 @@ function welcomeUser() { title: 'Welcome ' + myPeerName + '', imageAlt: 'mirotalk-welcome', imageUrl: welcomeImg, - html: ` + html: + `

Share this meeting invite others to join.

` + @@ -959,9 +990,16 @@ function handleRTCDataChannels(peer_id) { case 'mirotalk_chat_channel': try { let dataMessage = JSON.parse(msg.data); - handleDataChannelChat(dataMessage); + switch (dataMessage.type) { + case 'chat': + handleDataChannelChat(dataMessage); + break; + case 'speech': + handleDataChannelSpeechTranscript(dataMessage); + break; + } } catch (err) { - console.error('handleDataChannelChat', err); + console.error('mirotalk_chat_channel', err); } break; case 'mirotalk_file_sharing_channel': @@ -969,7 +1007,7 @@ function handleRTCDataChannels(peer_id) { let dataFile = msg.data; handleDataChannelFileSharing(dataFile); } catch (err) { - console.error('handleDataChannelFS', err); + console.error('mirotalk_file_sharing_channel', err); } break; } @@ -1245,7 +1283,7 @@ function setTheme(theme) { document.documentElement.style.setProperty('--private-msg-bg', 'rgba(252, 110, 110, 0.7)'); document.documentElement.style.setProperty('--right-msg-bg', 'rgba(0, 0, 0, 0.7)'); break; - // ... + // ... default: console.log('No theme found'); } @@ -1822,6 +1860,7 @@ function manageLeftButtons() { setRecordStreamBtn(); setFullScreenBtn(); setChatRoomBtn(); + setCaptionRoomBtn(); setChatEmojiBtn(); setMyHandBtn(); setMyWhiteboardBtn(); @@ -1836,7 +1875,7 @@ function manageLeftButtons() { * Copy - share room url button click event */ function setShareRoomBtn() { - shareRoomBtn.addEventListener('click', async(e) => { + shareRoomBtn.addEventListener('click', async (e) => { shareRoomUrl(); }); } @@ -1934,7 +1973,7 @@ function setFullScreenBtn() { */ function setChatRoomBtn() { // adapt chat room size for mobile - setChatRoomForMobile(); + setChatRoomAndCaptionForMobile(); // open hide chat room chatRoomBtn.addEventListener('click', (e) => { @@ -1959,19 +1998,6 @@ function setChatRoomBtn() { } }); - // ghost theme + undo - captionTheme.addEventListener('click', (e) => { - if (mirotalkTheme == 'ghost') return; - - if (e.target.className == 'fas fa-ghost') { - e.target.className = 'fas fa-undo'; - document.documentElement.style.setProperty('--msger-bg', 'rgba(0, 0, 0, 0.100)'); - } else { - e.target.className = 'fas fa-ghost'; - document.documentElement.style.setProperty('--msger-bg', 'linear-gradient(to left, #383838, #000000)'); - } - }); - // show msger participants section msgerCPBtn.addEventListener('click', (e) => { if (!thereIsPeerConnections()) { @@ -1991,11 +2017,6 @@ function setChatRoomBtn() { cleanMessages(); }); - // clean caption transcripts - captionClean.addEventListener('click', (e) => { - cleanCaptions(); - }); - // save chat messages to file msgerSaveBtn.addEventListener('click', (e) => { if (chatMessages.length != 0) { @@ -2005,37 +2026,12 @@ function setChatRoomBtn() { userLog('info', 'No chat messages to save'); }); - // save caption transcripts to file - captionSaveBtn.addEventListener('click', (e) => { - if (transcripts.length != 0) { - downloadCaptions(); - return; - } - userLog('info', 'No captions to save'); - }); - - // open hide chat room - captionBtn.addEventListener('click', (e) => { - if (!isCaptionBoxVisible) { - showCaptionDraggable(); - } else { - //hideChatRoomAndEmojiPicker(); - e.target.className = 'fas fa-closed-captioning'; - } - }); - - // close chat room - show left button and status menu if hide msgerClose.addEventListener('click', (e) => { hideChatRoomAndEmojiPicker(); showButtonsBarAndMenu(); }); - // close caption box - show left button and status menu if hide - captionClose.addEventListener('click', (e) => { - hideCaptionBox(); - showButtonsBarAndMenu(); - }); // open Video Url Player msgerVideoUrlBtn.addEventListener('click', (e) => { sendVideoUrl(); @@ -2051,7 +2047,7 @@ function setChatRoomBtn() { }); // on input check 4emoji from map - msgerInput.oninput = function() { + msgerInput.oninput = function () { for (let i in chatInputEmoji) { let regex = new RegExp(escapeSpecialChars(i), 'gim'); this.value = this.value.replace(regex, chatInputEmoji[i]); @@ -2066,6 +2062,70 @@ function setChatRoomBtn() { }); } +/** + * Caption room buttons click event + */ +function setCaptionRoomBtn() { + if ('webkitSpeechRecognition' in window) { + // open hide caption + captionBtn.addEventListener('click', (e) => { + if (!isCaptionBoxVisible) { + showCaptionDraggable(); + } else { + hideCaptionBox(); + } + }); + } else { + captionBtn.style.display = 'none'; + // https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API#browser_compatibility + } + + // ghost theme + undo + captionTheme.addEventListener('click', (e) => { + if (mirotalkTheme == 'ghost') return; + + if (e.target.className == 'fas fa-ghost') { + e.target.className = 'fas fa-undo'; + document.documentElement.style.setProperty('--msger-bg', 'rgba(0, 0, 0, 0.100)'); + } else { + e.target.className = 'fas fa-ghost'; + document.documentElement.style.setProperty('--msger-bg', 'linear-gradient(to left, #383838, #000000)'); + } + }); + + // clean caption transcripts + captionClean.addEventListener('click', (e) => { + cleanCaptions(); + }); + + // save caption transcripts to file + captionSaveBtn.addEventListener('click', (e) => { + if (transcripts.length != 0) { + downloadCaptions(); + return; + } + userLog('info', 'No captions to save'); + }); + + // close caption box - show left button and status menu if hide + captionClose.addEventListener('click', (e) => { + hideCaptionBox(); + showButtonsBarAndMenu(); + }); + + // hide it + speechRecognitionStop.style.display = 'none'; + + // start recognition speech + speechRecognitionStart.addEventListener('click', (e) => { + startSpeech(true); + }); + // stop recognition speech + speechRecognitionStop.addEventListener('click', (e) => { + startSpeech(false); + }); +} + /** * Emoji picker chat room button click event */ @@ -2088,7 +2148,7 @@ function setChatEmojiBtn() { * Set my hand button click event */ function setMyHandBtn() { - myHandBtn.addEventListener('click', async(e) => { + myHandBtn.addEventListener('click', async (e) => { setMyHandStatus(); }); } @@ -2376,10 +2436,10 @@ function getVideoConstraints(videoQuality) { switch (videoQuality) { case 'useVideo': return useVideo; - // Firefox not support set frameRate (OverconstrainedError) O.o + // Firefox not support set frameRate (OverconstrainedError) O.o case 'default': return { frameRate: frameRate }; - // video cam constraints default + // video cam constraints default case 'qvgaVideo': return { width: { exact: 320 }, @@ -2581,7 +2641,13 @@ function attachMediaStream(element, stream) { * if mobile and mySettings open do nothing return */ function showButtonsBarAndMenu() { - if (isButtonsVisible || (isMobileDevice && isChatRoomVisible) || (isMobileDevice && isMySettingsVisible)) return; + if ( + isButtonsVisible || + (isMobileDevice && isChatRoomVisible) || + (isMobileDevice && isCaptionBoxVisible) || + (isMobileDevice && isMySettingsVisible) + ) + return; toggleClassElements('statusMenu', 'inline'); buttonsBar.style.display = 'flex'; isButtonsVisible = true; @@ -2628,7 +2694,8 @@ async function shareRoomUrl() { title: 'Share the Room', // imageAlt: 'mirotalk-share', // imageUrl: shareUrlImg, - html: ` + html: + `

@@ -2716,9 +2783,9 @@ function handleAudio(e, init, force = null) { localMediaStream.getAudioTracks()[0].enabled = force != null ? force : !localMediaStream.getAudioTracks()[0].enabled; myAudioStatus = localMediaStream.getAudioTracks()[0].enabled; - force != null ? - (e.className = 'fas fa-microphone' + (myAudioStatus ? '' : '-slash')) : - (e.target.className = 'fas fa-microphone' + (myAudioStatus ? '' : '-slash')); + force != null + ? (e.className = 'fas fa-microphone' + (myAudioStatus ? '' : '-slash')) + : (e.target.className = 'fas fa-microphone' + (myAudioStatus ? '' : '-slash')); if (init) { audioBtn.className = 'fas fa-microphone' + (myAudioStatus ? '' : '-slash'); if (!isMobileDevice) { @@ -2729,12 +2796,6 @@ function handleAudio(e, init, force = null) { } } setMyAudioStatus(myAudioStatus); - if (myAudioStatus) { - start_stop_speech(true); //speech to text transcript function - } else { - start_stop_speech(false); //speech to text transcript function - } - } /** @@ -2747,9 +2808,9 @@ function handleVideo(e, init, force = null) { localMediaStream.getVideoTracks()[0].enabled = force != null ? force : !localMediaStream.getVideoTracks()[0].enabled; myVideoStatus = localMediaStream.getVideoTracks()[0].enabled; - force != null ? - (e.className = 'fas fa-video' + (myVideoStatus ? '' : '-slash')) : - (e.target.className = 'fas fa-video' + (myVideoStatus ? '' : '-slash')); + force != null + ? (e.className = 'fas fa-video' + (myVideoStatus ? '' : '-slash')) + : (e.target.className = 'fas fa-video' + (myVideoStatus ? '' : '-slash')); if (init) { videoBtn.className = 'fas fa-video' + (myVideoStatus ? '' : '-slash'); if (!isMobileDevice) { @@ -3198,13 +3259,15 @@ function createChatDataChannel(peer_id) { /** * Set the chat room on full screen mode for mobile */ -function setChatRoomForMobile() { +function setChatRoomAndCaptionForMobile() { if (isMobileDevice) { document.documentElement.style.setProperty('--msger-height', '99%'); document.documentElement.style.setProperty('--msger-width', '99%'); } else { // make chat room draggable for desktop dragElement(msgerDraggable, msgerHeader); + // make caption draggable for desktop + dragElement(captionDraggable, captionHeader); } } @@ -3231,7 +3294,6 @@ function showChatRoomDraggable() { } } - /** * Show caption box draggable on center screen position */ @@ -3318,8 +3380,6 @@ function cleanCaptions() { }); } - - /** * Hide chat room and emoji picker */ @@ -3343,14 +3403,12 @@ function hideChatRoomAndEmojiPicker() { */ function hideCaptionBox() { captionDraggable.style.display = 'none'; - msgerEmojiPicker.style.display = 'none'; - captionBtn.className = 'far fa-closed-captioning'; + captionBtn.className = 'fas fa-closed-captioning'; isCaptionBoxVisible = false; - isChatEmojiVisible = false; // only for desktop if (!isMobileDevice) { tippy(captionBtn, { - content: 'OPEN the caption box', + content: 'OPEN the caption', placement: 'right-start', }); } @@ -3401,6 +3459,49 @@ function handleDataChannelChat(dataMessage) { appendMessage(msgFrom, leftChatAvatar, 'left', msg, msgPrivate); } +/** + * Handle text transcipt getting from peers + * @param {*} config + */ +function handleDataChannelSpeechTranscript(config) { + handleSpeechTranscript(config); +} + +/** + * Handle text transcipt getting from peers + * @param {*} data + */ +function handleSpeechTranscript(config) { + if (!config) return; + + let time_stamp = getFormatDate(new Date()); + let name = config.peer_name; + let avatar_image = avatarApiUrl + '?name=' + name + '&size=32' + '&background=random&rounded=true'; + let transcipt = config.text_data; + + console.log('Handle speech transcript', config); + + const msgHTML = ` +
+
+
+
+
${name} : ${time_stamp}
+
+
${transcipt}
+
+
+ `; + captionChat.insertAdjacentHTML('beforeend', msgHTML); + captionChat.scrollTop += 500; + transcripts.push({ + time: time_stamp, + name: name, + caption: transcipt, + }); + playSound('speech'); +} + /** * Escape Special Chars * @param {*} regex @@ -3578,6 +3679,7 @@ function emitMsg(from, to, msg, privateMsg) { if (!msg) return; let chatMessage = { + type: 'chat', from: from, to: to, msg: msg, @@ -4050,8 +4152,10 @@ function disableAllPeers(element) { position: 'center', imageUrl: element == 'audio' ? audioOffImg : camOffImg, title: element == 'audio' ? 'Mute everyone except yourself?' : 'Hide everyone except yourself?', - text: element == 'audio' ? - "Once muted, you won't be able to unmute them, but they can unmute themselves at any time." : "Once hided, you won't be able to unhide them, but they can unhide themselves at any time.", + text: + element == 'audio' + ? "Once muted, you won't be able to unmute them, but they can unmute themselves at any time." + : "Once hided, you won't be able to unhide them, but they can unhide themselves at any time.", showDenyButton: true, confirmButtonText: element == 'audio' ? `Mute` : `Hide`, denyButtonText: `Cancel`, @@ -4092,8 +4196,10 @@ function disablePeer(peer_id, element) { position: 'center', imageUrl: element == 'audio' ? audioOffImg : camOffImg, title: element == 'audio' ? 'Mute this participant?' : 'Hide this participant?', - text: element == 'audio' ? - "Once muted, you won't be able to unmute them, but they can unmute themselves at any time." : "Once hided, you won't be able to unhide them, but they can unhide themselves at any time.", + text: + element == 'audio' + ? "Once muted, you won't be able to unmute them, but they can unmute themselves at any time." + : "Once hided, you won't be able to unhide them, but they can unhide themselves at any time.", showDenyButton: true, confirmButtonText: element == 'audio' ? `Mute` : `Hide`, denyButtonText: `Cancel`, @@ -4314,7 +4420,7 @@ function whiteboardAddObj(type) { if (result.isConfirmed) { let wbCanvasImgURL = result.value; if (isImageURL(wbCanvasImgURL)) { - fabric.Image.fromURL(wbCanvasImgURL, function(myImg) { + fabric.Image.fromURL(wbCanvasImgURL, function (myImg) { addWbCanvasObj(myImg); }); } else { @@ -4342,10 +4448,10 @@ function whiteboardAddObj(type) { let wbCanvasImg = result.value; if (wbCanvasImg && wbCanvasImg.size > 0) { let reader = new FileReader(); - reader.onload = function(event) { + reader.onload = function (event) { let imgObj = new Image(); imgObj.src = event.target.result; - imgObj.onload = function() { + imgObj.onload = function () { let image = new fabric.Image(imgObj); image.set({ top: 0, left: 0 }).scale(0.3); addWbCanvasObj(image); @@ -4432,16 +4538,16 @@ function addWbCanvasObj(obj) { * Whiteboard: Local listners */ function setupWhiteboardLocalListners() { - wbCanvas.on('mouse:down', function(e) { + wbCanvas.on('mouse:down', function (e) { mouseDown(e); }); - wbCanvas.on('mouse:up', function() { + wbCanvas.on('mouse:up', function () { mouseUp(); }); - wbCanvas.on('mouse:move', function() { + wbCanvas.on('mouse:move', function () { mouseMove(); }); - wbCanvas.on('object:added', function() { + wbCanvas.on('object:added', function () { objectAdded(); }); } @@ -4662,7 +4768,7 @@ function handleWhiteboardAction(config, logme = true) { case 'toggle': toggleWhiteboard(); break; - //... + //... } } @@ -5133,7 +5239,8 @@ function handleKickedOut(config) { position: 'center', imageUrl: kickedOutImg, title: 'Kicked out!', - html: `

` + + html: + `

` + `User ` + peer_name + `

will kick out you after milliseconds.`, @@ -5371,7 +5478,7 @@ function userLog(type, message) { title: message, }); break; - // ...... + // ...... default: alert(message); } @@ -5429,36 +5536,3 @@ function getSl(selector) { function getEcN(className) { return document.getElementsByClassName(className); } - -/** - * Handle text transcipt getting from peers - * @param {*} data - */ -function handleSpeechTranscript(config) { - let peer_id = config.peer_id; - let time_stamp = getFormatDate(new Date());; - let name = config.peer_name; - let avatar_image = avatarApiUrl + '?name=' + name + '&size=32' + '&background=random&rounded=true'; - let transcipt = config.text_data; - console.log(config); - - const msgHTML = ` -
-
-
-
-
${name}
-
${time_stamp}
-
-
${transcipt}
-
-
- `; - captionChat.insertAdjacentHTML('beforeend', msgHTML); - captionChat.scrollTop += 500; - transcripts.push({ - time: time_stamp, - name: name, - caption: transcipt, - }); -} \ No newline at end of file diff --git a/public/js/speechRecognition.js b/public/js/speechRecognition.js new file mode 100644 index 00000000..a26d422e --- /dev/null +++ b/public/js/speechRecognition.js @@ -0,0 +1,79 @@ +/** + * https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API + */ +let isWebkitSpeechRecognitionSupported = false; +let recognitionRunning = false; +let recognition; + +if ('webkitSpeechRecognition' in window) { + recognition = new webkitSpeechRecognition(); + recognition.maxAlternatives = 1; + recognition.continuous = true; + + recognition.onstart = function () { + console.log('Start speech recognition'); + speechRecognitionStart.style.display = 'none'; + speechRecognitionStop.style.display = 'block'; + }; + + // Detect the said words + recognition.onresult = (e) => { + let current = e.resultIndex; + // Get a transcript of what was said. + let transcript = e.results[current][0].transcript; + config = { + type: 'speech', + room_id: roomId, + peer_name: myPeerName, + text_data: transcript, + time_stamp: new Date(), + }; + // save also my speech to text + handleSpeechTranscript(config); + + if (thereIsPeerConnections()) { + // Send speech transcript through RTC Data Channels + for (let peer_id in chatDataChannels) { + if (chatDataChannels[peer_id].readyState === 'open') + chatDataChannels[peer_id].send(JSON.stringify(config)); + } + } + }; + + recognition.onerror = function (event) { + console.warn('Speech recognition error', event.error); + }; + + recognition.onend = function () { + console.log('Stop speech recognition'); + // if (recognitionRunning) recognition.start(); + speechRecognitionStop.style.display = 'none'; + speechRecognitionStart.style.display = 'block'; + }; + + isWebkitSpeechRecognitionSupported = true; + console.info('Browser supports webkitSpeechRecognition'); +} else { + console.warn( + 'This browser not supports webkitSpeechRecognition, check out supported browsers: https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API#browser_compatibility', + ); +} + +//start or stop decider +function startSpeech(config) { + if (isWebkitSpeechRecognitionSupported) { + if (config) { + try { + recognitionRunning = true; + recognition.start(); + } catch (error) { + console.log('Start speech', error); + } + } else { + recognitionRunning = false; + recognition.stop(); + } + } else { + userLog('info', 'This browser not supports webkitSpeechRecognition'); + } +} diff --git a/public/js/speech_recognition.js b/public/js/speech_recognition.js deleted file mode 100644 index 0a537ba2..00000000 --- a/public/js/speech_recognition.js +++ /dev/null @@ -1,38 +0,0 @@ -let recordingStarted = false; -// initialize SpeechRecognition object -let recognition = new webkitSpeechRecognition(); -recognition.maxAlternatives = 1; -recognition.continuous = true; -// Detect the said words -recognition.onresult = e => { - let current = event.resultIndex; - // Get a transcript of what was said. - let transcript = event.results[current][0].transcript; - // Add the current transcript with existing said values - config = { - room_id: roomId, - peer_name: myPeerName, - text_data: transcript, - time_stamp: new Date(), - }; - sendToServer('speech_transcript', config); //sending data to signaling server for specific room -} - -//start or stop decider -function start_stop_speech(config) { - if (config) { - try { - // Start recognition - recognition.start(); - } catch (error) { - console.log(error); - } - } else { - //stop recognition - recognition.stop(); - } -} - -navigator.getUserMedia({ audio: true }, startUserMedia, function(e) { - __log('No live audio input: ' + e); -}); \ No newline at end of file diff --git a/public/js/volume_meter.js b/public/js/volume_meter.js deleted file mode 100644 index 5a711357..00000000 --- a/public/js/volume_meter.js +++ /dev/null @@ -1,96 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2014 Chris Wilson - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -/* - -Usage: -audioNode = createAudioMeter(audioContext,clipLevel,averaging,clipLag); - -audioContext: the AudioContext you're using. -clipLevel: the level (0 to 1) that you would consider "clipping". - Defaults to 0.98. -averaging: how "smoothed" you would like the meter to be over time. - Should be between 0 and less than 1. Defaults to 0.95. -clipLag: how long you would like the "clipping" indicator to show - after clipping has occured, in milliseconds. Defaults to 750ms. - -Access the clipping through node.checkClipping(); use node.shutdown to get rid of it. -*/ - -function createAudioMeter(audioContext, clipLevel, averaging, clipLag) { - var processor = audioContext.createScriptProcessor(512); - processor.onaudioprocess = volumeAudioProcess; - processor.clipping = false; - processor.lastClip = 0; - processor.volume = 0; - processor.clipLevel = clipLevel || 0.98; - processor.averaging = averaging || 0.95; - processor.clipLag = clipLag || 750; - - // this will have no effect, since we don't copy the input to the output, - // but works around a current Chrome bug. - processor.connect(audioContext.destination); - - processor.checkClipping = - function() { - if (!this.clipping) - return false; - if ((this.lastClip + this.clipLag) < window.performance.now()) - this.clipping = false; - return this.clipping; - }; - - processor.shutdown = - function() { - this.disconnect(); - this.onaudioprocess = null; - }; - - return processor; -} - -function volumeAudioProcess(event) { - var buf = event.inputBuffer.getChannelData(0); - var bufLength = buf.length; - var sum = 0; - var x; - - // Do a root-mean-square on the samples: sum up the squares... - for (var i = 0; i < bufLength; i++) { - x = buf[i]; - if (Math.abs(x) >= this.clipLevel) { - this.clipping = true; - this.lastClip = window.performance.now(); - } - sum += x * x; - } - - // ... then take the square root of the sum. - var rms = Math.sqrt(sum / bufLength); - - // Now smooth this out with the averaging factor applied - // to the previous sample - take the max here because we - // want "fast attack, slow release." - this.volume = Math.max(rms, this.volume * this.averaging); -} \ No newline at end of file diff --git a/public/sounds/speech.mp3 b/public/sounds/speech.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..83f23a2ed7ccd8e969a774222305d7dad5f35965 GIT binary patch literal 2595 zcmeZtF=k-^0i}@OU{@f`$H2hslUSB!YN=4du0ssE=PU~8Q%!v0TZ z!BY;26P)}1AM?>!6U@TEyJ7DCKlAmJ{{LQ(0<@f&;e0C##GVBVjQo~UeP?O_tza-| z1`>JJtZF*-|9`|-oaAoUVABxr<4V#JE}5k*sBZmGHea_HE9vs zU2b0^`_@jWj>xP}IzC+mxxbvHAKW^)sb?Xh%*C_`ht%T!gs~e~Jk*#VR8q=4V_(ew zpo_^sX8k`G7dFvPJh6QN?}gX@|L5#^StY1s+A;P0KOd|JVKQQLOSpCkg#{VVT}7ZWf%n}1v!?4*8o1jmcYt!U>z55mZ()iI$ch^N z{}Cl-pltV$|AQ8zzs3SYwv%DDVn$>;xMRq(2kbbIJ@@%Purr)vIDoLnhSh|*0iM0H z;Qk>!wZe3O?77zuwC8LCl07`kP9g$edzyfex{50H1f=|LVBlrA&;~BD!HGvwv^@3a z98fU_E=H(f553F)h1(nE4|BM$mTNL_filAb21Z5$ZI@O?#RO>a3kzeMG!fwnvgbkn z2RTMRh6jjrV8Lp^gh&TC&4)=4Y0o|W56lb~8Wa%rpjIa^<0zp){<+(azuKb2e3;%* Ndtf0!8a>>?4FEJxhs^*0 literal 0 HcmV?d00001 diff --git a/public/view/client.html b/public/view/client.html index f1697780..3d744d36 100755 --- a/public/view/client.html +++ b/public/view/client.html @@ -1,216 +1,237 @@ + + - - + + - - gtag('config', 'G-3XM60XK9RQ'); - + - + MiroTalk WebRTC Video call, Chat Room & Screen Sharing. + + - MiroTalk WebRTC Video call, Chat Room & Screen Sharing. - - + - + + + + - - - - + - + + + + + + + - - - - - - - + - + + + - - - + + - - + + - - + - +
+

WebRTC

+
-
-

WebRTC

-
+ - - -
-

Loading...

- 
+        

+            
Loading...

+            
 Please allow camera & microphone
 access to use this app.
             

-    

+
- -
- - - - - - - - - - - - - - - -
+
+ + + + + + + + + + + + + + + +
- + - -
-
-
-
Chat
-
- - - - - -
-
+
+
+
+
Chat
+
+ + + + + +
+
-
+
- -
- - +
+ + +
+ + + +
+ + + + +
- + -
- - - - -
+
+
+
+
Send Private messages
+
+ +
+
+
+
+ +
+
+
+
+
+
+ +
- + -
-
-
-
Send Private messages
+ + +
+
+
+
Captions
- + + + +
-
-
- -
-
-
-
+ +
+
+ + + +
- -
+ - + - -
-
-
-
Captions
-
- - - - -
+
+
+
- -
-
-
- - - - -
-
- -
-
-
-
- - - - -
- -
+

-
-
- +
+ + + +
-
-
-
- -
-
-
-
- -
-
-
-
- -
- +
+
+
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+ -

- -
- -

- -
- -
- -
-
-
-
-    - -
-
-
-
-    -

-
- + +
+ +

+ +
+
-
-
-
- -
-    + +
+
+
+
+    + +

+
+ +
+
+ +
+
+ +
+ -

- -
- -
- - + + + - + - + - + - + - + -
- mirotalk-share
-
-
- - -
- - - - - -
-
- +
+ mirotalk-share
+
+
+ +
-
- -
- + - + - - - - - - - - - - - - - - +
+
+ +
+
+ +
- + - + + + + + + + + + + + + + + + + + - - - \ No newline at end of file + +