Merge pull request #243 from Deep-Blue-2013/master

Add Web Speech #242

Author: Oceania2018

src/web-live-chat/src/lib/helpers/typedefs.js

@@ -118,6 +118,14 @@ IRichContent.prototype.text;
  * @param {ChatResponseModel} message
  */
 
+/**
+ * Invoked when speech to text is detected.
+ *
+ * @callback OnSpeechToTextDetected
+ * @param {string} text
+ */
+
+
 // having to export an empty object here is annoying, 
 // but required for vscode to pass on your types. 
 export default {};

src/web-live-chat/src/lib/services/agent-service.js

@@ -1,10 +1,12 @@
 import { agentListUrl, agentDetailUrl } from '$lib/services/api-endpoints.js';
+import { setAuthorization } from '$lib/helpers/http';
 import axios from 'axios';
 
 /**
  * @returns {Promise<import('$typedefs').AgentModel[]>}
  */
 export async function getAgents() {
+    setAuthorization();
     const response = await axios.get(agentListUrl);
     return response.data;
 }
@@ -14,6 +16,7 @@ export async function getAgents() {
  * @returns {Promise<import('$typedefs').AgentModel>}
  */
 export async function getAgent(id) {
+    setAuthorization();
     let url = agentDetailUrl.replace("{id}", id);
     const response = await axios.get(url);
     return response.data;

src/web-live-chat/src/lib/services/web-speech.js

@@ -0,0 +1,61 @@
+// https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API/Using_the_Web_Speech_API
+const SpeechRecognition =
+  window.SpeechRecognition || window.webkitSpeechRecognition;
+const SpeechRecognitionEvent =
+  window.SpeechRecognitionEvent || window.webkitSpeechRecognitionEvent;
+
+const recognition = new SpeechRecognition();
+recognition.continuous = false;
+recognition.lang = "en-US";
+recognition.interimResults = false;
+recognition.maxAlternatives = 1;
+
+const synth = window.speechSynthesis;
+
+
+const utterThis = new SpeechSynthesisUtterance();
+utterThis.pitch = 1;
+utterThis.rate = 1;
+
+export const webSpeech = {
+      /** @type {import('$typedefs').OnSpeechToTextDetected} */
+    onSpeechToTextDetected: () => {},
+
+    start() {
+        recognition.start();
+        console.log("Ready to receive a voice command.");      
+    },
+
+    /** @param {string} transcript */
+    utter(transcript) {
+        setVoiceSynthesis();
+        utterThis.text = transcript
+        synth.speak(utterThis);
+    }
+}
+
+function setVoiceSynthesis() {
+    if (utterThis.voice == null) {
+        const voices = synth.getVoices();
+        for (let i = 0; i < voices.length; i++) {
+            if (voices[i].name === "Microsoft Michelle Online (Natural) - English (United States)") {
+              utterThis.voice = voices[i];
+              break;
+            }
+        }
+    }
+}
+
+recognition.onresult = (event) => {
+    const text = event.results[0][0].transcript;
+    console.log(`Confidence: ${text} ${event.results[0][0].confidence}`);
+    webSpeech.onSpeechToTextDetected(text);
+};
+
+recognition.onnomatch = (event) => {
+    console.log("I didn't recognize that color.");
+};
+
+recognition.onerror = (event) => {
+    console.log(`Error occurred in recognition: ${event.error}`);
+};

src/web-live-chat/src/routes/(home)/+page.svelte

@@ -1,6 +1,6 @@
 <div class="row" style="margin-top:150px;margin-left:25px;margin-right:25px;">
     <div class="col-lg-8 offset-lg-2">
         <p class="section-subtitle text-muted text-center pt-4 font-secondary">We craft digital, graphic and dimensional thinking, to create category leading brand experiences that have meaning and add a value for our clients.</p>
-        <div class="d-flex justify-content-center"><a href="/login" class="btn btn-primary">New Conversation</a></div>
+        <div class="d-flex justify-content-center"><a href="/login" class="btn btn-primary">Get Started</a></div>
     </div>
 </div>

src/web-live-chat/src/routes/chat/+page.svelte

@@ -0,0 +1,55 @@
+<script>
+    import { Container, Row, Col } from "sveltestrap";
+
+    // This page is used to initialize a new conversation for client
+    import { page } from '$app/stores';
+    import { onMount } from 'svelte';
+    import { getAgents } from '$lib/services/agent-service.js'
+
+    const params = $page.params;
+    let agentId = "undefined";
+    /** @type {import('$typedefs').AgentModel[]} */
+    let agents = [];
+
+    onMount(async () => {
+        agents = await getAgents();
+        agentId = agents[0].id;
+    });
+</script>
+
+<Container fluid>
+    <Row>
+        <div class="col-12">
+            <div style="margin-top: 10vh; margin-left:10vw;">
+                {#each agents as agent}
+                <div>
+                    <input
+                        class="form-check-input m-1"
+                        type="radio"
+                        name="agents"
+                        id={agent.id}
+                        value={agent.id}
+                        checked = {agentId == agent.id}
+                        on:click={() => agentId = agent.id}
+                    />
+                    <label class="form-check-label" for={agent.id}>
+                        {agent.name}
+                    </label>  
+                    <div class="mx-4">{agent.description}</div>
+                </div>
+                {/each}
+            </div>    
+        </div>
+    </Row>
+    <Row class="text-center">
+        <Col>
+            <p class="section-subtitle text-muted text-center pt-4 font-secondary">We craft digital, graphic and dimensional thinking, to create category leading brand experiences that have meaning and add a value for our clients.</p>
+            <div class="d-flex justify-content-center">
+                <a href="/chat/{agentId}" class="btn btn-primary">
+                    <i class="mdi mdi-chat" />
+                    <span>Start Conversation</span>
+                </a>
+            </div>
+        </Col>
+    </Row>
+</Container>

src/web-live-chat/src/routes/chat/[agentId]/[conversationId]/chat-box.svelte

@@ -15,6 +15,7 @@
 	import { onMount } from 'svelte';
 	import Link from 'svelte-link';
 	import { signalr } from '$lib/services/signalr-service.js';
+	import { webSpeech } from '$lib/services/web-speech.js';
     import { sendMessageToHub, GetDialogs } from '$lib/services/conversation-service.js';
 	import { format } from '$lib/helpers/datetime';
 	import RcText from './rc-text.svelte';
@@ -43,6 +44,7 @@
 
 	// @ts-ignore
     let scrollbar;
+	let microphoneIcon = "microphone-off";
 
     /** @type {import('$typedefs').ChatResponseModel[]} */
     let dialogs = [];
@@ -78,17 +80,36 @@
 
     /** @param {import('$typedefs').ChatResponseModel} message */
     function onMessageReceivedFromAssistant(message) {
-      dialogs.push(message);
-      refresh();
+		webSpeech.utter(message.text);
+		// clean rich content elements
+		dialogs.forEach(dialog => {
+			if (dialog.rich_content.messaging_type == "quick_reply") {
+				dialog.rich_content.quick_repies = [];
+			}
+		});
+
+		dialogs.push(message);
+		refresh();
     }    
 
-    async function sendMessage() {
+    async function sendTextMessage() {
       await sendMessageToHub(params.agentId, params.conversationId, text);
     }
 
+    async function startListen() {
+		microphoneIcon = "microphone";
+		webSpeech.onSpeechToTextDetected = async (transcript) => {
+			text = transcript;
+			await sendTextMessage();
+			microphoneIcon = "microphone-off";
+		}
+		webSpeech.start();
+    }	
+
     function refresh() {
       // trigger UI render
       dialogs = dialogs;
+
       setTimeout(() => {
 		const { viewport } = scrollbar.elements();
 		viewport.scrollTo({ top: viewport.scrollHeight, behavior: 'smooth' }); // set scroll offset
@@ -194,8 +215,16 @@
 
 			<div class="p-3 chat-input-section" style="height: 10vh">
 				<div class="row">
+					<div class="col-auto">
+						<button
+							type="submit"
+							class="btn btn-primary btn-rounded waves-effect waves-light"
+							on:click={startListen}>
+							<i class="mdi mdi-{microphoneIcon} md-36" />
+						</button>
+					</div>
 					<div class="col">
-						<div class="position-relative">
+						<div class="position-relative">						
 							<input type="text" class="form-control chat-input" bind:value={text} placeholder="Enter Message..." />
 							<div class="chat-input-links" id="tooltip-container">
 								<ul class="list-inline mb-0">
@@ -210,10 +239,10 @@
 						<button
 							type="submit"
 							class="btn btn-primary btn-rounded chat-send w-md waves-effect waves-light"
-							on:click={sendMessage}
+							on:click={sendTextMessage}
 							><span class="d-none d-sm-inline-block me-2">Send</span>
-							<i class="mdi mdi-send" /></button
-						>
+							<i class="mdi mdi-send" />
+						</button>
 					</div>
 				</div>
 			</div>

src/web-live-chat/src/routes/login/+page.svelte

@@ -28,7 +28,7 @@
 			isOpen = true;
 			msg = 'Authentication success';
 			status = 'success';
-			goto(`/chat/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a`);
+			goto(`/chat`);
 		});
 	}
 </script>