From 84c6315f3458a006c14de8b24ae1a1c30a9e3e75 Mon Sep 17 00:00:00 2001 From: Andy Tang Date: Sun, 18 Jun 2023 10:29:43 -0700 Subject: [PATCH 1/5] add new routes for different modalities --- app/components/UserVideoPane.js | 4 +-- app/interview/page.js | 37 ++++++++++++++++++++++++ app/page.js | 50 ++++++++++++++++++++++++++++++--- app/pitch/page.js | 37 ++++++++++++++++++++++++ app/practice/page.js | 11 ++++++-- 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 app/interview/page.js create mode 100644 app/pitch/page.js diff --git a/app/components/UserVideoPane.js b/app/components/UserVideoPane.js index 4cd97ec..273a74c 100644 --- a/app/components/UserVideoPane.js +++ b/app/components/UserVideoPane.js @@ -8,7 +8,7 @@ import { AudioRecorder } from "../../lib/media/audioRecorder"; dotenv.config(); // Load environment variables from .env file -const UserVideoPane = ({ task }) => { +const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { const videoRef = useRef(null); const [mediaStream, setMediaStream] = useState(null); const [microphonePermissionGranted, setMicrophonePermissionGranted] = useState(false); @@ -128,7 +128,7 @@ const UserVideoPane = ({ task }) => { console.log(videoContextEmotion); // Define the system prompt and user speech - const systemPrompt = "You are a student. A teacher has been tasked with the following: " + task + ". You should ask questions and act confused. Previous conversation: " + prePrompt; + const systemPrompt = taskPrefix + task + taskSuffix + prePrompt; console.log(systemPrompt); const userSpeech = concatenatedTranscriptions + " Context: The user had " + contextEmotion + " as the highest emotion in their speech and " + videoContextEmotion + " as the highest emotion in their body language during this current response."; diff --git a/app/interview/page.js b/app/interview/page.js new file mode 100644 index 0000000..52bd3c0 --- /dev/null +++ b/app/interview/page.js @@ -0,0 +1,37 @@ +"use client"; + +import React, { useEffect, useState } from 'react'; +import dynamic from 'next/dynamic'; + +const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), { + loading: () =>

Loading...

, + ssr: false, // Disable server-side rendering for this component +}); + +const PracticePage = ({ task }) => { + const [isComponentLoaded, setIsComponentLoaded] = useState(false); + + if (task == null || task == "") { + task = "Explain how to efficiently merge k sorted linked lists."; + } + const taskPrefix = "You are a technical lead in an interview, where the coding question is: "; + const taskSuffix = ". You should delve deep into technical details of their solution, and give constraints when asked. Your goal is to evaluate" + + " whether hiring them would be beneficial to the company."; + + useEffect(() => { + setIsComponentLoaded(true); + }, []); + + return ( + <> + {isComponentLoaded && +
+

Task: {task}

+ +
+ } + + ); +}; + +export default PracticePage; \ No newline at end of file diff --git a/app/page.js b/app/page.js index dda35b2..ac97386 100644 --- a/app/page.js +++ b/app/page.js @@ -1,16 +1,58 @@ +"use client"; + import Image from 'next/image' import Link from 'next/link'; +import React, { useState } from 'react'; export default function Home() { + const [inputValue, setInputValue] = useState('Help a student understand AI double descent.'); + return (

Welcome to Interview InSight!

Explore and practice with us.

-
- - Go to Practice Page - + +
+

I want to... setInputValue(e.target.value)} + />

+
+ +
+
+ + Go to Teaching Practice + +
+
+ + Go to Interview Preparation + +
+
+ + Go to Practice Pitch + +
diff --git a/app/pitch/page.js b/app/pitch/page.js new file mode 100644 index 0000000..bae816b --- /dev/null +++ b/app/pitch/page.js @@ -0,0 +1,37 @@ +"use client"; + +import React, { useEffect, useState } from 'react'; +import dynamic from 'next/dynamic'; + +const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), { + loading: () =>

Loading...

, + ssr: false, // Disable server-side rendering for this component +}); + +const PracticePage = ({ task }) => { + const [isComponentLoaded, setIsComponentLoaded] = useState(false); + + if (task == null || task == "") { + task = "Pitch a startup to empower users through AI teaching."; + } + const taskPrefix = "You are a venture capitalist looking to invest in new firms. A team has come to you to: "; + const taskSuffix = ". You should ask sharp questions about their market, their vision, their growth plan, and" + + " other aspects of their idea which would be relevant in predicting future commercial success."; + + useEffect(() => { + setIsComponentLoaded(true); + }, []); + + return ( + <> + {isComponentLoaded && +
+

Task: {task}

+ +
+ } + + ); +}; + +export default PracticePage; \ No newline at end of file diff --git a/app/practice/page.js b/app/practice/page.js index 118fe7d..e7505f1 100644 --- a/app/practice/page.js +++ b/app/practice/page.js @@ -8,9 +8,14 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const PracticePage = () => { +const PracticePage = ({ task }) => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); - const task = "Help a student understand double descent and why it is important in Machine Learning."; + + if (task == null || task == "") { + task = "Help a student understand double descent and why it is important in Machine Learning."; + } + const taskPrefix = "You are a student. A teacher has been tasked with the following: "; + const taskSuffix = ". You should ask questions and act confused. Previous conversation: "; useEffect(() => { setIsComponentLoaded(true); @@ -21,7 +26,7 @@ const PracticePage = () => { {isComponentLoaded &&

Task: {task}

- +
} From 7e9c6f196e3fe3607e3c24bca02089e6960e31fa Mon Sep 17 00:00:00 2001 From: Andy Tang Date: Sun, 18 Jun 2023 10:38:35 -0700 Subject: [PATCH 2/5] rename pages --- app/interview/page.js | 4 ++-- app/pitch/page.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/interview/page.js b/app/interview/page.js index 52bd3c0..3623fa8 100644 --- a/app/interview/page.js +++ b/app/interview/page.js @@ -8,7 +8,7 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const PracticePage = ({ task }) => { +const InterviewPage = ({ task }) => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); if (task == null || task == "") { @@ -34,4 +34,4 @@ const PracticePage = ({ task }) => { ); }; -export default PracticePage; \ No newline at end of file +export default InterviewPage; \ No newline at end of file diff --git a/app/pitch/page.js b/app/pitch/page.js index bae816b..0e352c6 100644 --- a/app/pitch/page.js +++ b/app/pitch/page.js @@ -8,7 +8,7 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const PracticePage = ({ task }) => { +const PitchPage = ({ task }) => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); if (task == null || task == "") { @@ -34,4 +34,4 @@ const PracticePage = ({ task }) => { ); }; -export default PracticePage; \ No newline at end of file +export default PitchPage; \ No newline at end of file From 421b0e3a51dcd9cb96091f21af297f00f2f9f61d Mon Sep 17 00:00:00 2001 From: Andy Tang Date: Sun, 18 Jun 2023 11:40:45 -0700 Subject: [PATCH 3/5] check to make sure hydration works --- app/components/UserVideoPane.js | 28 +++++++++++------------ app/interview/page.js | 27 +++++++++++++++++------ app/page.js | 6 ++--- app/pitch/page.js | 26 +++++++++++++++++----- app/practice/page.js | 26 ++++++++++++++++------ package-lock.json | 39 +++++++++++++++++++++++++++++++++ package.json | 1 + pages/api/gpt.js | 2 +- pages/api/websockets.js | 6 ++--- 9 files changed, 120 insertions(+), 41 deletions(-) diff --git a/app/components/UserVideoPane.js b/app/components/UserVideoPane.js index 273a74c..9062bec 100644 --- a/app/components/UserVideoPane.js +++ b/app/components/UserVideoPane.js @@ -70,14 +70,14 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { if (response.ok) { const jsonResponse = await response.json(); - console.log(jsonResponse); + // console.log(jsonResponse); // Use braces afterwards const { transcription } = jsonResponse; - console.log("Transcription:", transcription); + // console.log("Transcription:", transcription); // Do something with the transcription const concatenatedTranscriptions = transcriptionCaches.slice(lastIndex).join(' ') + transcription; - console.log("Cumulative: " + concatenatedTranscriptions); + // console.log("Cumulative: " + concatenatedTranscriptions); let prePrompt = ""; for(let i = 0; i < AIResponses.length; i++) { @@ -89,7 +89,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { prePrompt += ", "; } - console.log(prePrompt) + // console.log(prePrompt) const frequencyCounter = speechEmotions.reduce((counter, emotion) => { counter[emotion] = (counter[emotion] || 0) + 1; @@ -107,7 +107,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { } } - console.log(contextEmotion); + // console.log(contextEmotion); const videoFrequencyCounter = videoEmotions.reduce((counter, emotion) => { counter[emotion] = (counter[emotion] || 0) + 1; @@ -125,11 +125,11 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { } } - console.log(videoContextEmotion); + // console.log(videoContextEmotion); // Define the system prompt and user speech const systemPrompt = taskPrefix + task + taskSuffix + prePrompt; - console.log(systemPrompt); + // console.log(systemPrompt); const userSpeech = concatenatedTranscriptions + " Context: The user had " + contextEmotion + " as the highest emotion in their speech and " + videoContextEmotion + " as the highest emotion in their body language during this current response."; @@ -144,7 +144,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { axios.post("/api/ai-response", payload) .then(response => { const aiResponse = response.data; - console.log('AI Response:', aiResponse); + // console.log('AI Response:', aiResponse); setQuestion("GPT: " + aiResponse.assistantReply); userInputs.push("User: " + concatenatedTranscriptions); @@ -339,7 +339,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { if (response.ok) { const jsonResponse = await response.json(); - console.log(jsonResponse); + // console.log(jsonResponse); // Use braces afterwards const { transcription } = jsonResponse; @@ -348,7 +348,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { // Call the setTranscriptionCaches function to store the transcriptions array in caches setTranscriptionCaches(transcriptionCaches); - console.log("Transcriptions:", transcriptionCaches); + // console.log("Transcriptions:", transcriptionCaches); } else { console.error("Error:", response.status); // Handle the error @@ -366,14 +366,14 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { const newSocket = new WebSocket(url); newSocket.onopen = async () => { - console.log('WebSocket connection established'); + // console.log('WebSocket connection established'); // Perform any necessary initialization or authentication recorderRef.current = await AudioRecorder.create(); // Create a closure to capture the current state of `socket` (async (socket) => { - console.log(socket) + // console.log(socket) while (socket) { const blob = await recorderRef.current.record(recordingLengthMs); // console.log(blob); @@ -397,7 +397,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { }; newSocket.onclose = () => { - console.log('WebSocket connection closed -- attempting re-open'); + // console.log('WebSocket connection closed -- attempting re-open'); createWebSocketConnection(); // Perform any necessary cleanup or reconnection logic }; @@ -472,7 +472,7 @@ const UserVideoPane = ({ taskPrefix, taskSuffix, task }) => { return updatedData; }); - console.log(message["prosody"]); + // console.log(message["prosody"]); } }; diff --git a/app/interview/page.js b/app/interview/page.js index 3623fa8..a9dfe92 100644 --- a/app/interview/page.js +++ b/app/interview/page.js @@ -1,6 +1,6 @@ "use client"; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useState, useRef } from 'react'; import dynamic from 'next/dynamic'; const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), { @@ -8,12 +8,25 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const InterviewPage = ({ task }) => { +const InterviewPage = () => { + const taskRef = useRef(""); + + useEffect(() => { + const url = new URL(window.location.href); + const searchParams = new URLSearchParams(url.search); + const task = searchParams.get('task'); + + // console.log('task:', task); + + if (task == null || task === "") { + taskRef.current = "Explain how to efficiently merge k sorted linked lists."; + } else { + taskRef.current = task; + } + }, []); + const [isComponentLoaded, setIsComponentLoaded] = useState(false); - if (task == null || task == "") { - task = "Explain how to efficiently merge k sorted linked lists."; - } const taskPrefix = "You are a technical lead in an interview, where the coding question is: "; const taskSuffix = ". You should delve deep into technical details of their solution, and give constraints when asked. Your goal is to evaluate" + " whether hiring them would be beneficial to the company."; @@ -26,8 +39,8 @@ const InterviewPage = ({ task }) => { <> {isComponentLoaded &&
-

Task: {task}

- +

Task: {taskRef.current}

+
} diff --git a/app/page.js b/app/page.js index ac97386..939e35b 100644 --- a/app/page.js +++ b/app/page.js @@ -27,7 +27,7 @@ export default function Home() { Go to Teaching Practice @@ -37,7 +37,7 @@ export default function Home() { Go to Interview Preparation @@ -47,7 +47,7 @@ export default function Home() { Go to Practice Pitch diff --git a/app/pitch/page.js b/app/pitch/page.js index 0e352c6..63bc439 100644 --- a/app/pitch/page.js +++ b/app/pitch/page.js @@ -1,6 +1,6 @@ "use client"; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useState, useRef } from 'react'; import dynamic from 'next/dynamic'; const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), { @@ -10,10 +10,24 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the const PitchPage = ({ task }) => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); + const taskRef = useRef(""); + + useEffect(() => { + const url = new URL(window.location.href); + const searchParams = new URLSearchParams(url.search); + const task = searchParams.get('task'); + + // console.log(url); + // console.log(searchParams); + // console.log('task:', task); + + if (task == null || task == "") { + task = "Pitch a startup to empower users' teaching skills by using AI to simulate a learner."; + } else { + taskRef.current = task; + } + }, []); - if (task == null || task == "") { - task = "Pitch a startup to empower users through AI teaching."; - } const taskPrefix = "You are a venture capitalist looking to invest in new firms. A team has come to you to: "; const taskSuffix = ". You should ask sharp questions about their market, their vision, their growth plan, and" + " other aspects of their idea which would be relevant in predicting future commercial success."; @@ -26,8 +40,8 @@ const PitchPage = ({ task }) => { <> {isComponentLoaded &&
-

Task: {task}

- +

Task: {taskRef.current}

+
} diff --git a/app/practice/page.js b/app/practice/page.js index e7505f1..0b8c599 100644 --- a/app/practice/page.js +++ b/app/practice/page.js @@ -1,6 +1,6 @@ "use client"; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useState, useRef } from 'react'; import dynamic from 'next/dynamic'; const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').then((module) => module.default), { @@ -8,12 +8,24 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const PracticePage = ({ task }) => { +const PracticePage = () => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); + const taskRef = useRef(""); + + useEffect(() => { + const url = new URL(window.location.href); + const searchParams = new URLSearchParams(url.search); + const task = searchParams.get('task'); + + // console.log('task:', task); + + if (task == null || task === "") { + taskRef.current = "Describe the theoretical foundations of double descent and its importance in machine learning."; + } else { + taskRef.current = task; + } + }, []); - if (task == null || task == "") { - task = "Help a student understand double descent and why it is important in Machine Learning."; - } const taskPrefix = "You are a student. A teacher has been tasked with the following: "; const taskSuffix = ". You should ask questions and act confused. Previous conversation: "; @@ -25,8 +37,8 @@ const PracticePage = ({ task }) => { <> {isComponentLoaded &&
-

Task: {task}

- +

Task: {taskRef.current}

+
} diff --git a/package-lock.json b/package-lock.json index 6ef5438..78b5034 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "postcss": "^8.4.24", "react": "18.2.0", "react-dom": "18.2.0", + "react-router-dom": "^6.13.0", "socket.io-client": "^4.6.2", "stream": "^0.0.2", "ws": "^8.13.0" @@ -398,6 +399,14 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@remix-run/router": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.6.3.tgz", + "integrity": "sha512-EXJysQ7J3veRECd0kZFQwYYd5sJMcq2O/m60zu1W2l3oVQ9xtub8jTOtYRE0+M2iomyG/W3Ps7+vp2kna0C27Q==", + "engines": { + "node": ">=14" + } + }, "node_modules/@rushstack/eslint-patch": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.3.2.tgz", @@ -4055,6 +4064,36 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, + "node_modules/react-router": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-6.13.0.tgz", + "integrity": "sha512-Si6KnfEnJw7gUQkNa70dlpI1bul46FuSxX5t5WwlUBxE25DAz2BjVkwaK8Y2s242bQrZPXCpmwLPtIO5pv4tXg==", + "dependencies": { + "@remix-run/router": "1.6.3" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": ">=16.8" + } + }, + "node_modules/react-router-dom": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.13.0.tgz", + "integrity": "sha512-6Nqoqd7fgwxxVGdbiMHTpDHCYPq62d7Wk1Of7B82vH7ZPwwsRaIa22zRZKPPg413R5REVNiyuQPKDG1bubcOFA==", + "dependencies": { + "@remix-run/router": "1.6.3", + "react-router": "6.13.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "react": ">=16.8", + "react-dom": ">=16.8" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", diff --git a/package.json b/package.json index 0ba4c9e..0099ea3 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "postcss": "^8.4.24", "react": "18.2.0", "react-dom": "18.2.0", + "react-router-dom": "^6.13.0", "socket.io-client": "^4.6.2", "stream": "^0.0.2", "ws": "^8.13.0" diff --git a/pages/api/gpt.js b/pages/api/gpt.js index 4bb5388..27dc923 100644 --- a/pages/api/gpt.js +++ b/pages/api/gpt.js @@ -10,7 +10,7 @@ const bufferToStream = (buffer) => { export default async function handler(req, res) { if (req.method === 'POST') { try { - console.log(req.body); + // console.log(req.body); const base64Audio = req.body; const audioBuffer = Buffer.from(base64Audio, 'base64'); diff --git a/pages/api/websockets.js b/pages/api/websockets.js index 844e6f8..349ca3a 100644 --- a/pages/api/websockets.js +++ b/pages/api/websockets.js @@ -31,7 +31,7 @@ export default async function handler(req, res) { }); socket.onopen = () => { - console.log('WebSocket connection established'); + // console.log('WebSocket connection established'); // Perform any necessary initialization or authentication // Send the JSON message to the WebSocket server socket.send(JSON.stringify(jsonMessage)); @@ -39,14 +39,14 @@ export default async function handler(req, res) { socket.onmessage = (event) => { const message = JSON.parse(event.data); - console.log('Received message:', message); + // console.log('Received message:', message); // Process and handle the received data as needed // Send the response back to the client res.status(200).json({ message: 'WebSocket message received' }); }; socket.onclose = () => { - console.log('WebSocket connection closed'); + // console.log('WebSocket connection closed'); // Perform any necessary cleanup or reconnection logic }; } catch (error) { From 854a71796698fe196d66450b1434e9a89ad99806 Mon Sep 17 00:00:00 2001 From: Andy Tang Date: Sun, 18 Jun 2023 11:41:58 -0700 Subject: [PATCH 4/5] fixup --- app/interview/page.js | 2 +- app/pitch/page.js | 2 +- app/practice/page.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/interview/page.js b/app/interview/page.js index a9dfe92..efbd106 100644 --- a/app/interview/page.js +++ b/app/interview/page.js @@ -14,7 +14,7 @@ const InterviewPage = () => { useEffect(() => { const url = new URL(window.location.href); const searchParams = new URLSearchParams(url.search); - const task = searchParams.get('task'); + let task = searchParams.get('task'); // console.log('task:', task); diff --git a/app/pitch/page.js b/app/pitch/page.js index 63bc439..a758840 100644 --- a/app/pitch/page.js +++ b/app/pitch/page.js @@ -15,7 +15,7 @@ const PitchPage = ({ task }) => { useEffect(() => { const url = new URL(window.location.href); const searchParams = new URLSearchParams(url.search); - const task = searchParams.get('task'); + let task = searchParams.get('task'); // console.log(url); // console.log(searchParams); diff --git a/app/practice/page.js b/app/practice/page.js index 0b8c599..e61d53f 100644 --- a/app/practice/page.js +++ b/app/practice/page.js @@ -15,7 +15,7 @@ const PracticePage = () => { useEffect(() => { const url = new URL(window.location.href); const searchParams = new URLSearchParams(url.search); - const task = searchParams.get('task'); + let task = searchParams.get('task'); // console.log('task:', task); From 63bf73758ff13ff67d4cf59ae0bc4a2cc5261f2a Mon Sep 17 00:00:00 2001 From: Andy Tang Date: Sun, 18 Jun 2023 11:43:37 -0700 Subject: [PATCH 5/5] fixup --- app/pitch/page.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/pitch/page.js b/app/pitch/page.js index a758840..42a41d6 100644 --- a/app/pitch/page.js +++ b/app/pitch/page.js @@ -8,7 +8,7 @@ const DynamicVideoPane = dynamic(() => import('../components/UserVideoPane').the ssr: false, // Disable server-side rendering for this component }); -const PitchPage = ({ task }) => { +const PitchPage = () => { const [isComponentLoaded, setIsComponentLoaded] = useState(false); const taskRef = useRef("");