Skip to content

Retested + Adding a Sidebar UI #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yanheChen wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Retested + Adding a Sidebar UI #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e697661
Add a side panel / update the google meeting tag
yanheChen Feb 24, 2025
bd9404c
improve the prompt with newly gemini api endpoint
yanheChen Feb 26, 2025
0b20a4c
update the DOMTREE selector
yanheChen Feb 26, 2025
7f3f65f
updating the sidepanel UI
yanheChen Feb 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 32 additions & 2 deletions background/background.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,36 @@
See the License for the specific language governing permissions and
limitations under the License.
*/
chrome.action.onClicked.addListener(() => {
chrome.runtime.openOptionsPage();
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
if (message.type === 'SIDE_PANEL_TO_CONTENT') {
// Forward to active tab
chrome.tabs.query({active: true, currentWindow: true}, function(tabs) {
if (!tabs[0]?.id) return;

chrome.tabs.sendMessage(tabs[0].id, {
incoming: true,
type: 'SIDE_PANEL_COMMAND',
command: message.command,
data: message.data
});
});
} else if (message.type === 'UPDATE_SIDE_PANEL') {
// Make sure we have a valid tab ID from the sender
if (!sender.tab?.id) return;

// Get side panel options for the specific tab
chrome.sidePanel.getOptions({tabId: sender.tab.id}).then(options => {
if (options.enabled) {
chrome.runtime.sendMessage({
type: 'CONTENT_TO_SIDE_PANEL',
state: message.state
});
}
}).catch(error => {
console.error('Error getting side panel options:', error);
});
}
});

// Setup side panel behavior
chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });
99 changes: 83 additions & 16 deletions content/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,38 +102,73 @@ function fetchGoogleMeetCaptions() {
let captions = '';
let selfCaptions = '';
let allCaptions = '';

const gMeetCaptionsView =
document.querySelector('div[jscontroller="D1tHje"]');

if (gMeetCaptionsView) {
const divs = document.querySelectorAll('div[class="TBMuR bj4p3b"]');
for (const div of divs) {
let name = div.querySelector('div[class="zs7s8d jxFHg"]')!.textContent;
let wordSpans = Array.from(div.querySelectorAll('span'));
captions += name + ': ';
const sentence = wordSpans.map(span => span.textContent.trim()).join(' ');
if (name === 'You' || name.indexOf('Presentation') !== -1) {
const captionContainer = document.querySelector('div[jsname="dsyhDe"]');

if (captionContainer) {
const captionBlocks = document.querySelectorAll('div[jsname="tgaKEf"][class*="bh44bd"]');

for (const block of captionBlocks) {
const spans = Array.from(block.querySelectorAll('span'));
if (spans.length === 0) continue;

const sentence = spans.map(span => span.textContent.trim()).join(' ');

const parentBlock = block.closest('div[jsname="YSxPC"]');
const isUserCaption = determineIfUserCaption(parentBlock);

if (isUserCaption) {
if (selfCaptions.length > 0) {
selfCaptions += ' ';
}
selfCaptions += sentence;
}

captions += sentence + '\n';
allCaptions += sentence;
allCaptions += sentence + ' ';

console.log('Caption found:', sentence);
}
}

renderer.setCaptions(
Boolean(gMeetCaptionsView),
Boolean(captionContainer && allCaptions.length > 0),
captions,
selfCaptions,
allCaptions,
allCaptions
);

setTimeout(fetchGoogleMeetCaptions, APP_SETTINGS.fetchMeetCaptionFPSInMs);
}

// Helper function to determine if a caption block belongs to the current user
function determineIfUserCaption(element: Element) {
if (!element) return false;

const hasUserIndicators = element.classList.contains('wY1pdd');

const siblingElements: any[] | HTMLCollection = element.parentElement?.children || [];
for (const sibling of siblingElements) {
if (sibling.textContent?.includes('You')) {
return true;
}
}
let current = element;
for (let i = 0; i < 3; i++) {
if (!current.parentElement) break;
current = current.parentElement;

// Check if any parent element contains "You" text
if (current.textContent?.includes('You')) {
return true;
}
}

// Default to true for the first caption if we can't determine
// This is a fallback assuming the first caption is often the user's
const isFirstCaptionBlock = !element.previousElementSibling;
return isFirstCaptionBlock || hasUserIndicators;
}

function makeStreamFromTrack(track: MediaStreamTrack) {
const stream = new MediaStream();
stream.addTrack(track);
Expand All @@ -158,7 +193,6 @@ function patchNativeAPI() {

if (videoDeviceId !== 'virtual') {
restoreVideoMirrorMode();
console.log('Current this', this);
return MediaDevices.prototype.getUserMedia.call(this, constraints);
} else {
removeVideoMirrorMode();
Expand Down Expand Up @@ -216,5 +250,38 @@ function restoreVideoMirrorMode() {
}
}

window.addEventListener('message', (event) => {
const message = event.data;
if (!message.incoming) return;

if (message.type === 'SETTINGS') {
console.log('Received settings', message.settings);
applySettings(message.settings);
} else if (message.type === 'SIDE_PANEL_COMMAND') {
// Handle side panel specific commands
handleSidePanelCommand(message.command, message.data);
}
});

// Function to handle side panel commands
function handleSidePanelCommand(command: string, data: any) {
switch(command) {
case 'SET_SCENE':
renderer.setScene(data.scene);
break;
case 'TOGGLE_FEATURE':
// Handle feature toggle
break;
}

window.postMessage({
type: 'UPDATE_SIDE_PANEL',
outgoing: true,
state: {
currentScene: renderer.getCurrentScene(),
}
}, '*');
}

patchNativeAPI();
fetchGoogleMeetCaptions();
89 changes: 58 additions & 31 deletions content/interactive_image.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -720,11 +720,7 @@ declare global {
function createWidget(entity: Entity, isEmoji: boolean) {
const videos = document.querySelectorAll('video');
let video;
if (videos.length === 1) {
video = videos[0];
} else {
video = Array.from(videos).find((video) => getName(video).includes('You'));
}
video = videos[0];
const container = video.parentElement;

if (isEmoji) {
Expand All @@ -741,7 +737,7 @@ declare global {
//////////////////////////////////////////////////////////////////////////////
// MAIN
//////////////////////////////////////////////////////////////////////////////
let fetchCaptionsHandle: number|NodeJS.Timeout;
let fetchCaptionsHandle: number|NodeJS.Timeout;
async function fetchGoogleMeetCaptions() {
options.update();
if (!options.enabled) {
Expand All @@ -761,31 +757,62 @@ declare global {

return scheduleNextFetch();
}

function fetchCaptionsFromDOM() {
const gMeetCaptionsView = document.querySelector(`div[jscontroller="${gMeetControllerName}"]`);
if (!gMeetCaptionsView) return null;

const captionContainer = document.querySelector('div[jsname="dsyhDe"]');
if (!captionContainer) {
console.log('VC - No caption container found');
return null;
}

let selfCaptions = '';
let allCaptions = '';

const divs = document.querySelectorAll('div[class="TBMuR bj4p3b"]');
for (const div of divs) {
const name = div.querySelector('div[class="zs7s8d jxFHg"]').textContent;
const sentence = Array.from(div.querySelectorAll('span'))
.map(span => span.textContent.trim())
.join(' ');

if (name === 'You' || name === 'Your Presentation') {
selfCaptions += sentence;

const childDivs = Array.from(captionContainer.children[0].children);
if (childDivs.length <= 1) {
console.log('VC - Not enough caption content divs found');
return null;
}

const contentDivs = childDivs.slice(0, -1);

for (const contentDiv of contentDivs) {
const nameElement = contentDiv.querySelector('div[class="KcIKyf jxFHg"]');
const speakerName = nameElement ? nameElement.textContent.trim() : '';
const isUserCaption = speakerName === 'You';

// Find all caption text under the YSxPC divs in this content section
const captionSections = contentDiv.querySelectorAll('div[jsname="YSxPC"]');
for (const section of captionSections) {
// Find the caption text divs with jsname="tgaKEf"
const captionDivs = section.querySelectorAll('div[jsname="tgaKEf"]');
for (const captionDiv of captionDivs) {
const text = captionDiv.textContent.trim();
if (!text) continue;

// Add to self captions if this is the user
if (isUserCaption) {
if (selfCaptions) selfCaptions += ' ';
selfCaptions += text;
}

// Add to all captions with speaker name
if (allCaptions) allCaptions += ' ';
if (speakerName && !allCaptions.endsWith(`${speakerName}: `)) {
allCaptions += `${speakerName}: `;
}
allCaptions += text;

console.log(`VC - Caption found from ${speakerName || 'unknown'}: ${text.substring(0, 50)}${text.length > 50 ? '...' : ''}`);
}
}
allCaptions += sentence;
}

return { selfCaptions, allCaptions };
return (allCaptions || selfCaptions) ? { selfCaptions, allCaptions } : null;
}

async function processAndGenerateVisuals(captions: string) {
console.log('VC - processAndGenerateVisuals', captions);
const text = extractTextForProcessing(captions);
if (text === lastText) {
console.log('VC - same text will not generate new content: ', text);
Expand Down Expand Up @@ -903,19 +930,19 @@ declare global {

window.addEventListener('message', (event) => {
const message = event.data;

if (message.type !== 'CURRENT_SCENE') return;
if (message.type !== 'CURRENT_SCENE') return;
toggleEffect(message.scene === 'Interactive Images (beta)');
});

toggleEffect(true);

window.postMessage(
{
type: 'GET_CURRENT_SCENE',
outgoing: true,
},
'*');
{
type: 'GET_CURRENT_SCENE',
outgoing: true,
},
'*'
);

//////////////////////////////////////////////////////////////////////////////
// HELPERS
Expand Down
3 changes: 3 additions & 0 deletions manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
"service_worker": "background.js",
"type": "module"
},
"side_panel": {
"default_path": "sidepanel.html"
},
"icons": {
"16": "images/icon16.png",
"32": "images/icon32.png",
Expand Down
Loading