From f66644ac14f5e25d735008f003f13641cc3c6897 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Sun, 28 Apr 2024 15:32:42 -0600 Subject: [PATCH 01/13] feat(core/provenance, react-trrack-example): add native screenshots with one bug remaining Native screenshots are working, but the catpureNextRepaint() function doesn't yet capture the next repaint; it takes a screenshot before the next repaint --- apps/react-trrack-example/src/app/App.tsx | 15 +- .../src/app/components/Navbar.tsx | 5 +- packages/core/src/provenance/index.ts | 1 + .../core/src/provenance/screenshot-stream.ts | 220 ++++++++++++++++++ packages/core/src/provenance/types.ts | 2 +- 5 files changed, 238 insertions(+), 5 deletions(-) create mode 100644 packages/core/src/provenance/screenshot-stream.ts diff --git a/apps/react-trrack-example/src/app/App.tsx b/apps/react-trrack-example/src/app/App.tsx index 388ec21..d952d9c 100644 --- a/apps/react-trrack-example/src/app/App.tsx +++ b/apps/react-trrack-example/src/app/App.tsx @@ -1,9 +1,11 @@ -import { Box, Checkbox, List, ListItem, ListItemIcon, ListItemText, Typography } from '@mui/material'; +import { Box, Button, Checkbox, List, ListItem, ListItemIcon, ListItemText, Typography } from '@mui/material'; import Tree, { useTreeState } from 'react-hyper-tree'; import { TreeNode } from 'react-hyper-tree/dist/helpers/node'; import { Navbar } from './components/Navbar'; import { useTrrackTaskManager } from './store/trrack'; +import { useRef } from 'react'; +import { ScreenshotStream, downloadScreenshot } from '@trrack/core'; function App() { const trrackManager = useTrrackTaskManager(); @@ -18,9 +20,16 @@ function App() { open(required.data, trrackManager.trrack.current.id); + // Testing screenshot stream + const ss = useRef(new ScreenshotStream()); + return ( - + + + - {trrackManager.state.tasks.map((task) => ( + {trrackManager.state.tasks?.map((task) => ( { trrack.apply('Increment counter', actions.incrementCounter(1)); + ss.captureNextRepaint(); }} > @@ -30,6 +32,7 @@ export function Navbar({ t }: { t: Trrack }) { { trrack.apply('Decrement counter', actions.decrementCounter(1)); + ss.captureNextRepaint(); }} > diff --git a/packages/core/src/provenance/index.ts b/packages/core/src/provenance/index.ts index df8096e..7cc4647 100644 --- a/packages/core/src/provenance/index.ts +++ b/packages/core/src/provenance/index.ts @@ -2,3 +2,4 @@ export * from './trrack'; export * from './trrack-config-opts'; export * from './trrack-events'; export * from './types'; +export * from './screenshot-stream'; diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts new file mode 100644 index 0000000..65cf367 --- /dev/null +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -0,0 +1,220 @@ +/** + * Captures and stores a sequence of screenshots of the current tab. + * First, opens a MediaStream of the current tab, then captures screenshots + * on repaints after captureNextRepaint() is called. + * A screenshot can also be captured on-demand via capture(). + * Requires browser permissions to capture screen. + * Must be activated via start() and deactivated via stop(); failure to stop + * will result in a memory leak. + */ +export class ScreenshotStream { + /** + * Video element for capturing screenshots. Null if not started or stopped. + */ + private video: HTMLVideoElement | null = null; + + /** + * Array of captured screenshots. + */ + private screenshots: ImageData[] = []; + + /** + * Optional callback to run after each screenshot is captured. + */ + public newScreenshotCallback: ((frame: ImageData) => void) | null; + + /** + * Binds capture, stop, and captureNextRepaint to the class. + * @throws Error if the getDisplayMedia API is not available. + */ + constructor(newScreenshotCallback?: (frame: ImageData) => void) { + if (!navigator.mediaDevices?.getDisplayMedia) { + throw new Error( + 'MediaDevices API or getDisplayMedia() not available' + ); + } + + this.newScreenshotCallback = newScreenshotCallback ?? null; + + // We need to functions that can be used as callbacks to the class + this.capture = this.capture.bind(this); + this.stop = this.stop.bind(this); + this.captureNextRepaint = this.captureNextRepaint.bind(this); + } + + /** + * Starts the media stream needed to capture screenshots on-demand. + * Will prompt the user for permission to capture the screen. + * @throws Error if unable to start the recording; usually due to the user denying permission. + * @param callback Optional callback to run after the stream is started. + */ + public async start(callback?: () => void): Promise { + this.video = document.createElement('video'); + this.video.autoplay = true; + this.video.muted = true; + this.video.playsInline = true; + this.video.style.pointerEvents = 'none'; + this.video.style.visibility = 'hidden'; + this.video.style.position = 'fixed'; + this.video.style.top = '0'; + this.video.style.left = '0'; + + /*const displayMediaOptions: DisplayMediaStreamOptions = { + video: { + //cursor: "never", + displaySurface: "browser" + }, + audio: false, + // Below are explicit defaults because the actual varies by browser + // monitorTypeSurfaces: "exclude", // Don't offer whole-screen capture options + // preferCurrentTab: true, + // selfBrowserSurface: true, + // surfaceSwitching: "exclude", + };*/ + + try { + await navigator.mediaDevices + .getDisplayMedia(/*displayMediaOptions*/) + .then((stream) => { + // TS is not confident that this.video is not null (but I am), so we need to check + this.video ? (this.video.srcObject = stream) : null; + }); + } catch (e) { + this.video = null; + throw new Error(`Unable to start recording: ${e}`); + } + + if (this.video.srcObject) { + // Needs to be in the DOM to capture screenshots + document.body.appendChild(this.video); + callback ? callback() : null; + } else { + // I honestly don't know how we'd get here + throw new Error('Unable to start recording; no stream available'); + } + } + + /** + * Captures a screenshot and stores it in the screenshots array. + * Also pushes the screenshot to the newScreenshotCallback if available. + * Bound to the class in the constructor. + * @throws Error if recording has not been started. + * @throws Error if unable to get 2D rendering context. + * @returns The captured screenshot. + */ + public capture(): ImageData { + if (!this.video) { + throw new Error('Recording not started'); + } + + const videoSettings = (this.video.srcObject as MediaStream) + ?.getVideoTracks()[0] + .getSettings(); + const canvas = document.createElement('canvas'); + canvas.width = videoSettings.width || 0; + canvas.height = videoSettings.height || 0; + + const context = canvas.getContext('2d'); + if (!context) { + // GetContext can return undefined and null (probably due to lack of browser support) + throw new Error('Unable to get 2D rendering context'); + } + context.drawImage(this.video, 0, 0, canvas.width, canvas.height); + + const frame = context.getImageData(0, 0, canvas.width, canvas.height); + this.push(frame); + + canvas.remove(); + return frame; + } + + /** + * Captures a screenshot after the next repaint and adds it to the screenshot array. + * Useful if you want to screenshot a state change that you've just processed. + * Bound to the class in the constructor. + */ + public captureNextRepaint(): void { + requestAnimationFrame(() => { + const mc = new MessageChannel(); + mc.port1.onmessage = this.capture; + mc.port2.postMessage(undefined); + }); + } + + /** + * Stops the media stream and removes the video element from the DOM. + * Must be called to prevent memory leaks. + * Bound to the class in the constructor. + */ + public stop(): void { + if (this.video) { + this.video.srcObject = null; + this.video.remove(); + this.video = null; + } + } + + /** + * Pushes a screenshot frame to the `screenshots` array + * and invokes the `newScreenshotCallback` if available. + * @param frame - The screenshot frame to be pushed. + */ + private push(frame: ImageData): void { + this.screenshots.push(frame); + this.newScreenshotCallback ? this.newScreenshotCallback(frame) : null; + } + + /** + * Returns the nth most recent screenshot in the array of stored screenshots. + * @param n - The index of the screenshot to retrieve. + * 1 is the most recent screenshot, 0 is the least recent. + * @returns The nth screenshot. + */ + public getNth(n: number): ImageData { + if (n < 0 || n >= this.screenshots.length) { + throw new Error(`Screenshot index out of bounds: ${n}`); + } + return this.screenshots[this.screenshots.length - 1 - n]; + } + + /** + * Returns the number of stored screenshots. + * @returns The number of stored screenshots. + */ + public count(): number { + return this.screenshots.length; + } + + /** + * Returns a copy of the array of stored screenshots. + * @returns The stored screenshots. + */ + public getAll(): ImageData[] { + return [...this.screenshots]; + } +} + +/** + * Downloads a screenshot as a PNG file. + * @param frame - The screenshot frame to download. + * @param name - The name of the file to download. + */ +export function downloadScreenshot(frame: ImageData, name: string): void { + const canvas = document.createElement('canvas'); + canvas.width = frame.width; + canvas.height = frame.height; + + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Unable to get 2D rendering context'); + } + context.putImageData(frame, 0, 0); + + const a = document.createElement('a'); + a.href = canvas.toDataURL(); + a.download = name; + a.click(); + + canvas.remove(); + a.remove(); +} diff --git a/packages/core/src/provenance/types.ts b/packages/core/src/provenance/types.ts index d67c109..4f36924 100644 --- a/packages/core/src/provenance/types.ts +++ b/packages/core/src/provenance/types.ts @@ -1,5 +1,5 @@ import { PayloadAction } from '@reduxjs/toolkit'; -import { NodeId } from '@trrack/core'; +import { NodeId } from '../graph/components/node'; import { Artifact, CurrentChangeHandler, From a27221666f285a19c1f40f53860920c59d746118 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Sun, 28 Apr 2024 15:35:35 -0600 Subject: [PATCH 02/13] fix(react-trrack-example): accept linter formatting suggestions --- apps/react-trrack-example/src/app/App.tsx | 23 ++++++++++++++++--- .../src/app/components/Navbar.tsx | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/react-trrack-example/src/app/App.tsx b/apps/react-trrack-example/src/app/App.tsx index d952d9c..4546a8e 100644 --- a/apps/react-trrack-example/src/app/App.tsx +++ b/apps/react-trrack-example/src/app/App.tsx @@ -1,4 +1,13 @@ -import { Box, Button, Checkbox, List, ListItem, ListItemIcon, ListItemText, Typography } from '@mui/material'; +import { + Box, + Button, + Checkbox, + List, + ListItem, + ListItemIcon, + ListItemText, + Typography, +} from '@mui/material'; import Tree, { useTreeState } from 'react-hyper-tree'; import { TreeNode } from 'react-hyper-tree/dist/helpers/node'; @@ -26,10 +35,18 @@ function App() { return ( - - + Date: Sun, 28 Apr 2024 15:38:39 -0600 Subject: [PATCH 03/13] style(screenshot-stream.ts): remove commented-out code --- packages/core/src/provenance/screenshot-stream.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts index 65cf367..d7f05b6 100644 --- a/packages/core/src/provenance/screenshot-stream.ts +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -59,19 +59,6 @@ export class ScreenshotStream { this.video.style.top = '0'; this.video.style.left = '0'; - /*const displayMediaOptions: DisplayMediaStreamOptions = { - video: { - //cursor: "never", - displaySurface: "browser" - }, - audio: false, - // Below are explicit defaults because the actual varies by browser - // monitorTypeSurfaces: "exclude", // Don't offer whole-screen capture options - // preferCurrentTab: true, - // selfBrowserSurface: true, - // surfaceSwitching: "exclude", - };*/ - try { await navigator.mediaDevices .getDisplayMedia(/*displayMediaOptions*/) From 7b2f3f7fcc2c3e2d827a692055e09d4176bcce64 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Mon, 29 Apr 2024 17:50:23 -0600 Subject: [PATCH 04/13] feat(screenshot-stream): use timeout to delay captures instead of repaint logic --- .../src/app/components/Navbar.tsx | 4 +- .../core/src/provenance/screenshot-stream.ts | 48 ++++++++++++------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/apps/react-trrack-example/src/app/components/Navbar.tsx b/apps/react-trrack-example/src/app/components/Navbar.tsx index b03ce9c..905d6b4 100644 --- a/apps/react-trrack-example/src/app/components/Navbar.tsx +++ b/apps/react-trrack-example/src/app/components/Navbar.tsx @@ -21,7 +21,7 @@ export function Navbar({ t, ss }: { t: Trrack; ss: ScreenshotStream }) { { trrack.apply('Increment counter', actions.incrementCounter(1)); - ss.captureNextRepaint(); + ss.delayCapture(100); }} > @@ -32,7 +32,7 @@ export function Navbar({ t, ss }: { t: Trrack; ss: ScreenshotStream }) { { trrack.apply('Decrement counter', actions.decrementCounter(1)); - ss.captureNextRepaint(); + ss.delayCapture(100); }} > diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts index d7f05b6..275338a 100644 --- a/packages/core/src/provenance/screenshot-stream.ts +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -1,11 +1,11 @@ /** * Captures and stores a sequence of screenshots of the current tab. - * First, opens a MediaStream of the current tab, then captures screenshots - * on repaints after captureNextRepaint() is called. - * A screenshot can also be captured on-demand via capture(). + * First, opens a MediaStream of the current tab, then can capture screenshots. + * A screenshot can also be captured on-demand via capture() or after a delay via delayCapture(). * Requires browser permissions to capture screen. * Must be activated via start() and deactivated via stop(); failure to stop * will result in a memory leak. + * All functions are bound to the class, so they can be passed as callbacks. */ export class ScreenshotStream { /** @@ -18,13 +18,20 @@ export class ScreenshotStream { */ private screenshots: ImageData[] = []; + /** + * ID of the timeout for delayCapture. + * Null if no timeout is active. + */ + private timeout: NodeJS.Timeout | null = null; + /** * Optional callback to run after each screenshot is captured. */ public newScreenshotCallback: ((frame: ImageData) => void) | null; /** - * Binds capture, stop, and captureNextRepaint to the class. + * Checks that the getDisplayMedia API is available + * and binds all functions to the class. * @throws Error if the getDisplayMedia API is not available. */ constructor(newScreenshotCallback?: (frame: ImageData) => void) { @@ -36,10 +43,14 @@ export class ScreenshotStream { this.newScreenshotCallback = newScreenshotCallback ?? null; - // We need to functions that can be used as callbacks to the class this.capture = this.capture.bind(this); this.stop = this.stop.bind(this); - this.captureNextRepaint = this.captureNextRepaint.bind(this); + this.delayCapture = this.delayCapture.bind(this); + this.push = this.push.bind(this); + this.getNth = this.getNth.bind(this); + this.count = this.count.bind(this); + this.getAll = this.getAll.bind(this); + this.start = this.start.bind(this); } /** @@ -84,7 +95,6 @@ export class ScreenshotStream { /** * Captures a screenshot and stores it in the screenshots array. * Also pushes the screenshot to the newScreenshotCallback if available. - * Bound to the class in the constructor. * @throws Error if recording has not been started. * @throws Error if unable to get 2D rendering context. * @returns The captured screenshot. @@ -116,22 +126,26 @@ export class ScreenshotStream { } /** - * Captures a screenshot after the next repaint and adds it to the screenshot array. - * Useful if you want to screenshot a state change that you've just processed. - * Bound to the class in the constructor. + * Captures a screenshot after a timeout delay. + * If one timeout is already active, a screenshot is taken immediately + * and the old timeout is cleared and replaced with the new delay. + * @param timeout The delay in milliseconds before capturing the screenshot. */ - public captureNextRepaint(): void { - requestAnimationFrame(() => { - const mc = new MessageChannel(); - mc.port1.onmessage = this.capture; - mc.port2.postMessage(undefined); - }); + public delayCapture(timeout: number): void { + if (this.timeout) { + clearTimeout(this.timeout); + this.timeout = null; + this.capture(); + } + this.timeout = setTimeout(() => { + this.capture(); + this.timeout = null; + }, timeout); } /** * Stops the media stream and removes the video element from the DOM. * Must be called to prevent memory leaks. - * Bound to the class in the constructor. */ public stop(): void { if (this.video) { From 28e0fc89c05c808b47f53da52bf7ce0b54c31c40 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Sun, 12 May 2024 13:44:54 -0600 Subject: [PATCH 05/13] docs(reg.ts): document registry --- packages/core/src/registry/reg.ts | 92 ++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 13 deletions(-) diff --git a/packages/core/src/registry/reg.ts b/packages/core/src/registry/reg.ts index 4adfcca..67407b9 100644 --- a/packages/core/src/registry/reg.ts +++ b/packages/core/src/registry/reg.ts @@ -13,31 +13,86 @@ import { enablePatches(); - +/** + * Represents a registered trrack action. + */ type TrrackActionRegisteredObject = { - func: TrrackActionFunction | ProduceWrappedStateChangeFunction; + func: + | TrrackActionFunction + | ProduceWrappedStateChangeFunction; config: TrrackActionConfig; }; -function prepareAction(action: TrrackActionFunction | StateChangeFunction) { - return action.length === 2 ? produce(action) as unknown as ProduceWrappedStateChangeFunction : action as TrrackActionFunction; +/** + * Prepares an action function for registration by wrapping it with the `produce` function if it has two arguments. + * @param action - The action function to prepare. + * @returns The prepared action function. + */ +function prepareAction( + action: + | TrrackActionFunction + | StateChangeFunction +) { + return action.length === 2 + ? (produce(action) as unknown as ProduceWrappedStateChangeFunction) + : (action as TrrackActionFunction); } +/** + * Represents a registry for managing trrack actions. + */ export class Registry { + /** + * Creates a new instance of the `Registry` class. + * @returns A new instance of the `Registry` class. + */ static create(): Registry { return new Registry(); } + /** + * The registry for storing TrrackActionRegisteredObject objects. + */ private registry: Map; + /** + * Creates a new instance of the `Registry` class. + */ private constructor() { this.registry = new Map(); } + /** + * Checks if an action with the specified name is registered in the registry. + * @param name - The name of the action to check. + * @returns `true` if the action is registered, `false` otherwise. + */ has(name: string) { return this.registry.has(name); } + /** + * Registers a new action in the registry. + * + * @template DoActionType - The type of the action to be registered. + * @template UndoActionType - The type of the undo action associated with the registered action. + * @template DoActionPayload - The payload type for the action. + * @template UndoActionPayload - The payload type for the undo action. + * @template State - The state type. + * + * @param {DoActionType} type - The type of the action. + * @param {TrrackActionFunction + * | StateChangeFunction} actionFunction + * - The action function or state change function associated with the action. + * @param {Object} [config] - Optional configuration for the action. + * @param {Event} [config.eventType] - The event type associated with the action. + * @param {Label | LabelGenerator} [config.label] - The label or label generator for the action. + * + * @throws {Error} If the action function has more than two arguments. + * @throws {Error} If the action is already registered. + * + * @returns {Action} The created action. + */ register< DoActionType extends string, UndoActionType extends string, @@ -46,25 +101,30 @@ export class Registry { State = any >( type: DoActionType, - actionFunction: TrrackActionFunction< - DoActionType, - UndoActionType, - UndoActionPayload, - DoActionPayload - > | StateChangeFunction, + actionFunction: + | TrrackActionFunction< + DoActionType, + UndoActionType, + UndoActionPayload, + DoActionPayload + > + | StateChangeFunction, config?: { eventType: Event; - label: Label | LabelGenerator + label: Label | LabelGenerator; } ) { const isState = actionFunction.length === 2; if (actionFunction.length > 2) - throw new Error('Incorrect action function signature. Action function can only have two arguments at most!'); + throw new Error( + 'Incorrect action function signature. Action function can only have two arguments at most!' + ); if (this.has(type)) throw new Error(`Already registered: ${type}`); - const { label = type, eventType = type as unknown as Event } = config || {}; + const { label = type, eventType = type as unknown as Event } = + config || {}; this.registry.set(type, { func: prepareAction(actionFunction), @@ -81,6 +141,12 @@ export class Registry { return createAction(type); } + /** + * Gets the registered action with the specified type. + * @param type - The type of the action to get. + * @returns The registered action. + * @throws An error if the action is not registered. + */ get(type: string) { const action = this.registry.get(type); From 9a5b5b9f415d67507406ab1d77a2ec67c6036c72 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Mon, 13 May 2024 12:46:10 -0600 Subject: [PATCH 06/13] refactor(screenshot-stream.ts, types.ts): refactor screenshot stream to use functional approach ScreenshotStream now uses the functional approach that the rest of Trrack uses instead of OO/class based --- .../core/src/provenance/screenshot-stream.ts | 162 +++++++++--------- packages/core/src/provenance/types.ts | 10 ++ 2 files changed, 92 insertions(+), 80 deletions(-) diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts index 275338a..362f698 100644 --- a/packages/core/src/provenance/screenshot-stream.ts +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -1,4 +1,5 @@ /** + * Factory function to create an instance of ScreenshotStream. * Captures and stores a sequence of screenshots of the current tab. * First, opens a MediaStream of the current tab, then can capture screenshots. * A screenshot can also be captured on-demand via capture() or after a delay via delayCapture(). @@ -7,84 +8,73 @@ * will result in a memory leak. * All functions are bound to the class, so they can be passed as callbacks. */ -export class ScreenshotStream { +export function intitializeScreenshotStream( + newScreenshotCallbacks?: ((frame: ImageData) => void)[] +) { + /// Fields /** * Video element for capturing screenshots. Null if not started or stopped. */ - private video: HTMLVideoElement | null = null; + let video: HTMLVideoElement | null = null; /** * Array of captured screenshots. */ - private screenshots: ImageData[] = []; + const screenshots: ImageData[] = []; /** * ID of the timeout for delayCapture. * Null if no timeout is active. */ - private timeout: NodeJS.Timeout | null = null; + let currentTimeout: NodeJS.Timeout | null = null; /** - * Optional callback to run after each screenshot is captured. + * Optional callbacks to run after each screenshot is captured. */ - public newScreenshotCallback: ((frame: ImageData) => void) | null; + const newScreenshotCallback: ((frame: ImageData) => void)[] | null = + newScreenshotCallbacks ?? null; - /** - * Checks that the getDisplayMedia API is available - * and binds all functions to the class. - * @throws Error if the getDisplayMedia API is not available. - */ - constructor(newScreenshotCallback?: (frame: ImageData) => void) { - if (!navigator.mediaDevices?.getDisplayMedia) { - throw new Error( - 'MediaDevices API or getDisplayMedia() not available' - ); - } - - this.newScreenshotCallback = newScreenshotCallback ?? null; - - this.capture = this.capture.bind(this); - this.stop = this.stop.bind(this); - this.delayCapture = this.delayCapture.bind(this); - this.push = this.push.bind(this); - this.getNth = this.getNth.bind(this); - this.count = this.count.bind(this); - this.getAll = this.getAll.bind(this); - this.start = this.start.bind(this); + /// Constructor + if (!navigator.mediaDevices?.getDisplayMedia) { + throw new Error('MediaDevices API or getDisplayMedia() not available'); } + /// Methods /** * Starts the media stream needed to capture screenshots on-demand. * Will prompt the user for permission to capture the screen. * @throws Error if unable to start the recording; usually due to the user denying permission. * @param callback Optional callback to run after the stream is started. */ - public async start(callback?: () => void): Promise { - this.video = document.createElement('video'); - this.video.autoplay = true; - this.video.muted = true; - this.video.playsInline = true; - this.video.style.pointerEvents = 'none'; - this.video.style.visibility = 'hidden'; - this.video.style.position = 'fixed'; - this.video.style.top = '0'; - this.video.style.left = '0'; + async function start(callback?: () => void): Promise { + video = document.createElement('video'); + video.autoplay = true; + video.muted = true; + video.playsInline = true; + video.style.pointerEvents = 'none'; + video.style.visibility = 'hidden'; + video.style.position = 'fixed'; + video.style.top = '0'; + video.style.left = '0'; try { await navigator.mediaDevices - .getDisplayMedia(/*displayMediaOptions*/) + // Need to cast because TS doesn't know about preferCurrentTab + .getDisplayMedia({ + preferCurrentTab: true, + } as DisplayMediaStreamOptions) .then((stream) => { - // TS is not confident that this.video is not null (but I am), so we need to check - this.video ? (this.video.srcObject = stream) : null; + // TS is not confident that video is not null (but I am), so we need to check + video ? (video.srcObject = stream) : null; }); } catch (e) { - this.video = null; + video = null; throw new Error(`Unable to start recording: ${e}`); } - if (this.video.srcObject) { + if (video.srcObject) { // Needs to be in the DOM to capture screenshots - document.body.appendChild(this.video); + document.body.appendChild(video); callback ? callback() : null; } else { // I honestly don't know how we'd get here @@ -92,6 +82,18 @@ export class ScreenshotStream { } } + /** + * Pushes a screenshot frame to the screenshots array + * and invokes the newScreenshotCallbacks if available. + * @param frame - The screenshot frame to be pushed. + */ + function push(frame: ImageData): void { + screenshots.push(frame); + for (const callback of newScreenshotCallback ?? []) { + callback(frame); + } + } + /** * Captures a screenshot and stores it in the screenshots array. * Also pushes the screenshot to the newScreenshotCallback if available. @@ -99,12 +101,12 @@ export class ScreenshotStream { * @throws Error if unable to get 2D rendering context. * @returns The captured screenshot. */ - public capture(): ImageData { - if (!this.video) { + function capture(): ImageData { + if (!video) { throw new Error('Recording not started'); } - const videoSettings = (this.video.srcObject as MediaStream) + const videoSettings = (video.srcObject as MediaStream) ?.getVideoTracks()[0] .getSettings(); const canvas = document.createElement('canvas'); @@ -116,10 +118,10 @@ export class ScreenshotStream { // GetContext can return undefined and null (probably due to lack of browser support) throw new Error('Unable to get 2D rendering context'); } - context.drawImage(this.video, 0, 0, canvas.width, canvas.height); + context.drawImage(video, 0, 0, canvas.width, canvas.height); const frame = context.getImageData(0, 0, canvas.width, canvas.height); - this.push(frame); + push(frame); canvas.remove(); return frame; @@ -131,15 +133,15 @@ export class ScreenshotStream { * and the old timeout is cleared and replaced with the new delay. * @param timeout The delay in milliseconds before capturing the screenshot. */ - public delayCapture(timeout: number): void { - if (this.timeout) { - clearTimeout(this.timeout); - this.timeout = null; - this.capture(); + function delayCapture(timeout: number): void { + if (currentTimeout) { + clearTimeout(currentTimeout); + currentTimeout = null; + capture(); } - this.timeout = setTimeout(() => { - this.capture(); - this.timeout = null; + currentTimeout = setTimeout(() => { + capture(); + currentTimeout = null; }, timeout); } @@ -147,52 +149,52 @@ export class ScreenshotStream { * Stops the media stream and removes the video element from the DOM. * Must be called to prevent memory leaks. */ - public stop(): void { - if (this.video) { - this.video.srcObject = null; - this.video.remove(); - this.video = null; + function stop(): void { + if (video) { + video.srcObject = null; + video.remove(); + video = null; } } - /** - * Pushes a screenshot frame to the `screenshots` array - * and invokes the `newScreenshotCallback` if available. - * @param frame - The screenshot frame to be pushed. - */ - private push(frame: ImageData): void { - this.screenshots.push(frame); - this.newScreenshotCallback ? this.newScreenshotCallback(frame) : null; - } - /** * Returns the nth most recent screenshot in the array of stored screenshots. * @param n - The index of the screenshot to retrieve. - * 1 is the most recent screenshot, 0 is the least recent. + * 1 is the most recent screenshot, 0 is the least recent. * @returns The nth screenshot. */ - public getNth(n: number): ImageData { - if (n < 0 || n >= this.screenshots.length) { + function getNth(n: number): ImageData { + if (n < 0 || n >= screenshots.length) { throw new Error(`Screenshot index out of bounds: ${n}`); } - return this.screenshots[this.screenshots.length - 1 - n]; + return screenshots[screenshots.length - 1 - n]; } /** * Returns the number of stored screenshots. * @returns The number of stored screenshots. */ - public count(): number { - return this.screenshots.length; + function count(): number { + return screenshots.length; } /** * Returns a copy of the array of stored screenshots. * @returns The stored screenshots. */ - public getAll(): ImageData[] { - return [...this.screenshots]; + function getAll(): ImageData[] { + return [...screenshots]; } + + return { + start, + capture, + delayCapture, + stop, + getNth, + count, + getAll, + }; } /** diff --git a/packages/core/src/provenance/types.ts b/packages/core/src/provenance/types.ts index 4f36924..5ea8160 100644 --- a/packages/core/src/provenance/types.ts +++ b/packages/core/src/provenance/types.ts @@ -22,6 +22,16 @@ export type RecordActionArgs = { onlySideEffects?: boolean; }; +export interface ScreenshotStream { + start(): void; + capture(): void; + delayCapture(): void; + stop(): void; + getNth(n: number): ImageData; + count(): number; + getAll(): ImageData[]; +} + export interface Trrack { registry: Registry; isTraversing: boolean; From e9d1c291555099ac3d738f527afe48b9f01879ea Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Mon, 13 May 2024 17:04:25 -0600 Subject: [PATCH 07/13] feat(core/provenance): integrate screenshots with trrack actions Adds an API to the event registry allowing users to define whether actions should trigger screenshots and the associated delay. Adds an API to the trrack object for screenshots. --- .../core/src/provenance/screenshot-stream.ts | 30 +++++++++++++----- packages/core/src/provenance/trrack.ts | 31 +++++++++++++++++++ packages/core/src/provenance/types.ts | 10 ++++-- packages/core/src/registry/reg.ts | 26 ++++++++++++++-- 4 files changed, 84 insertions(+), 13 deletions(-) diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts index 362f698..37e04ff 100644 --- a/packages/core/src/provenance/screenshot-stream.ts +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -1,3 +1,5 @@ +import { ScreenshotStream } from './types'; + /** * Factory function to create an instance of ScreenshotStream. * Captures and stores a sequence of screenshots of the current tab. @@ -10,7 +12,7 @@ */ export function intitializeScreenshotStream( newScreenshotCallbacks?: ((frame: ImageData) => void)[] -) { +): ScreenshotStream { /// Fields /** * Video element for capturing screenshots. Null if not started or stopped. @@ -62,7 +64,7 @@ export function intitializeScreenshotStream( // Need to cast because TS doesn't know about preferCurrentTab .getDisplayMedia({ preferCurrentTab: true, - } as DisplayMediaStreamOptions) + } as MediaStreamConstraints) .then((stream) => { // TS is not confident that video is not null (but I am), so we need to check video ? (video.srcObject = stream) : null; @@ -132,17 +134,21 @@ export function intitializeScreenshotStream( * If one timeout is already active, a screenshot is taken immediately * and the old timeout is cleared and replaced with the new delay. * @param timeout The delay in milliseconds before capturing the screenshot. + * If 0, the screenshot is captured immediately. */ function delayCapture(timeout: number): void { if (currentTimeout) { clearTimeout(currentTimeout); currentTimeout = null; capture(); - } - currentTimeout = setTimeout(() => { + } else if (timeout == 0) { capture(); - currentTimeout = null; - }, timeout); + } else { + currentTimeout = setTimeout(() => { + capture(); + currentTimeout = null; + }, timeout); + } } /** @@ -159,8 +165,7 @@ export function intitializeScreenshotStream( /** * Returns the nth most recent screenshot in the array of stored screenshots. - * @param n - The index of the screenshot to retrieve. - * 1 is the most recent screenshot, 0 is the least recent. + * @param n - The index of the screenshot to retrieve. 0 is the most recent. * @returns The nth screenshot. */ function getNth(n: number): ImageData { @@ -186,6 +191,14 @@ export function intitializeScreenshotStream( return [...screenshots]; } + /** + * Returns whether the screenshot stream is currently recording. + * @returns Whether the screenshot stream is currently recording. + */ + function isRecording(): boolean { + return video !== null; + } + return { start, capture, @@ -194,6 +207,7 @@ export function intitializeScreenshotStream( getNth, count, getAll, + isRecording, }; } diff --git a/packages/core/src/provenance/trrack.ts b/packages/core/src/provenance/trrack.ts index f228cd0..a2c8688 100644 --- a/packages/core/src/provenance/trrack.ts +++ b/packages/core/src/provenance/trrack.ts @@ -24,6 +24,7 @@ import { } from '../registry'; import { ConfigureTrrackOptions } from './trrack-config-opts'; import { TrrackEvents } from './trrack-events'; +import { intitializeScreenshotStream } from './screenshot-stream'; function getState( node: ProvenanceNode, @@ -78,6 +79,12 @@ export function initializeTrrack({ const eventManager = initEventManager(); const graph = initializeProvenanceGraph(initialState); + /** + * Retrieves a node from the graph based on its ID. + * + * @param id - The ID of the node. + * @returns The node with the specified ID. + */ function getNode(id: NodeId) { return graph.backend.nodes[id]; } @@ -90,6 +97,8 @@ export function initializeTrrack({ isTraversing = false; }); + const screenshots = intitializeScreenshotStream(); + const metadata = { add( metadata: Record, @@ -297,6 +306,9 @@ export function initializeTrrack({ eventType: action.config.eventType as Event, }); } + + if (screenshots.isRecording() && action.triggersScreenshot) + screenshots.delayCapture(action.transitionTime); }, async to(node: NodeId) { eventManager.fire(TrrackEvents.TRAVERSAL_START); @@ -308,9 +320,22 @@ export function initializeTrrack({ ); const sideEffectsToApply: Array> = []; + // Only take a screenshot if we find a node in the path that triggers one. Use + // the max transition timer encountered when screenshotting + let maxTimer = -1; for (let i = 0; i < path.length - 1; ++i) { const currentNode = getNode(path[i]); + + if (screenshots.isRecording()) { + const action = registry.get(currentNode.event); + if ( + action.triggersScreenshot && + action.transitionTime > maxTimer + ) + maxTimer = action.transitionTime; + } + const nextNode = getNode(path[i + 1]); const isUndo = isNextNodeUp(currentNode, nextNode); @@ -334,6 +359,10 @@ export function initializeTrrack({ graph.update(graph.changeCurrent(node)); + if (screenshots.isRecording() && maxTimer >= 0) { + screenshots.delayCapture(maxTimer); + } + eventManager.fire(TrrackEvents.TRAVERSAL_END); }, undo() { @@ -367,6 +396,7 @@ export function initializeTrrack({ }); }, done() { + if (screenshots.isRecording()) screenshots.stop(); console.log('Setup later for URL sharing.'); }, tree() { @@ -401,6 +431,7 @@ export function initializeTrrack({ artifact, annotations, bookmarks, + screenshots, }; } diff --git a/packages/core/src/provenance/types.ts b/packages/core/src/provenance/types.ts index 5ea8160..d6dcb44 100644 --- a/packages/core/src/provenance/types.ts +++ b/packages/core/src/provenance/types.ts @@ -24,12 +24,13 @@ export type RecordActionArgs = { export interface ScreenshotStream { start(): void; - capture(): void; - delayCapture(): void; + capture(): ImageData; + delayCapture(timeout: number): void; stop(): void; getNth(n: number): ImageData; count(): number; getAll(): ImageData[]; + isRecording(): boolean; } export interface Trrack { @@ -88,4 +89,9 @@ export interface Trrack { exportObject(): ProvenanceGraph; import(graphString: string): void; importObject(graph: ProvenanceGraph): void; + /** + * Interface for capturing screenshots. When activated, + * captures a screenshot after certain actions fire. + */ + screenshots: ScreenshotStream; } diff --git a/packages/core/src/registry/reg.ts b/packages/core/src/registry/reg.ts index 67407b9..01b98af 100644 --- a/packages/core/src/registry/reg.ts +++ b/packages/core/src/registry/reg.ts @@ -14,13 +14,20 @@ import { enablePatches(); /** - * Represents a registered trrack action. + * @typedef {Object} TrrackActionRegisteredObject - Represents a registered trrack action. + * @property {TrrackActionFunction + * | ProduceWrappedStateChangeFunction} func + * - The action function or state change function associated with the action. + * @property {TrrackActionConfig} config - The configuration for the action. + * @property {number} transitionTime - The transition time for the action. */ type TrrackActionRegisteredObject = { func: | TrrackActionFunction | ProduceWrappedStateChangeFunction; config: TrrackActionConfig; + triggersScreenshot: boolean; + transitionTime: number; }; /** @@ -82,14 +89,21 @@ export class Registry { * * @param {DoActionType} type - The type of the action. * @param {TrrackActionFunction - * | StateChangeFunction} actionFunction - * - The action function or state change function associated with the action. + * | StateChangeFunction} actionFunction + * - The action function or state change function associated with the action. + * @param {number} [transitionTime=100] - The amount of time taken for the effect of the action to be reflected in the + * browser display. When screenshotting is enabled, the screenshot will be taken after this delay. + * If set to 0, a screenshot is taken immediately; this usually does not give the browser enough time to update + * the display, so the screenshot may not reflect the changes. + * @param {boolean} [triggersScreenshot=false] - Indicates whether the action triggers a screenshot if screenshots + * have been enabled for the current Trrack instance. * @param {Object} [config] - Optional configuration for the action. * @param {Event} [config.eventType] - The event type associated with the action. * @param {Label | LabelGenerator} [config.label] - The label or label generator for the action. * * @throws {Error} If the action function has more than two arguments. * @throws {Error} If the action is already registered. + * @throws {Error} If the transition time is negative. * * @returns {Action} The created action. */ @@ -109,6 +123,8 @@ export class Registry { DoActionPayload > | StateChangeFunction, + triggersScreenshot = false, + transitionTime = 100, config?: { eventType: Event; label: Label | LabelGenerator; @@ -120,6 +136,8 @@ export class Registry { throw new Error( 'Incorrect action function signature. Action function can only have two arguments at most!' ); + if (transitionTime < 0) + throw new Error('Transition time cannot be negative'); if (this.has(type)) throw new Error(`Already registered: ${type}`); @@ -136,6 +154,8 @@ export class Registry { : label, eventType, }, + transitionTime, + triggersScreenshot, }); return createAction(type); From d2cb3d2f08f3c690c0f7157daf6e1eb64ecfc174 Mon Sep 17 00:00:00 2001 From: Nate Lanza Date: Mon, 13 May 2024 17:06:33 -0600 Subject: [PATCH 08/13] chore(apps/react-trrack/example): integrate with new screenshot APIs --- apps/react-trrack-example/src/app/App.tsx | 13 ++++------ .../src/app/components/Navbar.tsx | 4 +-- .../src/app/store/trrack.ts | 26 ++++++++++--------- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/apps/react-trrack-example/src/app/App.tsx b/apps/react-trrack-example/src/app/App.tsx index 4546a8e..d894920 100644 --- a/apps/react-trrack-example/src/app/App.tsx +++ b/apps/react-trrack-example/src/app/App.tsx @@ -13,8 +13,7 @@ import { TreeNode } from 'react-hyper-tree/dist/helpers/node'; import { Navbar } from './components/Navbar'; import { useTrrackTaskManager } from './store/trrack'; -import { useRef } from 'react'; -import { ScreenshotStream, downloadScreenshot } from '@trrack/core'; +import { downloadScreenshot } from '@trrack/core'; function App() { const trrackManager = useTrrackTaskManager(); @@ -30,19 +29,17 @@ function App() { open(required.data, trrackManager.trrack.current.id); // Testing screenshot stream - const ss = useRef(new ScreenshotStream()); + const ss = trrackManager.trrack.screenshots; return ( - -