diff --git a/apps/react-trrack-example/src/app/App.tsx b/apps/react-trrack-example/src/app/App.tsx index 388ec21..2d3a9a4 100644 --- a/apps/react-trrack-example/src/app/App.tsx +++ b/apps/react-trrack-example/src/app/App.tsx @@ -1,9 +1,19 @@ -import { Box, Checkbox, List, ListItem, ListItemIcon, ListItemText, Typography } from '@mui/material'; +import { + Box, + Button, + Checkbox, + List, + ListItem, + ListItemIcon, + ListItemText, + Typography, +} from '@mui/material'; import Tree, { useTreeState } from 'react-hyper-tree'; import { TreeNode } from 'react-hyper-tree/dist/helpers/node'; import { Navbar } from './components/Navbar'; import { useTrrackTaskManager } from './store/trrack'; +import { downloadScreenshot } from '@trrack/core'; function App() { const trrackManager = useTrrackTaskManager(); @@ -18,9 +28,26 @@ function App() { open(required.data, trrackManager.trrack.current.id); + // Testing screenshot stream + const ss = trrackManager.trrack.screenshots; + return ( + + - {trrackManager.state.tasks.map((task) => ( + {trrackManager.state.tasks?.map((task) => ( c + add); return { undo: { - type: 'decrement-counter', - payload: add, - meta: { - hasSideEffects: true, + type: 'decrement-counter', + payload: add, + meta: { + hasSideEffects: true, + }, }, - } }; - } + }, + true ); const decrementCounter = reg.register( @@ -63,14 +64,15 @@ export function useTrrackTaskManager() { setCounter((c) => c - sub); return { undo: { - type: 'increment-counter', - payload: sub, - meta: { - hasSideEffects: true, + type: 'increment-counter', + payload: sub, + meta: { + hasSideEffects: true, + }, }, - } }; - } + }, + true ); return { diff --git a/packages/core/src/provenance/index.ts b/packages/core/src/provenance/index.ts index df8096e..7cc4647 100644 --- a/packages/core/src/provenance/index.ts +++ b/packages/core/src/provenance/index.ts @@ -2,3 +2,4 @@ export * from './trrack'; export * from './trrack-config-opts'; export * from './trrack-events'; export * from './types'; +export * from './screenshot-stream'; diff --git a/packages/core/src/provenance/screenshot-stream.ts b/packages/core/src/provenance/screenshot-stream.ts new file mode 100644 index 0000000..1be9084 --- /dev/null +++ b/packages/core/src/provenance/screenshot-stream.ts @@ -0,0 +1,269 @@ +import { ScreenshotStream } from './types'; + +/** + * Factory function to create an instance of ScreenshotStream. + * Captures and stores a sequence of screenshots of the current tab. + * First, opens a MediaStream of the current tab, then can capture screenshots. + * A screenshot can also be captured on-demand via capture() or after a delay via delayCapture(). + * Requires browser permissions to capture screen. + * Must be activated via start() and deactivated via stop(); failure to stop + * will result in a memory leak. + * All functions are bound to the class, so they can be passed as callbacks. + */ +export function intitializeScreenshotStream(): ScreenshotStream { + /// Fields + /** + * Video element for capturing screenshots. Null if not started or stopped. + */ + let video: HTMLVideoElement | null = null; + + /** + * Array of captured screenshots. + */ + const screenshots: ImageData[] = []; + + /** + * ID of the timeout for delayCapture. + * Null if no timeout is active. + */ + let currentTimeout: NodeJS.Timeout | null = null; + + /** + * Optional callbacks to run after each screenshot is captured. + */ + const newScreenshotCallbacks: ((frame: ImageData) => void)[] = []; + + /// Constructor + if (!navigator.mediaDevices?.getDisplayMedia) { + throw new Error('MediaDevices API or getDisplayMedia() not available'); + } + + /// Methods + /** + * Stops the media stream and removes the video element from the DOM. + * Must be called to prevent memory leaks. + */ + function stop(): void { + if (video) { + video.srcObject = null; + video.remove(); + video = null; + } + } + + /** + * False if the MediaStream has not been initialized via start(), or has been stopped. + * True if we have a MediaStream and can capture screenshots. + * @returns Whether a screenshot can be captured. + */ + function canCapture(): boolean { + return video !== null; + } + + /** + * Starts the media stream needed to capture screenshots on-demand. + * Will prompt the user for permission to capture the screen. + * Immediately captures a first screenshot. + * Does nothing if the stream is already started. + * @throws Error if unable to start the recording; usually due to the user denying permission. + * @param callback Optional callback to run after the stream is started. + */ + async function start(callback?: () => void): Promise { + if (canCapture()) { + return; + } + + video = document.createElement('video'); + video.autoplay = true; + video.muted = true; + video.playsInline = true; + video.style.pointerEvents = 'none'; + video.style.visibility = 'hidden'; + video.style.position = 'fixed'; + video.style.top = '0'; + video.style.left = '0'; + + try { + await navigator.mediaDevices + // Need to cast because TS doesn't know about preferCurrentTab + .getDisplayMedia({ + preferCurrentTab: true, + } as MediaStreamConstraints) + .then((stream) => { + // TS is not confident that video is not null (but I am), so we need to check + video ? (video.srcObject = stream) : null; + stream.getVideoTracks()[0].onended = stop; + }); + } catch (e) { + video = null; + throw new Error(`Unable to start recording: ${e}`); + } + + if (video.srcObject) { + // Needs to be in the DOM to capture screenshots + document.body.appendChild(video); + callback ? callback() : null; + } else { + // I honestly don't know how we'd get here + throw new Error('Unable to start recording; no stream available'); + } + + // We should capture initial state + capture(); + } + + /** + * Pushes a screenshot frame to the screenshots array + * and invokes the newScreenshotCallbacks if available. + * @param frame - The screenshot frame to be pushed. + */ + function push(frame: ImageData): void { + screenshots.push(frame); + for (const callback of newScreenshotCallbacks ?? []) { + callback(frame); + } + } + + /** + * Captures a screenshot and stores it in the screenshots array. + * Also pushes the screenshot to the newScreenshotCallback if available. + * @throws Error if recording has not been started. + * @throws Error if unable to get 2D rendering context. + * @returns The captured screenshot. + */ + function capture(): ImageData { + // We need the null check for ts, but canCapture() does that check already. + // We include canCapture() in case the implementation changes. + if (!canCapture() || !video) { + throw new Error('Recording not started'); + } + + const videoSettings = (video.srcObject as MediaStream) + ?.getVideoTracks()[0] + .getSettings(); + const canvas = document.createElement('canvas'); + canvas.width = videoSettings.width || 0; + canvas.height = videoSettings.height || 0; + + const context = canvas.getContext('2d'); + if (!context) { + // GetContext can return undefined and null (probably due to lack of browser support) + throw new Error('Unable to get 2D rendering context'); + } + context.drawImage(video, 0, 0, canvas.width, canvas.height); + + const frame = context.getImageData(0, 0, canvas.width, canvas.height); + push(frame); + + canvas.remove(); + return frame; + } + + /** + * Captures a screenshot after a timeout delay. + * If one timeout is already active, a screenshot is taken immediately + * and the old timeout is cleared and replaced with the new delay. + * @param timeout The delay in milliseconds before capturing the screenshot. + * If 0, the screenshot is captured immediately. + */ + function delayCapture(timeout: number): void { + if (currentTimeout) { + clearTimeout(currentTimeout); + currentTimeout = null; + capture(); + } else if (timeout == 0) { + capture(); + } else { + currentTimeout = setTimeout(() => { + capture(); + currentTimeout = null; + }, timeout); + } + } + + /** + * Returns the nth most recent screenshot in the array of stored screenshots. + * @param n - The index of the screenshot to retrieve. 0 is the most recent. + * @returns The nth screenshot. + */ + function getNth(n: number): ImageData | null { + if (screenshots.length === 0) { + return null; + } + + if (n < 0 || n >= screenshots.length) { + throw new Error(`Screenshot index out of bounds: ${n}`); + } + return screenshots[screenshots.length - 1 - n]; + } + + /** + * Returns the number of stored screenshots. + * @returns The number of stored screenshots. + */ + function count(): number { + return screenshots.length; + } + + /** + * Returns a copy of the array of stored screenshots. + * @returns The stored screenshots. + */ + function getAll(): ImageData[] { + return [...screenshots]; + } + + /** + * Registers a listener to be called when a new screenshot is captured. + * @param listener - The listener to be called when a new screenshot is captured. + * @returns A function to remove the listener. + */ + function registerScreenshotListener( + listener: (image: ImageData) => void + ): () => void { + newScreenshotCallbacks.push(listener); + return () => { + const index = newScreenshotCallbacks.indexOf(listener); + if (index !== -1) { + newScreenshotCallbacks.splice(index, 1); + } + }; + } + + return { + start, + capture, + delayCapture, + stop, + getNth, + count, + getAll, + canCapture: canCapture, + registerScreenshotListener, + }; +} + +/** + * Downloads a screenshot as a PNG file. + * @param frame - The screenshot frame to download. + * @param name - The name of the file to download. + */ +export function downloadScreenshot(frame: ImageData, name: string): void { + const canvas = document.createElement('canvas'); + canvas.width = frame.width; + canvas.height = frame.height; + + const context = canvas.getContext('2d'); + if (!context) { + throw new Error('Unable to get 2D rendering context'); + } + context.putImageData(frame, 0, 0); + + const a = document.createElement('a'); + a.href = canvas.toDataURL(); + a.download = name; + a.click(); + + canvas.remove(); + a.remove(); +} diff --git a/packages/core/src/provenance/trrack.ts b/packages/core/src/provenance/trrack.ts index f228cd0..cddc1d9 100644 --- a/packages/core/src/provenance/trrack.ts +++ b/packages/core/src/provenance/trrack.ts @@ -24,6 +24,7 @@ import { } from '../registry'; import { ConfigureTrrackOptions } from './trrack-config-opts'; import { TrrackEvents } from './trrack-events'; +import { intitializeScreenshotStream } from './screenshot-stream'; function getState( node: ProvenanceNode, @@ -78,6 +79,12 @@ export function initializeTrrack({ const eventManager = initEventManager(); const graph = initializeProvenanceGraph(initialState); + /** + * Retrieves a node from the graph based on its ID. + * + * @param id - The ID of the node. + * @returns The node with the specified ID. + */ function getNode(id: NodeId) { return graph.backend.nodes[id]; } @@ -90,6 +97,8 @@ export function initializeTrrack({ isTraversing = false; }); + const screenshots = intitializeScreenshotStream(); + const metadata = { add( metadata: Record, @@ -297,6 +306,9 @@ export function initializeTrrack({ eventType: action.config.eventType as Event, }); } + + if (screenshots.canCapture() && action.triggersScreenshot) + screenshots.delayCapture(action.transitionTime); }, async to(node: NodeId) { eventManager.fire(TrrackEvents.TRAVERSAL_START); @@ -308,9 +320,22 @@ export function initializeTrrack({ ); const sideEffectsToApply: Array> = []; + // Only take a screenshot if we find a node in the path that triggers one. Use + // the max transition timer encountered when screenshotting + let maxTimer = -1; for (let i = 0; i < path.length - 1; ++i) { const currentNode = getNode(path[i]); + + if (screenshots.canCapture()) { + const action = registry.get(currentNode.event); + if ( + action.triggersScreenshot && + action.transitionTime > maxTimer + ) + maxTimer = action.transitionTime; + } + const nextNode = getNode(path[i + 1]); const isUndo = isNextNodeUp(currentNode, nextNode); @@ -334,6 +359,10 @@ export function initializeTrrack({ graph.update(graph.changeCurrent(node)); + if (screenshots.canCapture() && maxTimer >= 0) { + screenshots.delayCapture(maxTimer); + } + eventManager.fire(TrrackEvents.TRAVERSAL_END); }, undo() { @@ -367,6 +396,7 @@ export function initializeTrrack({ }); }, done() { + if (screenshots.canCapture()) screenshots.stop(); console.log('Setup later for URL sharing.'); }, tree() { @@ -401,6 +431,7 @@ export function initializeTrrack({ artifact, annotations, bookmarks, + screenshots, }; } diff --git a/packages/core/src/provenance/types.ts b/packages/core/src/provenance/types.ts index d67c109..1773d59 100644 --- a/packages/core/src/provenance/types.ts +++ b/packages/core/src/provenance/types.ts @@ -1,5 +1,5 @@ import { PayloadAction } from '@reduxjs/toolkit'; -import { NodeId } from '@trrack/core'; +import { NodeId } from '../graph/components/node'; import { Artifact, CurrentChangeHandler, @@ -22,6 +22,54 @@ export type RecordActionArgs = { onlySideEffects?: boolean; }; +export interface ScreenshotStream { + /** + * Presents a dialog to the user to record the current tab, begins a video stream, + * and enables the capture of screenshots. + */ + start(): void; + /** + * Immediately captures a screenshot from the video stream. + * @returns The captured screenshot. + */ + capture(): ImageData; + /** + * Captures a screenshot after a delay. + * @param timeout - The amount of time to delay the capture in ms. + */ + delayCapture(timeout: number): void; + /** + * Stops the video stream and screenshot capture. + */ + stop(): void; + /** + * Returns the nth most recent screenshot in the array of stored screenshots. + * @param n - The index of the screenshot to retrieve. 0 is the most recent. + * @returns The nth screenshot. + */ + getNth(n: number): ImageData | null; + /** + * Returns the number of stored screenshots. + */ + count(): number; + /** + * Returns a copy of the array of stored screenshots. + */ + getAll(): ImageData[]; + /** + * @returns whether capturing is allowed. Generally is false before start() is called and after stop() is called. + */ + canCapture(): boolean; + /** + * Registers a listener to be called when a screenshot is captured. + * @param listener - The listener to register. + * @returns A function to unregister the listener. + */ + registerScreenshotListener( + listener: (image: ImageData) => void + ): () => void; +} + export interface Trrack { registry: Registry; isTraversing: boolean; @@ -78,4 +126,9 @@ export interface Trrack { exportObject(): ProvenanceGraph; import(graphString: string): void; importObject(graph: ProvenanceGraph): void; + /** + * Interface for capturing screenshots. When activated, + * captures a screenshot after certain actions fire. + */ + screenshots: ScreenshotStream; } diff --git a/packages/core/src/registry/reg.ts b/packages/core/src/registry/reg.ts index 4adfcca..01b98af 100644 --- a/packages/core/src/registry/reg.ts +++ b/packages/core/src/registry/reg.ts @@ -13,31 +13,100 @@ import { enablePatches(); - +/** + * @typedef {Object} TrrackActionRegisteredObject - Represents a registered trrack action. + * @property {TrrackActionFunction + * | ProduceWrappedStateChangeFunction} func + * - The action function or state change function associated with the action. + * @property {TrrackActionConfig} config - The configuration for the action. + * @property {number} transitionTime - The transition time for the action. + */ type TrrackActionRegisteredObject = { - func: TrrackActionFunction | ProduceWrappedStateChangeFunction; + func: + | TrrackActionFunction + | ProduceWrappedStateChangeFunction; config: TrrackActionConfig; + triggersScreenshot: boolean; + transitionTime: number; }; -function prepareAction(action: TrrackActionFunction | StateChangeFunction) { - return action.length === 2 ? produce(action) as unknown as ProduceWrappedStateChangeFunction : action as TrrackActionFunction; +/** + * Prepares an action function for registration by wrapping it with the `produce` function if it has two arguments. + * @param action - The action function to prepare. + * @returns The prepared action function. + */ +function prepareAction( + action: + | TrrackActionFunction + | StateChangeFunction +) { + return action.length === 2 + ? (produce(action) as unknown as ProduceWrappedStateChangeFunction) + : (action as TrrackActionFunction); } +/** + * Represents a registry for managing trrack actions. + */ export class Registry { + /** + * Creates a new instance of the `Registry` class. + * @returns A new instance of the `Registry` class. + */ static create(): Registry { return new Registry(); } + /** + * The registry for storing TrrackActionRegisteredObject objects. + */ private registry: Map; + /** + * Creates a new instance of the `Registry` class. + */ private constructor() { this.registry = new Map(); } + /** + * Checks if an action with the specified name is registered in the registry. + * @param name - The name of the action to check. + * @returns `true` if the action is registered, `false` otherwise. + */ has(name: string) { return this.registry.has(name); } + /** + * Registers a new action in the registry. + * + * @template DoActionType - The type of the action to be registered. + * @template UndoActionType - The type of the undo action associated with the registered action. + * @template DoActionPayload - The payload type for the action. + * @template UndoActionPayload - The payload type for the undo action. + * @template State - The state type. + * + * @param {DoActionType} type - The type of the action. + * @param {TrrackActionFunction + * | StateChangeFunction} actionFunction + * - The action function or state change function associated with the action. + * @param {number} [transitionTime=100] - The amount of time taken for the effect of the action to be reflected in the + * browser display. When screenshotting is enabled, the screenshot will be taken after this delay. + * If set to 0, a screenshot is taken immediately; this usually does not give the browser enough time to update + * the display, so the screenshot may not reflect the changes. + * @param {boolean} [triggersScreenshot=false] - Indicates whether the action triggers a screenshot if screenshots + * have been enabled for the current Trrack instance. + * @param {Object} [config] - Optional configuration for the action. + * @param {Event} [config.eventType] - The event type associated with the action. + * @param {Label | LabelGenerator} [config.label] - The label or label generator for the action. + * + * @throws {Error} If the action function has more than two arguments. + * @throws {Error} If the action is already registered. + * @throws {Error} If the transition time is negative. + * + * @returns {Action} The created action. + */ register< DoActionType extends string, UndoActionType extends string, @@ -46,25 +115,34 @@ export class Registry { State = any >( type: DoActionType, - actionFunction: TrrackActionFunction< - DoActionType, - UndoActionType, - UndoActionPayload, - DoActionPayload - > | StateChangeFunction, + actionFunction: + | TrrackActionFunction< + DoActionType, + UndoActionType, + UndoActionPayload, + DoActionPayload + > + | StateChangeFunction, + triggersScreenshot = false, + transitionTime = 100, config?: { eventType: Event; - label: Label | LabelGenerator + label: Label | LabelGenerator; } ) { const isState = actionFunction.length === 2; if (actionFunction.length > 2) - throw new Error('Incorrect action function signature. Action function can only have two arguments at most!'); + throw new Error( + 'Incorrect action function signature. Action function can only have two arguments at most!' + ); + if (transitionTime < 0) + throw new Error('Transition time cannot be negative'); if (this.has(type)) throw new Error(`Already registered: ${type}`); - const { label = type, eventType = type as unknown as Event } = config || {}; + const { label = type, eventType = type as unknown as Event } = + config || {}; this.registry.set(type, { func: prepareAction(actionFunction), @@ -76,11 +154,19 @@ export class Registry { : label, eventType, }, + transitionTime, + triggersScreenshot, }); return createAction(type); } + /** + * Gets the registered action with the specified type. + * @param type - The type of the action to get. + * @returns The registered action. + * @throws An error if the action is not registered. + */ get(type: string) { const action = this.registry.get(type);