From 94ff9a0075fb4c7bcc1cfd656097859a634bcd22 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 22 Nov 2024 10:50:04 -0800 Subject: [PATCH 01/18] add json-schema spec for the workflow --- json-schema/workflow.schema.json | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 json-schema/workflow.schema.json diff --git a/json-schema/workflow.schema.json b/json-schema/workflow.schema.json new file mode 100644 index 00000000..15e3fcec --- /dev/null +++ b/json-schema/workflow.schema.json @@ -0,0 +1,107 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Workflow", + "type": "object", + "properties": { + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/Task" + }, + "description": "Worfklow tasks." + }, + "name": { + "type": "string", + "description": "The name of the workflow." + }, + "parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Optional parameters for the workflow." + }, + "schedule": { + "type": "string", + "description": "Optional schedule in cron format." + }, + "timezone": { + "type": "string", + "description": "Timezone for the schedule." + } + }, + "required": [ + "tasks", + "name" + ], + "definitions": { + "Task": { + "type": "object", + "properties": { + "input_uri": { + "type": "string", + "description": "The URI of the input file." + }, + "runtime_environment_name": { + "type": "string", + "description": "Name of the runtime environment." + }, + "runtime_environment_parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Parameters for the runtime environment." + }, + "output_formats": { + "type": "array", + "items": { + "type": "string" + } + }, + "parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Task-specific parameters." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags for categorizing the job." + }, + "name": { + "type": "string", + "description": "Name of the job." + }, + "compute_type": { + "type": "string", + "description": "Type of compute resource to use." + }, + "package_input_folder": { + "type": "boolean", + "description": "Whether to package the input folder." + }, + "depends_on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "DAG node IDs of tasks this task depends on." + }, + "node_id": { + "type": "string", + "description": "DAG node ID of this task." + } + }, + "required": [ + "input_uri", + "name", + "node_id" + ] + } + } +} \ No newline at end of file From ad8e9d4bf2667c9aef773eed31023bb5d6bb37fa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:51:27 +0000 Subject: [PATCH 02/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- json-schema/workflow.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json-schema/workflow.schema.json b/json-schema/workflow.schema.json index 15e3fcec..ff1911f0 100644 --- a/json-schema/workflow.schema.json +++ b/json-schema/workflow.schema.json @@ -104,4 +104,4 @@ ] } } -} \ No newline at end of file +} From 9cd8800395518b4d60626f6101a754257f6ef380 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 22 Nov 2024 19:07:28 -0800 Subject: [PATCH 03/18] add version field at the schema top level --- json-schema/workflow.schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/json-schema/workflow.schema.json b/json-schema/workflow.schema.json index ff1911f0..6877f646 100644 --- a/json-schema/workflow.schema.json +++ b/json-schema/workflow.schema.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "version": "0.0.1", "title": "Workflow", "type": "object", "properties": { From 4b7b0b92a68d8dbfba64c03c62961a0c30f32b2f Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 22 Nov 2024 21:13:51 -0800 Subject: [PATCH 04/18] run prettier --- json-schema/workflow.schema.json | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/json-schema/workflow.schema.json b/json-schema/workflow.schema.json index 6877f646..b8154383 100644 --- a/json-schema/workflow.schema.json +++ b/json-schema/workflow.schema.json @@ -31,10 +31,7 @@ "description": "Timezone for the schedule." } }, - "required": [ - "tasks", - "name" - ], + "required": ["tasks", "name"], "definitions": { "Task": { "type": "object", @@ -98,11 +95,7 @@ "description": "DAG node ID of this task." } }, - "required": [ - "input_uri", - "name", - "node_id" - ] + "required": ["input_uri", "name", "node_id"] } } } From 5f166e886486420f891cadf251be6c559905ab9d Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Fri, 22 Nov 2024 19:07:28 -0800 Subject: [PATCH 05/18] add version field at the schema top level --- .../schema}/workflow.schema.json | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) rename {json-schema => src/schema}/workflow.schema.json (92%) diff --git a/json-schema/workflow.schema.json b/src/schema/workflow.schema.json similarity index 92% rename from json-schema/workflow.schema.json rename to src/schema/workflow.schema.json index b8154383..e4701622 100644 --- a/json-schema/workflow.schema.json +++ b/src/schema/workflow.schema.json @@ -1,9 +1,12 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "version": "0.0.1", "title": "Workflow", "type": "object", "properties": { + "schemaVersion": { + "type": "string", + "default": "0.0.1" + }, "tasks": { "type": "array", "items": { @@ -31,7 +34,10 @@ "description": "Timezone for the schedule." } }, - "required": ["tasks", "name"], + "required": [ + "tasks", + "name" + ], "definitions": { "Task": { "type": "object", @@ -95,7 +101,11 @@ "description": "DAG node ID of this task." } }, - "required": ["input_uri", "name", "node_id"] + "required": [ + "input_uri", + "name", + "node_id" + ] } } -} +} \ No newline at end of file From ad8688680e3472db4ee009c3c530dc03a9a1712e Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 4 Dec 2024 19:52:07 -0800 Subject: [PATCH 06/18] generate Workflow interfaces from json-schema via jjson2ts --- .gitignore | 3 + package.json | 4 +- src/workflows/schema/workflow.schema.json | 111 ++++++++++++++++++++ yarn.lock | 117 +++++++++++++++++++++- 4 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 src/workflows/schema/workflow.schema.json diff --git a/.gitignore b/.gitignore index 61233810..61a1afa9 100644 --- a/.gitignore +++ b/.gitignore @@ -135,3 +135,6 @@ dev/*.ipynb # Jest coverage reports and a side effect coverage junit.xml + +# Interfaces generated from json-schema +src/workflows/_interface diff --git a/package.json b/package.json index 8c60bbe5..558d05ed 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", "build:lib": "tsc", + "build:schema:js": "json2ts -i src/workflows/schema -o src/workflows/_interface --no-unknownAny --unreachableDefinitions --cwd ./src/workflows/schema", "clean": "jlpm clean:lib", "clean:lib": "rimraf lib tsconfig.tsbuildinfo", "clean:lintcache": "rimraf .eslintcache .stylelintcache", @@ -74,6 +75,7 @@ "@mui/system": "^5.10.6", "@types/react-dom": "^18.0.5", "cronstrue": "^2.12.0", + "json-schema-to-typescript": "^15.0.3", "react": "^18.2.0", "react-dom": "^18.2.0", "tzdata": "^1.0.33" @@ -141,4 +143,4 @@ ] } } -} +} \ No newline at end of file diff --git a/src/workflows/schema/workflow.schema.json b/src/workflows/schema/workflow.schema.json new file mode 100644 index 00000000..e4701622 --- /dev/null +++ b/src/workflows/schema/workflow.schema.json @@ -0,0 +1,111 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Workflow", + "type": "object", + "properties": { + "schemaVersion": { + "type": "string", + "default": "0.0.1" + }, + "tasks": { + "type": "array", + "items": { + "$ref": "#/definitions/Task" + }, + "description": "Worfklow tasks." + }, + "name": { + "type": "string", + "description": "The name of the workflow." + }, + "parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Optional parameters for the workflow." + }, + "schedule": { + "type": "string", + "description": "Optional schedule in cron format." + }, + "timezone": { + "type": "string", + "description": "Timezone for the schedule." + } + }, + "required": [ + "tasks", + "name" + ], + "definitions": { + "Task": { + "type": "object", + "properties": { + "input_uri": { + "type": "string", + "description": "The URI of the input file." + }, + "runtime_environment_name": { + "type": "string", + "description": "Name of the runtime environment." + }, + "runtime_environment_parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Parameters for the runtime environment." + }, + "output_formats": { + "type": "array", + "items": { + "type": "string" + } + }, + "parameters": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Task-specific parameters." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tags for categorizing the job." + }, + "name": { + "type": "string", + "description": "Name of the job." + }, + "compute_type": { + "type": "string", + "description": "Type of compute resource to use." + }, + "package_input_folder": { + "type": "boolean", + "description": "Whether to package the input folder." + }, + "depends_on": { + "type": "array", + "items": { + "type": "string" + }, + "description": "DAG node IDs of tasks this task depends on." + }, + "node_id": { + "type": "string", + "description": "DAG node ID of this task." + } + }, + "required": [ + "input_uri", + "name", + "node_id" + ] + } + } +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index e50f85cf..0e1e80d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25,6 +25,17 @@ __metadata: languageName: node linkType: hard +"@apidevtools/json-schema-ref-parser@npm:^11.5.5": + version: 11.7.2 + resolution: "@apidevtools/json-schema-ref-parser@npm:11.7.2" + dependencies: + "@jsdevtools/ono": ^7.1.3 + "@types/json-schema": ^7.0.15 + js-yaml: ^4.1.0 + checksum: 44096e5cd5a03b17ee5eb0a3b9e9a4db85d87da8ae2abda264eae615f2a43e3e6ba5ca208e1161d4d946755b121c10a9550e88792a725951f2c4cff6df0d8a19 + languageName: node + linkType: hard + "@babel/code-frame@npm:7.12.11": version: 7.12.11 resolution: "@babel/code-frame@npm:7.12.11" @@ -3321,6 +3332,13 @@ __metadata: languageName: node linkType: hard +"@jsdevtools/ono@npm:^7.1.3": + version: 7.1.3 + resolution: "@jsdevtools/ono@npm:7.1.3" + checksum: 2297fcd472ba810bffe8519d2249171132844c7174f3a16634f9260761c8c78bc0428a4190b5b6d72d45673c13918ab9844d706c3ed4ef8f62ab11a2627a08ad + languageName: node + linkType: hard + "@jupyter/ydoc@npm:^1.0.2": version: 1.0.2 resolution: "@jupyter/ydoc@npm:1.0.2" @@ -3839,6 +3857,7 @@ __metadata: eslint-config-prettier: ^6.15.0 eslint-plugin-prettier: ^3.1.4 jest: ^29 + json-schema-to-typescript: ^15.0.3 mkdirp: ^1.0.3 npm-run-all: ^4.1.5 prettier: ^2.1.1 @@ -4751,6 +4770,13 @@ __metadata: languageName: node linkType: hard +"@types/json-schema@npm:^7.0.15": + version: 7.0.15 + resolution: "@types/json-schema@npm:7.0.15" + checksum: 97ed0cb44d4070aecea772b7b2e2ed971e10c81ec87dd4ecc160322ffa55ff330dace1793489540e3e318d90942064bb697cc0f8989391797792d919737b3b98 + languageName: node + linkType: hard + "@types/json-schema@npm:^7.0.9": version: 7.0.12 resolution: "@types/json-schema@npm:7.0.12" @@ -4758,6 +4784,13 @@ __metadata: languageName: node linkType: hard +"@types/lodash@npm:^4.17.7": + version: 4.17.13 + resolution: "@types/lodash@npm:4.17.13" + checksum: d0bf8fbd950be71946e0076b30fd40d492293baea75f05931b6b5b906fd62583708c6229abdb95b30205ad24ce1ed2f48bc9d419364f682320edd03405cc0c7e + languageName: node + linkType: hard + "@types/minimist@npm:^1.2.0": version: 1.2.2 resolution: "@types/minimist@npm:1.2.2" @@ -5465,6 +5498,13 @@ __metadata: languageName: node linkType: hard +"argparse@npm:^2.0.1": + version: 2.0.1 + resolution: "argparse@npm:2.0.1" + checksum: 83644b56493e89a254bae05702abf3a1101b4fa4d0ca31df1c9985275a5a5bd47b3c27b7fa0b71098d41114d8ca000e6ed90cad764b306f8a503665e4d517ced + languageName: node + linkType: hard + "array-union@npm:^2.1.0": version: 2.1.0 resolution: "array-union@npm:2.1.0" @@ -6967,6 +7007,18 @@ __metadata: languageName: node linkType: hard +"fdir@npm:^6.4.2": + version: 6.4.2 + resolution: "fdir@npm:6.4.2" + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + checksum: 517ad31c495f1c0778238eef574a7818788efaaf2ce1969ffa18c70793e2951a9763dfa2e6720b8fcef615e602a3cbb47f9b8aea9da0b02147579ab36043f22f + languageName: node + linkType: hard + "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -7722,7 +7774,7 @@ __metadata: languageName: node linkType: hard -"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1": +"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3": version: 4.0.3 resolution: "is-glob@npm:4.0.3" dependencies: @@ -8446,6 +8498,17 @@ __metadata: languageName: node linkType: hard +"js-yaml@npm:^4.1.0": + version: 4.1.0 + resolution: "js-yaml@npm:4.1.0" + dependencies: + argparse: ^2.0.1 + bin: + js-yaml: bin/js-yaml.js + checksum: c7830dfd456c3ef2c6e355cc5a92e6700ceafa1d14bba54497b34a99f0376cecbb3e9ac14d3e5849b426d5a5140709a66237a8c991c675431271c4ce5504151a + languageName: node + linkType: hard + "jsdom@npm:^20.0.0": version: 20.0.3 resolution: "jsdom@npm:20.0.3" @@ -8537,6 +8600,25 @@ __metadata: languageName: node linkType: hard +"json-schema-to-typescript@npm:^15.0.3": + version: 15.0.3 + resolution: "json-schema-to-typescript@npm:15.0.3" + dependencies: + "@apidevtools/json-schema-ref-parser": ^11.5.5 + "@types/json-schema": ^7.0.15 + "@types/lodash": ^4.17.7 + is-glob: ^4.0.3 + js-yaml: ^4.1.0 + lodash: ^4.17.21 + minimist: ^1.2.8 + prettier: ^3.2.5 + tinyglobby: ^0.2.9 + bin: + json2ts: dist/src/cli.js + checksum: e2337655d4a6f9ec0baccecda1e6d29bfdc5c04afed03d27669e87652334c1db5b8f7b3ad84623ad987ebe3e83e931dc181380bcd8b03d21ee3812a1f8a56239 + languageName: node + linkType: hard + "json-schema-traverse@npm:^0.4.1": version: 0.4.1 resolution: "json-schema-traverse@npm:0.4.1" @@ -9022,6 +9104,13 @@ __metadata: languageName: node linkType: hard +"minimist@npm:^1.2.8": + version: 1.2.8 + resolution: "minimist@npm:1.2.8" + checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 + languageName: node + linkType: hard + "minimist@npm:~1.2.0": version: 1.2.6 resolution: "minimist@npm:1.2.6" @@ -9571,6 +9660,13 @@ __metadata: languageName: node linkType: hard +"picomatch@npm:^4.0.2": + version: 4.0.2 + resolution: "picomatch@npm:4.0.2" + checksum: a7a5188c954f82c6585720e9143297ccd0e35ad8072231608086ca950bee672d51b0ef676254af0788205e59bd4e4deb4e7708769226bed725bf13370a7d1464 + languageName: node + linkType: hard + "pidtree@npm:^0.3.0": version: 0.3.1 resolution: "pidtree@npm:0.3.1" @@ -9730,6 +9826,15 @@ __metadata: languageName: node linkType: hard +"prettier@npm:^3.2.5": + version: 3.4.2 + resolution: "prettier@npm:3.4.2" + bin: + prettier: bin/prettier.cjs + checksum: 061c84513db62d3944c8dc8df36584dad82883ce4e49efcdbedd8703dce5b173c33fd9d2a4e1725d642a3b713c932b55418342eaa347479bc4a9cca114a04cd0 + languageName: node + linkType: hard + "pretty-format@npm:^29.0.0, pretty-format@npm:^29.6.1": version: 29.6.1 resolution: "pretty-format@npm:29.6.1" @@ -11004,6 +11109,16 @@ __metadata: languageName: node linkType: hard +"tinyglobby@npm:^0.2.9": + version: 0.2.10 + resolution: "tinyglobby@npm:0.2.10" + dependencies: + fdir: ^6.4.2 + picomatch: ^4.0.2 + checksum: 7e2ffe262ebc149036bdef37c56b32d02d52cf09efa7d43dbdab2ea3c12844a4da881058835ce4c74d1891190e5ad5ec5133560a11ec8314849b68ad0d99d3f4 + languageName: node + linkType: hard + "tmpl@npm:1.0.5": version: 1.0.5 resolution: "tmpl@npm:1.0.5" From 025c390fb791e063cc5d00012eb98c4e693e766c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 19:16:08 +0000 Subject: [PATCH 07/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- package.json | 2 +- src/schema/workflow.schema.json | 2 +- src/workflows/schema/workflow.schema.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 558d05ed..f92e21e1 100644 --- a/package.json +++ b/package.json @@ -143,4 +143,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/schema/workflow.schema.json b/src/schema/workflow.schema.json index e4701622..0af54a62 100644 --- a/src/schema/workflow.schema.json +++ b/src/schema/workflow.schema.json @@ -108,4 +108,4 @@ ] } } -} \ No newline at end of file +} diff --git a/src/workflows/schema/workflow.schema.json b/src/workflows/schema/workflow.schema.json index e4701622..0af54a62 100644 --- a/src/workflows/schema/workflow.schema.json +++ b/src/workflows/schema/workflow.schema.json @@ -108,4 +108,4 @@ ] } } -} \ No newline at end of file +} From d88f4dc4276c37b80189660965129ca721a4d31a Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Mon, 9 Dec 2024 14:07:52 -0800 Subject: [PATCH 08/18] Add WorkflowDoc, remove dupplicate workflow schema --- src/schema/workflow.schema.json | 111 -------------------------------- src/workflows/workflowDoc.ts | 44 +++++++++++++ 2 files changed, 44 insertions(+), 111 deletions(-) delete mode 100644 src/schema/workflow.schema.json create mode 100644 src/workflows/workflowDoc.ts diff --git a/src/schema/workflow.schema.json b/src/schema/workflow.schema.json deleted file mode 100644 index 0af54a62..00000000 --- a/src/schema/workflow.schema.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Workflow", - "type": "object", - "properties": { - "schemaVersion": { - "type": "string", - "default": "0.0.1" - }, - "tasks": { - "type": "array", - "items": { - "$ref": "#/definitions/Task" - }, - "description": "Worfklow tasks." - }, - "name": { - "type": "string", - "description": "The name of the workflow." - }, - "parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Optional parameters for the workflow." - }, - "schedule": { - "type": "string", - "description": "Optional schedule in cron format." - }, - "timezone": { - "type": "string", - "description": "Timezone for the schedule." - } - }, - "required": [ - "tasks", - "name" - ], - "definitions": { - "Task": { - "type": "object", - "properties": { - "input_uri": { - "type": "string", - "description": "The URI of the input file." - }, - "runtime_environment_name": { - "type": "string", - "description": "Name of the runtime environment." - }, - "runtime_environment_parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Parameters for the runtime environment." - }, - "output_formats": { - "type": "array", - "items": { - "type": "string" - } - }, - "parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Task-specific parameters." - }, - "tags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Tags for categorizing the job." - }, - "name": { - "type": "string", - "description": "Name of the job." - }, - "compute_type": { - "type": "string", - "description": "Type of compute resource to use." - }, - "package_input_folder": { - "type": "boolean", - "description": "Whether to package the input folder." - }, - "depends_on": { - "type": "array", - "items": { - "type": "string" - }, - "description": "DAG node IDs of tasks this task depends on." - }, - "node_id": { - "type": "string", - "description": "DAG node ID of this task." - } - }, - "required": [ - "input_uri", - "name", - "node_id" - ] - } - } -} diff --git a/src/workflows/workflowDoc.ts b/src/workflows/workflowDoc.ts new file mode 100644 index 00000000..bc46526b --- /dev/null +++ b/src/workflows/workflowDoc.ts @@ -0,0 +1,44 @@ +import { YDocument } from '@jupyter/ydoc'; +import * as Y from 'yjs'; +import { Workflow, Task } from './_interface/workflow.schema'; + +export class WorkflowDoc extends YDocument implements Workflow { + private _tasks: Y.Array>; + private _name: Y.Text; + private _parameters: Y.Map; + private _schedule: Y.Text; + private _timezone: Y.Text; + + constructor() { + super(); + + this._tasks = this.ydoc.getArray>('tasks'); + this._name = this.ydoc.getText('name'); + this._parameters = this.ydoc.getMap('parameters'); + this._schedule = this.ydoc.getText('schedule'); + this._timezone = this.ydoc.getText('timezone'); + + this._tasks.observeDeep(this._tasksObserver); + //TODO: add other observers + } + + // Getter and setter methods + get tasks(): Task[] { + return this._tasks.map(task => task.toJSON() as Task); + } + + set tasks(value: Task[]) { + this.transact(() => { + this._tasks.delete(0, this._tasks.length); + value.forEach(task => { + this._tasks.push([Y.Map.from(Object.entries(task))]); + }); + }); + } + + //TODO: add other getters/setters + + private _tasksObserver = (events: Y.YEvent[]) => { + // TODO: Handle task changes + }; +} From 4e5c6f2b008a597b75ca0aa87d823e4c51504384 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 13:59:14 -0800 Subject: [PATCH 09/18] add simple YDoc-based workflow doc, interfaces, and json schema for it --- src/workflows/interfaces.ts | 21 +++++ src/workflows/schema/workflow.schema.json | 102 +--------------------- src/workflows/workflowDoc.ts | 54 +++++------- 3 files changed, 46 insertions(+), 131 deletions(-) create mode 100644 src/workflows/interfaces.ts diff --git a/src/workflows/interfaces.ts b/src/workflows/interfaces.ts new file mode 100644 index 00000000..6ebd010b --- /dev/null +++ b/src/workflows/interfaces.ts @@ -0,0 +1,21 @@ +import { DocumentChange, StateChange, YDocument } from '@jupyter/ydoc'; +import { ISignal } from '@lumino/signaling'; + +export interface IWorkflowDoc extends YDocument { + name: string; + + getName(): string | undefined; + setName(name: string): void; + + nameChanged: ISignal; +} + +export interface IWorkflowDocChange extends DocumentChange { + nameChange?: StringChange; + stateChange?: StateChange[]; +} + +export type StringChange = { + oldValue?: string; + newValue?: string; +}; diff --git a/src/workflows/schema/workflow.schema.json b/src/workflows/schema/workflow.schema.json index 0af54a62..4c203dfc 100644 --- a/src/workflows/schema/workflow.schema.json +++ b/src/workflows/schema/workflow.schema.json @@ -3,109 +3,9 @@ "title": "Workflow", "type": "object", "properties": { - "schemaVersion": { - "type": "string", - "default": "0.0.1" - }, - "tasks": { - "type": "array", - "items": { - "$ref": "#/definitions/Task" - }, - "description": "Worfklow tasks." - }, "name": { "type": "string", "description": "The name of the workflow." - }, - "parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Optional parameters for the workflow." - }, - "schedule": { - "type": "string", - "description": "Optional schedule in cron format." - }, - "timezone": { - "type": "string", - "description": "Timezone for the schedule." - } - }, - "required": [ - "tasks", - "name" - ], - "definitions": { - "Task": { - "type": "object", - "properties": { - "input_uri": { - "type": "string", - "description": "The URI of the input file." - }, - "runtime_environment_name": { - "type": "string", - "description": "Name of the runtime environment." - }, - "runtime_environment_parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Parameters for the runtime environment." - }, - "output_formats": { - "type": "array", - "items": { - "type": "string" - } - }, - "parameters": { - "type": "object", - "additionalProperties": { - "type": "string" - }, - "description": "Task-specific parameters." - }, - "tags": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Tags for categorizing the job." - }, - "name": { - "type": "string", - "description": "Name of the job." - }, - "compute_type": { - "type": "string", - "description": "Type of compute resource to use." - }, - "package_input_folder": { - "type": "boolean", - "description": "Whether to package the input folder." - }, - "depends_on": { - "type": "array", - "items": { - "type": "string" - }, - "description": "DAG node IDs of tasks this task depends on." - }, - "node_id": { - "type": "string", - "description": "DAG node ID of this task." - } - }, - "required": [ - "input_uri", - "name", - "node_id" - ] } } -} +} \ No newline at end of file diff --git a/src/workflows/workflowDoc.ts b/src/workflows/workflowDoc.ts index bc46526b..95e94054 100644 --- a/src/workflows/workflowDoc.ts +++ b/src/workflows/workflowDoc.ts @@ -1,44 +1,38 @@ import { YDocument } from '@jupyter/ydoc'; -import * as Y from 'yjs'; -import { Workflow, Task } from './_interface/workflow.schema'; +import { Text as YText, Doc as YDoc } from 'yjs'; +import { IWorkflowDoc, IWorkflowDocChange, StringChange } from './interfaces'; +import { ISignal, Signal } from '@lumino/signaling'; -export class WorkflowDoc extends YDocument implements Workflow { - private _tasks: Y.Array>; - private _name: Y.Text; - private _parameters: Y.Map; - private _schedule: Y.Text; - private _timezone: Y.Text; +export class WorkflowDoc + extends YDocument + implements IWorkflowDoc +{ + private _name: YText; + private _nameChanged = new Signal(this); constructor() { super(); - this._tasks = this.ydoc.getArray>('tasks'); this._name = this.ydoc.getText('name'); - this._parameters = this.ydoc.getMap('parameters'); - this._schedule = this.ydoc.getText('schedule'); - this._timezone = this.ydoc.getText('timezone'); - - this._tasks.observeDeep(this._tasksObserver); - //TODO: add other observers } - // Getter and setter methods - get tasks(): Task[] { - return this._tasks.map(task => task.toJSON() as Task); + get nameChanged(): ISignal { + return this._nameChanged; } - - set tasks(value: Task[]) { - this.transact(() => { - this._tasks.delete(0, this._tasks.length); - value.forEach(task => { - this._tasks.push([Y.Map.from(Object.entries(task))]); - }); - }); + get name(): string { + return this._name.toString(); + } + get version(): string { + return '0.0.1'; } - //TODO: add other getters/setters + getName(): string | undefined { + return this.name; + } - private _tasksObserver = (events: Y.YEvent[]) => { - // TODO: Handle task changes - }; + setName(name: string): void { + const newName = new YText(); + newName.insert(0, name); + this._name = newName; + } } From d754e11c670e03cd2330eae048b6ca87d029516b Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 14:03:51 -0800 Subject: [PATCH 10/18] use wildcard import for Y as reccomended in the docs --- src/workflows/workflowDoc.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/workflows/workflowDoc.ts b/src/workflows/workflowDoc.ts index 95e94054..c4f65c48 100644 --- a/src/workflows/workflowDoc.ts +++ b/src/workflows/workflowDoc.ts @@ -1,5 +1,5 @@ import { YDocument } from '@jupyter/ydoc'; -import { Text as YText, Doc as YDoc } from 'yjs'; +import * as Y from 'yjs'; import { IWorkflowDoc, IWorkflowDocChange, StringChange } from './interfaces'; import { ISignal, Signal } from '@lumino/signaling'; @@ -7,15 +7,14 @@ export class WorkflowDoc extends YDocument implements IWorkflowDoc { - private _name: YText; - private _nameChanged = new Signal(this); - constructor() { super(); this._name = this.ydoc.getText('name'); } + private _name: Y.Text; + private _nameChanged = new Signal(this); get nameChanged(): ISignal { return this._nameChanged; } @@ -31,7 +30,7 @@ export class WorkflowDoc } setName(name: string): void { - const newName = new YText(); + const newName = new Y.Text(); newName.insert(0, name); this._name = newName; } From 4c5d6e02c29843e40c4ec989525e44eb844012e1 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 14:15:00 -0800 Subject: [PATCH 11/18] hardcode new name value for testing --- src/workflows/workflowDoc.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/workflows/workflowDoc.ts b/src/workflows/workflowDoc.ts index c4f65c48..b690f77a 100644 --- a/src/workflows/workflowDoc.ts +++ b/src/workflows/workflowDoc.ts @@ -11,8 +11,13 @@ export class WorkflowDoc super(); this._name = this.ydoc.getText('name'); + this._name.observe(this._nameObserver); } + private _nameObserver = (event: Y.YTextEvent): void => { + this._nameChanged.emit({ newValue: 'WOW NEW VALUE' }); + }; + private _name: Y.Text; private _nameChanged = new Signal(this); get nameChanged(): ISignal { From b93e1ce54e6cb965fc1c20cb05833b0c7e34cfd1 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 14:37:00 -0800 Subject: [PATCH 12/18] add workflow model --- src/workflows/workflowModel.ts | 129 +++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 src/workflows/workflowModel.ts diff --git a/src/workflows/workflowModel.ts b/src/workflows/workflowModel.ts new file mode 100644 index 00000000..7a85527b --- /dev/null +++ b/src/workflows/workflowModel.ts @@ -0,0 +1,129 @@ +import { DocumentRegistry } from '@jupyterlab/docregistry'; +import { PartialJSONObject } from '@lumino/coreutils'; +import { ISignal, Signal } from '@lumino/signaling'; +import { IWorkflowDoc } from './interfaces'; +import { WorkflowDoc } from './workflowDoc'; + +export interface IWorkflowModel extends DocumentRegistry.IModel { + sharedModel: IWorkflowDoc; +} + +export class WorkflowModel implements IWorkflowModel { + constructor(options: DocumentRegistry.IModelOptions) { + this._sharedModel = options.sharedModel ?? this.createSharedModel(); + this._isDisposed = false; + this._dirty = false; + this._readOnly = false; + + // Listen to changes on the shared model + this._sharedModel.nameChanged.connect(this._onNameChanged, this); + } + + /** + * Create a default shared model if one is not provided. + */ + protected createSharedModel(): IWorkflowDoc { + return new WorkflowDoc(); + } + + get sharedModel(): IWorkflowDoc { + return this._sharedModel; + } + + get isDisposed(): boolean { + return this._isDisposed; + } + + get dirty(): boolean { + return this._dirty; + } + + set dirty(value: boolean) { + this._dirty = value; + } + + get readOnly(): boolean { + return this._readOnly; + } + + set readOnly(value: boolean) { + this._readOnly = value; + } + + get contentChanged(): ISignal { + return this._contentChanged; + } + + get stateChanged(): ISignal { + return this._stateChanged; + } + + /** + * Convert the model to string (JSON in this case). + * We only have a `name` field, so just return a JSON string with that. + */ + toString(): string { + const data = { name: this.sharedModel.getName() }; + return JSON.stringify(data, null, 2); + } + + /** + * Load from a string. Assume it’s JSON with a `name` field. + */ + fromString(data: string): void { + const jsonData = JSON.parse(data); + if (jsonData.name && typeof jsonData.name === 'string') { + this.sharedModel.transact(() => { + this.sharedModel.setName(jsonData.name); + }); + this.dirty = true; + this._contentChanged.emit(void 0); + } + } + + toJSON(): PartialJSONObject { + return JSON.parse(this.toString()); + } + + fromJSON(data: PartialJSONObject): void { + if (data.name && typeof data.name === 'string') { + this.sharedModel.transact(() => { + this.sharedModel.setName(data.name as string); + }); + this.dirty = true; + this._contentChanged.emit(void 0); + } + } + + initialize(): void { + // No initialization needed for this simple example + } + + dispose(): void { + if (this.isDisposed) { + return; + } + this._isDisposed = true; + + // Disconnect signals + this._sharedModel.nameChanged.disconnect(this._onNameChanged, this); + + Signal.clearData(this); + } + + private _onNameChanged(): void { + this.dirty = true; + this._contentChanged.emit(void 0); + } + + private _sharedModel: IWorkflowDoc; + private _dirty: boolean; + private _readOnly: boolean; + private _isDisposed: boolean; + + private _contentChanged = new Signal(this); + private _stateChanged = new Signal(this); + + readonly defaultKernelName = ''; + readonly defaultKernelLanguage = ''; +} From 9c165b7c79f0afbe15655961b708b8e585728e74 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 22:37:38 +0000 Subject: [PATCH 13/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/workflows/schema/workflow.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workflows/schema/workflow.schema.json b/src/workflows/schema/workflow.schema.json index 4c203dfc..c9b8aac5 100644 --- a/src/workflows/schema/workflow.schema.json +++ b/src/workflows/schema/workflow.schema.json @@ -8,4 +8,4 @@ "description": "The name of the workflow." } } -} \ No newline at end of file +} From 19a6bdee2dfc53f5c9a94816418d4a899f4806e3 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 16:55:36 -0800 Subject: [PATCH 14/18] add WorkflowModelFactory --- src/workflows/workflowModel.ts | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/workflows/workflowModel.ts b/src/workflows/workflowModel.ts index 7a85527b..ae2875ce 100644 --- a/src/workflows/workflowModel.ts +++ b/src/workflows/workflowModel.ts @@ -3,6 +3,7 @@ import { PartialJSONObject } from '@lumino/coreutils'; import { ISignal, Signal } from '@lumino/signaling'; import { IWorkflowDoc } from './interfaces'; import { WorkflowDoc } from './workflowDoc'; +import { Contents } from '@jupyterlab/services'; export interface IWorkflowModel extends DocumentRegistry.IModel { sharedModel: IWorkflowDoc; @@ -127,3 +128,43 @@ export class WorkflowModel implements IWorkflowModel { readonly defaultKernelName = ''; readonly defaultKernelLanguage = ''; } + +export class WorkflowModelFactory + implements DocumentRegistry.IModelFactory +{ + //TODO: set conditionally + readonly collaborative = true; + + get name(): string { + return 'workflow-model-factory'; + } + + get contentType(): Contents.ContentType { + return 'file'; + } + + get fileFormat(): Contents.FileFormat { + return 'text'; + } + + get isDisposed(): boolean { + return this._disposed; + } + + dispose(): void { + this._disposed = true; + } + + preferredLanguage(path: string): string { + return ''; + } + + createNew( + options: DocumentRegistry.IModelOptions + ): WorkflowModel { + const model = new WorkflowModel(options); + return model; + } + + private _disposed = false; +} From 63105357875aff3d68d79acadfb4a7e8a00b7dbd Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Wed, 11 Dec 2024 16:56:55 -0800 Subject: [PATCH 15/18] add WorkflowWidget and WorkflowWidgetFactory --- src/workflows/workflowWidgetFactory.ts | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/workflows/workflowWidgetFactory.ts diff --git a/src/workflows/workflowWidgetFactory.ts b/src/workflows/workflowWidgetFactory.ts new file mode 100644 index 00000000..2c34b30a --- /dev/null +++ b/src/workflows/workflowWidgetFactory.ts @@ -0,0 +1,62 @@ +import { ABCWidgetFactory, DocumentRegistry } from '@jupyterlab/docregistry'; +import { Widget } from '@lumino/widgets'; +import { IWorkflowModel } from './workflowModel'; + +export class WorkflowWidget extends Widget { + constructor(context: DocumentRegistry.IContext) { + super(); + this.addClass('jp-WorkflowWidget'); + this._context = context; + + this._input = document.createElement('input'); + this._input.type = 'text'; + this._input.value = this._context.model.sharedModel.getName() ?? ''; + + this._input.addEventListener('input', () => { + // Update the shared model when the user types + this._context.model.sharedModel.transact(() => { + this._context.model.sharedModel.setName(this._input.value); + }); + }); + + this.node.appendChild(this._input); + + // Listen for remote changes + this._context.model.sharedModel.nameChanged.connect( + this._onNameChanged, + this + ); + + // Listen to contentChanged + this._context.model.contentChanged.connect(() => { + console.log('Content changed, doc may have changed externally.'); + }); + } + + dispose(): void { + super.dispose(); + this._context.model.sharedModel.nameChanged.disconnect( + this._onNameChanged, + this + ); + } + + private _onNameChanged(): void { + // Update the input value to reflect remote changes + this._input.value = this._context.model.sharedModel.getName() ?? ''; + } + + private _context: DocumentRegistry.IContext; + private _input: HTMLInputElement; +} + +export class WorkflowWidgetFactory extends ABCWidgetFactory< + Widget, + IWorkflowModel +> { + protected createNewWidget( + context: DocumentRegistry.IContext + ): Widget { + return new WorkflowWidget(context); + } +} From 66a730583ae26781e5d3d417021dc1cde69bb6af Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Thu, 19 Dec 2024 13:14:06 -0800 Subject: [PATCH 16/18] update WorkflowDocumentWidget, createNewWidget --- src/workflows/workflowWidgetFactory.ts | 28 ++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/workflows/workflowWidgetFactory.ts b/src/workflows/workflowWidgetFactory.ts index 2c34b30a..3456109f 100644 --- a/src/workflows/workflowWidgetFactory.ts +++ b/src/workflows/workflowWidgetFactory.ts @@ -1,4 +1,8 @@ -import { ABCWidgetFactory, DocumentRegistry } from '@jupyterlab/docregistry'; +import { + ABCWidgetFactory, + DocumentRegistry, + DocumentWidget +} from '@jupyterlab/docregistry'; import { Widget } from '@lumino/widgets'; import { IWorkflowModel } from './workflowModel'; @@ -50,13 +54,29 @@ export class WorkflowWidget extends Widget { private _input: HTMLInputElement; } -export class WorkflowWidgetFactory extends ABCWidgetFactory< +export class WorkflowDocumentWidget extends DocumentWidget< Widget, IWorkflowModel +> { + constructor(options: DocumentWidget.IOptions) { + super(options); + } +} + +export class WorkflowWidgetFactory extends ABCWidgetFactory< + WorkflowDocumentWidget, + IWorkflowModel > { protected createNewWidget( context: DocumentRegistry.IContext - ): Widget { - return new WorkflowWidget(context); + ): WorkflowDocumentWidget { + const content = new WorkflowWidget(context); + + const widget = new WorkflowDocumentWidget({ + context, + content + }); + + return widget; } } From cc9c0b0a0a93f63e5755aa26aa207d60eb08da5a Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Thu, 19 Dec 2024 13:14:36 -0800 Subject: [PATCH 17/18] update Y.Text name in a transactional way --- src/workflows/workflowDoc.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/workflows/workflowDoc.ts b/src/workflows/workflowDoc.ts index b690f77a..14a397b4 100644 --- a/src/workflows/workflowDoc.ts +++ b/src/workflows/workflowDoc.ts @@ -11,21 +11,27 @@ export class WorkflowDoc super(); this._name = this.ydoc.getText('name'); + this._previousName = this._name.toString(); this._name.observe(this._nameObserver); } private _nameObserver = (event: Y.YTextEvent): void => { - this._nameChanged.emit({ newValue: 'WOW NEW VALUE' }); + const oldValue = this._previousName; + const newValue = this._name.toString(); + + this._previousName = newValue; + + this._nameChanged.emit({ oldValue, newValue }); }; - private _name: Y.Text; - private _nameChanged = new Signal(this); get nameChanged(): ISignal { return this._nameChanged; } + get name(): string { return this._name.toString(); } + get version(): string { return '0.0.1'; } @@ -35,8 +41,14 @@ export class WorkflowDoc } setName(name: string): void { - const newName = new Y.Text(); - newName.insert(0, name); - this._name = newName; + const currentLength = this._name.length; + if (currentLength > 0) { + this._name.delete(0, currentLength); + } + this._name.insert(0, name); } + + private _name: Y.Text; + private _previousName: string; + private _nameChanged = new Signal(this); } From 3fd008cb41947a6dfb87a67b2b15226c3cf36a23 Mon Sep 17 00:00:00 2001 From: Andrii Ieroshenko Date: Thu, 19 Dec 2024 13:47:24 -0800 Subject: [PATCH 18/18] add test widget to activatePlugin --- src/index.tsx | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/index.tsx b/src/index.tsx index fe7a71ec..9281ef5f 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -27,6 +27,8 @@ import { NotebookJobsPanel } from './notebook-jobs-panel'; import { Scheduler } from './tokens'; import { SERVER_EXTENSION_404_JSX } from './util/errors'; import { MakeNameValid } from './util/job-name-validation'; +import { WorkflowModelFactory } from './workflows/workflowModel'; +import { WorkflowWidgetFactory } from './workflows/workflowWidgetFactory'; export namespace CommandIDs { export const deleteJob = 'scheduling:delete-job'; @@ -194,10 +196,58 @@ function activatePlugin( telemetryHandler: Scheduler.TelemetryHandler, launcher: ILauncher | null ): void { + // Hardcoded boolean for testing. If true, set up workflow widget instead of scheduler UI + const showWorkflowsWidget = true; + const trans = translator.load('jupyterlab'); const api = new SchedulerService({}); verifyServerExtension({ api, translator }); + if (showWorkflowsWidget) { + const WORKFLOW_FACTORY = 'Workflow Editor'; + const WORKFLOW_CONTENT_TYPE = 'workflow'; + const WORKFLOW_FILE_EXT = '.jwf'; + + // Register the workflow file type + app.docRegistry.addFileType({ + name: WORKFLOW_CONTENT_TYPE, + displayName: 'Workflow File', + extensions: [WORKFLOW_FILE_EXT], + fileFormat: 'text', + contentType: 'file', + mimeTypes: ['application/json'] + }); + + // Register the workflow model factory + const modelFactory = new WorkflowModelFactory(); + app.docRegistry.addModelFactory(modelFactory); + + // Register the workflow widget factory + const widgetFactory = new WorkflowWidgetFactory({ + name: WORKFLOW_FACTORY, + modelName: modelFactory.name, + fileTypes: [WORKFLOW_CONTENT_TYPE], + defaultFor: [WORKFLOW_CONTENT_TYPE] + }); + app.docRegistry.addWidgetFactory(widgetFactory); + + // Create a new untitled .jwf file and open it + void app.commands + .execute('docmanager:new-untitled', { + type: 'file', + ext: '.jwf' + }) + .then(model => { + if (model) { + void app.commands.execute('docmanager:open', { + path: model.path + }); + } + }); + + return; + } + const { commands } = app; const fileBrowserTracker = browserFactory.tracker; const widgetTracker = new WidgetTracker>({