From cb6575732335b884903ba1fc54c687f1c70a9e4e Mon Sep 17 00:00:00 2001 From: Derrick Brittain Date: Thu, 13 Jun 2024 12:53:18 -0400 Subject: [PATCH 1/3] feat: make migration view page --- materializationengine/admin.py | 56 +++++++++++++++++- templates/admin/migration.html | 102 +++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 templates/admin/migration.html diff --git a/materializationengine/admin.py b/materializationengine/admin.py index b78bccd3..5b87bc85 100644 --- a/materializationengine/admin.py +++ b/materializationengine/admin.py @@ -1,10 +1,64 @@ -from flask_admin import Admin +from flask_admin import Admin, BaseView, expose from flask_admin.contrib.sqla import ModelView from dynamicannotationdb.models import AnalysisVersion, AnalysisTable +from flask import request +from flask import redirect, url_for, request, current_app +from dynamicannotationdb.migration import DynamicMigration, run_alembic_migration +from sqlalchemy.engine.url import make_url +from materializationengine.info_client import ( + get_datastacks, + get_relevant_datastack_info, +) def setup_admin(app, db): admin = Admin(app, name="materializationengine") + admin.add_view(MigrationView(name="Migration")) admin.add_view(ModelView(AnalysisVersion, db.session)) admin.add_view(ModelView(AnalysisTable, db.session)) return admin + + + +def get_allowed_aligned_volumes(): + with current_app.app_context(): + datastacks = get_datastacks() + aligned_volumes = [] + for datastack in datastacks: + aligned_volume_name, pcg_table_name = get_relevant_datastack_info(datastack) + aligned_volumes.append(aligned_volume_name) + return aligned_volumes + +class MigrationView(BaseView): + @expose('/') + def index(self): + aligned_volumes = get_allowed_aligned_volumes() + return self.render('admin/migration.html', aligned_volumes=aligned_volumes) + + @expose('/migrate_static_schemas', methods=['POST']) + def migrate_static_schemas(self): + sql_url = request.form.get('sql_url') + aligned_volume = request.form.get('aligned_volume') + sql_base_uri = sql_url.rpartition("/")[0] + sql_uri = make_url(f"{sql_base_uri}/{aligned_volume}") + migrator = run_alembic_migration(str(sql_uri)) + return self.render('admin/migration.html', message=migrator, aligned_volumes=get_allowed_aligned_volumes()) + + @expose('/migrate_annotation_schemas', methods=['POST']) + def migrate_annotation_schemas(self): + sql_url = request.form.get('sql_url') + aligned_volume = request.form.get('aligned_volume') + dry_run = request.form.get('dry_run') == 'true' + migrator = DynamicMigration(sql_url, aligned_volume) + migrations = migrator.upgrade_annotation_models(dry_run=dry_run) + return self.render('admin/migration.html', message=migrations, aligned_volumes=get_allowed_aligned_volumes()) + + @expose('/migrate_foreign_key_constraints', methods=['POST']) + def migrate_foreign_key_constraints(self): + sql_url = request.form.get('sql_url') + aligned_volume = request.form.get('aligned_volume') + dry_run = request.form.get('dry_run') == 'true' + migrator = DynamicMigration(sql_url, aligned_volume) + fkey_constraint_mapping = migrator.apply_cascade_option_to_tables(dry_run=dry_run) + return self.render('admin/migration.html', message=fkey_constraint_mapping, aligned_volumes=get_allowed_aligned_volumes()) + diff --git a/templates/admin/migration.html b/templates/admin/migration.html new file mode 100644 index 00000000..6bedac6a --- /dev/null +++ b/templates/admin/migration.html @@ -0,0 +1,102 @@ + + + + + + Migrations + + + +
+

Database Migrations

+ +
+
+ Migrate Static Schemas +
+
+
+
+ + +
+
+ + +
+ +
+
+
+ +
+
+ Migrate Annotation Schemas +
+
+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+ +
+
+ Alter Foreign Key Constraints +
+
+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+ + {% if message %} +
+

Migration Result:

+ 
{{ message }}
+
+ {% endif %} +
+ + + + + + From e4001e93c61ad261f9fa2f802acdad2ba94bad40 Mon Sep 17 00:00:00 2001 From: Derrick Brittain Date: Fri, 14 Jun 2024 10:30:41 -0400 Subject: [PATCH 2/3] fix: add admin protection to migration code --- materializationengine/admin.py | 87 +++++++++++++++++++++++++--------- templates/admin/403.html | 20 ++++++++ 2 files changed, 84 insertions(+), 23 deletions(-) create mode 100644 templates/admin/403.html diff --git a/materializationengine/admin.py b/materializationengine/admin.py index 5b87bc85..200cc990 100644 --- a/materializationengine/admin.py +++ b/materializationengine/admin.py @@ -1,8 +1,8 @@ -from flask_admin import Admin, BaseView, expose +from flask_admin import Admin, AdminIndexView, BaseView, expose from flask_admin.contrib.sqla import ModelView from dynamicannotationdb.models import AnalysisVersion, AnalysisTable -from flask import request -from flask import redirect, url_for, request, current_app +from flask import request, current_app, g, render_template +from middle_auth_client import auth_required from dynamicannotationdb.migration import DynamicMigration, run_alembic_migration from sqlalchemy.engine.url import make_url from materializationengine.info_client import ( @@ -11,15 +11,28 @@ ) +class MatAdminIndexView(AdminIndexView): + @expose("/", methods=["GET"]) + @auth_required + def index(self): + return super(MatAdminIndexView, self).index() + + def is_accessible(self): + @auth_required + def helper(): + return True + + return helper() + + def setup_admin(app, db): - admin = Admin(app, name="materializationengine") + admin = Admin(app, name="materializationengine", index_view=MatAdminIndexView()) admin.add_view(MigrationView(name="Migration")) admin.add_view(ModelView(AnalysisVersion, db.session)) admin.add_view(ModelView(AnalysisTable, db.session)) return admin - def get_allowed_aligned_volumes(): with current_app.app_context(): datastacks = get_datastacks() @@ -29,36 +42,64 @@ def get_allowed_aligned_volumes(): aligned_volumes.append(aligned_volume_name) return aligned_volumes + class MigrationView(BaseView): - @expose('/') + def is_accessible(self): + @auth_required + def helper(): + return True + + return helper() and g.get("auth_user", {}).get("admin", False) + + def inaccessible_callback(self, name, **kwargs): + return render_template("admin/403.html"), 403 + + @expose("/") + @auth_required def index(self): aligned_volumes = get_allowed_aligned_volumes() - return self.render('admin/migration.html', aligned_volumes=aligned_volumes) + return self.render("admin/migration.html", aligned_volumes=aligned_volumes) - @expose('/migrate_static_schemas', methods=['POST']) + @expose("/migrate_static_schemas", methods=["POST"]) + @auth_required def migrate_static_schemas(self): - sql_url = request.form.get('sql_url') - aligned_volume = request.form.get('aligned_volume') + sql_url = request.form.get("sql_url") + aligned_volume = request.form.get("aligned_volume") sql_base_uri = sql_url.rpartition("/")[0] sql_uri = make_url(f"{sql_base_uri}/{aligned_volume}") migrator = run_alembic_migration(str(sql_uri)) - return self.render('admin/migration.html', message=migrator, aligned_volumes=get_allowed_aligned_volumes()) + return self.render( + "admin/migration.html", + message=migrator, + aligned_volumes=get_allowed_aligned_volumes(), + ) - @expose('/migrate_annotation_schemas', methods=['POST']) + @expose("/migrate_annotation_schemas", methods=["POST"]) + @auth_required def migrate_annotation_schemas(self): - sql_url = request.form.get('sql_url') - aligned_volume = request.form.get('aligned_volume') - dry_run = request.form.get('dry_run') == 'true' + sql_url = request.form.get("sql_url") + aligned_volume = request.form.get("aligned_volume") + dry_run = request.form.get("dry_run") == "true" migrator = DynamicMigration(sql_url, aligned_volume) migrations = migrator.upgrade_annotation_models(dry_run=dry_run) - return self.render('admin/migration.html', message=migrations, aligned_volumes=get_allowed_aligned_volumes()) + return self.render( + "admin/migration.html", + message=migrations, + aligned_volumes=get_allowed_aligned_volumes(), + ) - @expose('/migrate_foreign_key_constraints', methods=['POST']) + @expose("/migrate_foreign_key_constraints", methods=["POST"]) + @auth_required def migrate_foreign_key_constraints(self): - sql_url = request.form.get('sql_url') - aligned_volume = request.form.get('aligned_volume') - dry_run = request.form.get('dry_run') == 'true' + sql_url = request.form.get("sql_url") + aligned_volume = request.form.get("aligned_volume") + dry_run = request.form.get("dry_run") == "true" migrator = DynamicMigration(sql_url, aligned_volume) - fkey_constraint_mapping = migrator.apply_cascade_option_to_tables(dry_run=dry_run) - return self.render('admin/migration.html', message=fkey_constraint_mapping, aligned_volumes=get_allowed_aligned_volumes()) - + fkey_constraint_mapping = migrator.apply_cascade_option_to_tables( + dry_run=dry_run + ) + return self.render( + "admin/migration.html", + message=fkey_constraint_mapping, + aligned_volumes=get_allowed_aligned_volumes(), + ) diff --git a/templates/admin/403.html b/templates/admin/403.html new file mode 100644 index 00000000..071522dc --- /dev/null +++ b/templates/admin/403.html @@ -0,0 +1,20 @@ + + + + + + Access Denied + + + +
+ + Go to Home +
+ + From de572f9898e49147261e9b27716d46ee0ba0988d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 15 Jun 2024 20:39:28 +0200 Subject: [PATCH 3/3] adding auth required admin --- materializationengine/admin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/materializationengine/admin.py b/materializationengine/admin.py index 200cc990..b4241b68 100644 --- a/materializationengine/admin.py +++ b/materializationengine/admin.py @@ -2,7 +2,7 @@ from flask_admin.contrib.sqla import ModelView from dynamicannotationdb.models import AnalysisVersion, AnalysisTable from flask import request, current_app, g, render_template -from middle_auth_client import auth_required +from middle_auth_client import auth_required, auth_requires_admin from dynamicannotationdb.migration import DynamicMigration, run_alembic_migration from sqlalchemy.engine.url import make_url from materializationengine.info_client import ( @@ -55,13 +55,13 @@ def inaccessible_callback(self, name, **kwargs): return render_template("admin/403.html"), 403 @expose("/") - @auth_required + @auth_requires_admin def index(self): aligned_volumes = get_allowed_aligned_volumes() return self.render("admin/migration.html", aligned_volumes=aligned_volumes) @expose("/migrate_static_schemas", methods=["POST"]) - @auth_required + @auth_requires_admin def migrate_static_schemas(self): sql_url = request.form.get("sql_url") aligned_volume = request.form.get("aligned_volume") @@ -75,7 +75,7 @@ def migrate_static_schemas(self): ) @expose("/migrate_annotation_schemas", methods=["POST"]) - @auth_required + @auth_requires_admin def migrate_annotation_schemas(self): sql_url = request.form.get("sql_url") aligned_volume = request.form.get("aligned_volume") @@ -89,7 +89,7 @@ def migrate_annotation_schemas(self): ) @expose("/migrate_foreign_key_constraints", methods=["POST"]) - @auth_required + @auth_requires_admin def migrate_foreign_key_constraints(self): sql_url = request.form.get("sql_url") aligned_volume = request.form.get("aligned_volume")