From 7aae9173f347a08b98e0728846a000e37324a18d Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Wed, 10 Jan 2024 13:59:15 -0500 Subject: [PATCH 01/28] DButils: changed to 2.0 metadata binding syntax * doesn't work yet but fixes some errors --- dbprocessing/DButils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index e1a1fcf..62af562 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -235,7 +235,7 @@ def openDB(self, engine, db_var=None, verbose=False, echo=False): (t, v, tb) = sys.exc_info() raise DBError('Error creating engine: ' + str(v)) try: - metadata = sqlalchemy.MetaData(bind=engineIns) + metadata = sqlalchemy.MetaData() # a session is what you use to actually talk to the DB, set one up with the current engine Session = sessionmaker(bind=engineIns) session = Session() @@ -281,7 +281,7 @@ def _createTableObjects(self, verbose=False): if verbose: print(val) if not hasattr(self, val): # then make it myclass = type(str(val), (object,), dict()) - tableobj = Table(table_dict[val], self.metadata, autoload=True) + tableobj = Table(table_dict[val], self.metadata, autoload_with=self.engine) mapper(myclass, tableobj) setattr(self, str(val), myclass) if verbose: print("Class %s created" % (val)) @@ -3862,7 +3862,7 @@ def addUnixTimeTable(self): raise RuntimeError('Unixtime table already seems to exist.') unixtime = sqlalchemy.Table( 'unixtime', self.metadata, *tables.definition('unixtime')) - self.metadata.create_all(tables=[unixtime]) + self.metadata.create_all(self.engine, tables=[unixtime]) # Make object for the new table definition (skips existing tables) self._createTableObjects() unx0 = datetime.datetime(1970, 1, 1) @@ -3895,6 +3895,5 @@ def create_tables(filename='dbprocessing_default.db', dialect='sqlite'): data_table = sqlalchemy.schema.Table( name, metadata, *tables.definition(name)) engine = sqlalchemy.engine.create_engine(url, echo=False) - metadata.bind = engine - metadata.create_all(checkfirst=True) + metadata.create_all(checkfirst=True, bind=engine) engine.dispose() From 38f12f2e22ce793e793d58c53dd75af3c703fb5a Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Wed, 10 Jan 2024 14:41:05 -0500 Subject: [PATCH 02/28] DButils: changed to declarative mapping --- dbprocessing/DButils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index 62af562..d774f27 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -39,7 +39,7 @@ import sqlalchemy.schema import sqlalchemy.sql.expression from sqlalchemy import Table -from sqlalchemy.orm import mapper +from sqlalchemy.orm import registry from sqlalchemy.orm import sessionmaker import sqlalchemy.orm.exc from sqlalchemy.exc import IntegrityError @@ -277,12 +277,13 @@ def _createTableObjects(self, verbose=False): ## pass ## missions = Table('missions', metadata, autoload=True) ## mapper(Missions, missions) + mapper_registry = registry() for val in table_dict: if verbose: print(val) if not hasattr(self, val): # then make it myclass = type(str(val), (object,), dict()) tableobj = Table(table_dict[val], self.metadata, autoload_with=self.engine) - mapper(myclass, tableobj) + mapper_registry.map_imperatively(myclass, tableobj) setattr(self, str(val), myclass) if verbose: print("Class %s created" % (val)) if verbose: DBlogging.dblogger.debug("Class %s created" % (val)) From 60998149a729def78c52e144bced062bfded908d Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Wed, 10 Jan 2024 19:45:40 -0500 Subject: [PATCH 03/28] dbp_testing.py: add bind to table drop --- unit_tests/dbp_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/dbp_testing.py b/unit_tests/dbp_testing.py index 775aa7b..6b63d85 100644 --- a/unit_tests/dbp_testing.py +++ b/unit_tests/dbp_testing.py @@ -282,7 +282,7 @@ def loadData(self, filename): # persist_selectable added 1.3 (mapped_table deprecated) tbl = insp.persist_selectable\ if hasattr(insp, 'persist_selectable') else insp.mapped_table - tbl.drop() + tbl.drop(bind=self.dbu.engine) self.dbu.metadata.remove(tbl) del self.dbu.Unixtime if data['productprocesslink']\ From 9903e220d947e8109c5ec075b272bc513a427631 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Wed, 17 Jan 2024 10:31:43 -0500 Subject: [PATCH 04/28] DButils.getTraceback: get the sqlalchemy 2.0 join syntax right --- dbprocessing/DButils.py | 100 ++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index d774f27..83c0134 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -3245,18 +3245,23 @@ def getTraceback(self, table, in_id, in_id2=None): 'instrumentproductlink', 'satellite', 'mission'] in_id = self.getFileID(in_id) - - sq = (self.session.query(self.File, self.Product, - self.Inspector, self.Instrument, - self.Instrumentproductlink, self.Satellite, - self.Mission) + sq = (self.session.query(self.File, + self.Product, + self.Inspector, + self.Instrument, + self.Instrumentproductlink, + self.Satellite, + self.Mission, + ) .filter_by(file_id=in_id) - .join((self.Product, self.File.product_id == self.Product.product_id)) - .join((self.Inspector, self.Product.product_id == self.Inspector.product)) - .join((self.Instrumentproductlink, self.Product.product_id == self.Instrumentproductlink.product_id)) - .join((self.Instrument, self.Instrumentproductlink.instrument_id == self.Instrument.instrument_id)) - .join((self.Satellite, self.Instrument.satellite_id == self.Satellite.satellite_id)) - .join((self.Mission, self.Satellite.mission_id == self.Mission.mission_id)).all()) + .join(self.Product, self.Product.product_id == self.File.product_id) + .join(self.Inspector, self.Inspector.product == self.Product.product_id) + .join(self.Instrumentproductlink, self.Instrumentproductlink.product_id == self.Product.product_id) + .join(self.Instrument, self.Instrument.instrument_id == self.Instrumentproductlink.instrument_id) + .join(self.Satellite, self.Satellite.satellite_id == self.Instrument.satellite_id) + .join(self.Mission, self.Mission.mission_id == self.Satellite.mission_id) + .all() + ) if not sq: # did not find a matchm this is a dberror raise DBError("file {0} did not have a traceback, this is a problem, fix it".format(in_id)) @@ -3282,18 +3287,23 @@ def getTraceback(self, table, in_id, in_id2=None): if sq[0][1].output_timebase != 'RUN': vars = ['code', 'process', 'product', 'instrument', 'instrumentproductlink', 'satellite', 'mission'] - sq = (self.session.query(self.Code, self.Process, - self.Product, self.Instrument, - self.Instrumentproductlink, self.Satellite, + sq = (self.session.query(self.Code, + self.Process, + self.Product, + self.Instrument, + self.Instrumentproductlink, + self.Satellite, self.Mission) .filter_by(code_id=in_id) - .join((self.Process, self.Code.process_id == self.Process.process_id)) - .join((self.Product, self.Product.product_id == self.Process.output_product)) - .join((self.Inspector, self.Product.product_id == self.Inspector.product)) - .join((self.Instrumentproductlink, self.Product.product_id == self.Instrumentproductlink.product_id)) - .join((self.Instrument, self.Instrumentproductlink.instrument_id == self.Instrument.instrument_id)) - .join((self.Satellite, self.Instrument.satellite_id == self.Satellite.satellite_id)) - .join((self.Mission, self.Satellite.mission_id == self.Mission.mission_id)).all()) + .join(self.Process, self.Process.process_id == self.Code.process_id) + .join(self.Product, self.Product.product_id == self.Process.output_product) + .join(self.Inspector, self.Inspector.product == self.Product.product_id) + .join(self.Instrumentproductlink, self.Instrumentproductlink.product_id == self.Product.product_id) + .join(self.Instrument, self.Instrument.instrument_id == self.Instrumentproductlink.instrument_id) + .join(self.Satellite, self.Satellite.satellite_id == self.Instrument.satellite_id) + .join(self.Mission, self.Mission.mission_id == self.Satellite.mission_id) + .all() + ) if not sq: # did not find a match this is a dberror raise DBError("code {0} did not have a traceback, this is a problem, fix it".format(in_id)) @@ -3313,17 +3323,21 @@ def getTraceback(self, table, in_id, in_id2=None): 'instrumentproductlink', 'satellite', 'mission'] in_id = self.getProductID(in_id) - sq = (self.session.query(self.Product, - self.Inspector, self.Instrument, - self.Instrumentproductlink, self.Satellite, - self.Mission) - .filter_by(product_id=in_id) - .join((self.Inspector, self.Product.product_id == self.Inspector.product)) - .join((self.Instrumentproductlink, self.Product.product_id == self.Instrumentproductlink.product_id)) - .join((self.Instrument, self.Instrumentproductlink.instrument_id == self.Instrument.instrument_id)) - .join((self.Satellite, self.Instrument.satellite_id == self.Satellite.satellite_id)) - .join((self.Mission, self.Satellite.mission_id == self.Mission.mission_id)).all()) + self.Inspector, + self.Instrument, + self.Instrumentproductlink, + self.Satellite, + self.Mission, + ) + .filter_by(product_id=in_id) + .join(self.Inspector, self.Inspector.product == self.Product.product_id) + .join(self.Instrumentproductlink, self.Instrumentproductlink.product_id == self.Product.product_id) + .join(self.Instrument, self.Instrument.instrument_id == self.Instrumentproductlink.instrument_id) + .join(self.Satellite, self.Satellite.satellite_id == self.Instrument.satellite_id ) + .join(self.Mission, self.Mission.mission_id == self.Satellite.mission_id ) + .all() + ) if not sq: # did not find a match this is a dberror raise DBError("product {0} did not have a traceback, this is a problem, fix it".format(in_id)) @@ -3339,18 +3353,22 @@ def getTraceback(self, table, in_id, in_id2=None): 'instrumentproductlink', 'satellite', 'mission'] in_id = self.getProcessID(in_id) - sq = (self.session.query(self.Process, - self.Product, self.Instrument, - self.Instrumentproductlink, self.Satellite, - self.Mission) + self.Product, + self.Instrument, + self.Instrumentproductlink, + self.Satellite, + self.Mission, + ) .filter_by(process_id=in_id) - .join((self.Product, self.Product.product_id == self.Process.output_product)) - .join((self.Inspector, self.Product.product_id == self.Inspector.product)) - .join((self.Instrumentproductlink, self.Product.product_id == self.Instrumentproductlink.product_id)) - .join((self.Instrument, self.Instrumentproductlink.instrument_id == self.Instrument.instrument_id)) - .join((self.Satellite, self.Instrument.satellite_id == self.Satellite.satellite_id)) - .join((self.Mission, self.Satellite.mission_id == self.Mission.mission_id)).all()) + .join(self.Product, self.Product.product_id == self.Process.output_product) + .join(self.Inspector, self.Inspector.product == self.Product.product_id) + .join(self.Instrumentproductlink, self.Instrumentproductlink.product_id == self.Product.product_id) + .join(self.Instrument, self.Instrument.instrument_id == self.Instrumentproductlink.instrument_id) + .join(self.Satellite, self.Satellite.satellite_id == self.Instrument.satellite_id) + .join(self.Mission, self.Mission.mission_id == self.Satellite.mission_id) + .all() + ) if not sq: # did not find a match this is a dberror raise DBError("process {0} did not have a traceback, this is a problem, fix it".format(in_id)) From 8eb5697ae57095ea685806694fd35bb84412d179 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 23 Jan 2024 21:56:26 -0500 Subject: [PATCH 05/28] DButils.py: fixed join methods --- dbprocessing/DButils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index 83c0134..b6b96f2 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -1997,7 +1997,7 @@ def getFileFullPath(self, filename): if isinstance(filename, str_classes): filename = self.getFileID(filename) sq = self.session.query(self.File.filename, self.Product.relative_path).filter( - self.File.file_id == filename).join((self.Product, self.File.product_id == self.Product.product_id)).one() + self.File.file_id == filename).join(self.Product, self.Product.product_id == self.File.product_id).one() path = os.path.join(self.MissionDirectory, *(sq[1].split(posixpath.sep) + [sq[0]])) if '{' in path: @@ -3279,7 +3279,7 @@ def getTraceback(self, table, in_id, in_id2=None): vars = ['code', 'process'] sq = (self.session.query(self.Code, self.Process) .filter_by(code_id=in_id) - .join((self.Process, self.Code.process_id == self.Process.process_id)).all()) + .join(self.Process, self.Process.process_id == self.Code.process_id).all()) if not sq: # did not find a match this is a dberror raise DBError("code {0} did not have a traceback, this is a problem, fix it".format(in_id)) From 028dbf5f38da0a59783011877ce1286a3b633575 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 23 Jan 2024 23:24:33 -0500 Subject: [PATCH 06/28] moved the bind from Metadata constructor to table creation --- unit_tests/test_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_tables.py b/unit_tests/test_tables.py index 1744827..dc60cbe 100755 --- a/unit_tests/test_tables.py +++ b/unit_tests/test_tables.py @@ -30,7 +30,7 @@ def setUp(self): self.engine = sqlalchemy.create_engine( 'sqlite:///{}'.format(os.path.join(self.td, 'test.sqlite')), echo=False) - self.metadata = sqlalchemy.schema.MetaData(bind=self.engine) + self.metadata = sqlalchemy.schema.MetaData() def tearDown(self): """Delete test database""" @@ -54,7 +54,7 @@ def makeTables(self, *tables): name: sqlalchemy.schema.Table( name, self.metadata, *dbprocessing.tables.definition(name)) for name in tables} - self.metadata.create_all() + self.metadata.create_all(bind=self.engine) actual = sqlalchemy.inspection.inspect(self.engine)\ .get_table_names() self.assertEqual(sorted(tables), sorted(actual)) From 9c54844c8027c5736cda1538ac417b2ef7df13fd Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Thu, 25 Jan 2024 03:01:07 -0500 Subject: [PATCH 07/28] DButils.py: fixed assertion error --- dbprocessing/DButils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index b6b96f2..d08e454 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -2740,7 +2740,7 @@ def getProductID(self, product_name): # no file_id found raise DBNoData("No product_name %s found in the DB" % (product_name)) # Numerical product ID, make sure it exists - sq = self.session.query(self.Product).get(product_name) + sq = self.session.get(self.Product, product_name) if sq is not None: return sq.product_id else: @@ -3560,7 +3560,7 @@ def getEntry(self, table, args): retval = None if isinstance(args, (int, collections.abc.Iterable)) \ and not isinstance(args, str_classes): # PK: int, non-str sequence - retval = self.session.query(getattr(self, table)).get(args) + retval = self.session.get(getattr(self, table), args) if retval is None: # Not valid PK type, or PK not found # see if it was a name if ('get' + table + 'ID') in dir(self): From 4becd0d8eacc14ddf5a8c3c6fddb15606c9e612f Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 30 Jan 2024 17:29:03 -0500 Subject: [PATCH 08/28] DButils.py: fixed all warning messages --- dbprocessing/DButils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index d08e454..ae0be12 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -2083,7 +2083,7 @@ def getProcessID(self, proc_name): """ try: proc_id = int(proc_name) - proc_name = self.session.query(self.Process).get(proc_id) + proc_name = self.session.get(self.Process, proc_id) if proc_name is None: raise NoResultFound('No row was found for id={0}'.format(proc_id)) except ValueError: # it is not a number @@ -2128,7 +2128,7 @@ def getInstrumentID(self, name, satellite_id=None): """ try: i_id = int(name) - sq = self.session.query(self.Instrument).get(i_id) + sq = self.session.get(self.Instrument, i_id) if sq is None: raise DBNoData("No instrument_id {0} found in the DB".format(i_id)) return sq.instrument_id @@ -2203,7 +2203,7 @@ def getFileID(self, filename): return filename.file_id try: f_id = int(filename) - sq = self.session.query(self.File).get(f_id) + sq = self.session.get(self.File, f_id) if sq is None: raise DBNoData("No file_id {0} found in the DB".format(filename)) return sq.file_id @@ -2233,7 +2233,7 @@ def getCodeID(self, codename): """ try: c_id = int(codename) - code = self.session.query(self.Code).get(c_id) + code = self.session.get(self.Code, c_id) if code is None: raise DBNoData("No code id {0} found in the DB".format(c_id)) except TypeError: # came in as list or tuple @@ -2766,7 +2766,7 @@ def getSatelliteID(self, """ try: sat_id = int(sat_name) - sq = self.session.query(self.Satellite).get(sat_id) + sq = self.session.get(self.Satellite, sat_id) if sq is None: raise NoResultFound("No satellite id={0} found".format(sat_id)) return sq.satellite_id @@ -3075,7 +3075,7 @@ def getMissionID(self, mission_name): """ try: m_id = int(mission_name) - ms = self.session.query(self.Mission).get(m_id) + ms = self.session.get(self.Mission, m_id) if ms is None: raise DBNoData('Invalid mission id {0}'.format(m_id)) except (ValueError, TypeError): @@ -3566,7 +3566,7 @@ def getEntry(self, table, args): if ('get' + table + 'ID') in dir(self): cmd = 'get' + table + 'ID' pk = getattr(self, cmd)(args) - retval = self.session.query(getattr(self, table)).get(pk) + retval = self.session.get((getattr(self, table)),pk) # This code will make it consistently raise DBNoData if nothing is found, # but codebase needs to be scrubbed for callers that expect None instead. # else: From 812baa87a90f495721eb4e22e4ee624713d04a9b Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Fri, 2 Feb 2024 14:09:04 -0500 Subject: [PATCH 09/28] undid change so CI will run 2.0 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92ddc2b..c1c8395 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: pip install --user python-dateutil sudo apt update sudo apt install python3-pip - pip3 install --user "sqlalchemy<2.0" + pip3 install --user sqlalchemy pip3 install --user python-dateutil - persist_to_workspace: root: ~/.local From 7a12790e5966721b772dacf4b81211f397de2c35 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 6 Feb 2024 11:52:16 -0500 Subject: [PATCH 10/28] added bind to drop_all and removed self from dbu.engine --- unit_tests/dbp_testing.py | 2 +- unit_tests/test_CreateDB.py | 2 +- unit_tests/test_DButils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/dbp_testing.py b/unit_tests/dbp_testing.py index 6b63d85..aae9ef7 100644 --- a/unit_tests/dbp_testing.py +++ b/unit_tests/dbp_testing.py @@ -229,7 +229,7 @@ def removeTestDB(self): """ if self.pg: self.dbu.session.close() - self.dbu.metadata.drop_all() + self.dbu.metadata.drop_all(bind=self.dbu.engine) self.dbu.closeDB() # Before the database is removed... del self.dbu shutil.rmtree(self.td) diff --git a/unit_tests/test_CreateDB.py b/unit_tests/test_CreateDB.py index 17cfe85..6476a5e 100644 --- a/unit_tests/test_CreateDB.py +++ b/unit_tests/test_CreateDB.py @@ -26,7 +26,7 @@ def test1(self): dbu = DButils.DButils(testdb) if pg: dbu.session.close() - dbu.metadata.drop_all() + dbu.metadata.drop_all(bind=dbu.engine) del dbu finally: shutil.rmtree(td) diff --git a/unit_tests/test_DButils.py b/unit_tests/test_DButils.py index da58fbb..6656573 100755 --- a/unit_tests/test_DButils.py +++ b/unit_tests/test_DButils.py @@ -60,7 +60,7 @@ def tearDown(self): super(DBUtilsEmptyTests, self).tearDown() if self.pg: self.dbu.session.close() - self.dbu.metadata.drop_all() + self.dbu.metadata.drop_all(bind=self.dbu.engine) self.dbu.closeDB() del self.dbu shutil.rmtree(self.td) From 83c49641995daf35a6cbb0b3683dd7e0251d10f6 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Thu, 22 Feb 2024 12:05:53 -0500 Subject: [PATCH 11/28] casted string with sqlalchemy.sql.text() --- unit_tests/dbp_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/dbp_testing.py b/unit_tests/dbp_testing.py index aae9ef7..5e916a0 100644 --- a/unit_tests/dbp_testing.py +++ b/unit_tests/dbp_testing.py @@ -306,7 +306,7 @@ def loadData(self, filename): sel = "SELECT pg_catalog.setval(pg_get_serial_sequence("\ "'{table}', '{column}'), {maxid})".format( table=t, column=idcolumn, maxid=maxid) - self.dbu.session.execute(sel) + self.dbu.session.execute(sqlalchemy.sql.text(sel)) self.dbu.commitDB() # Re-reference directories since new data loaded self.dbu.MissionDirectory = self.dbu.getMissionDirectory() From fbe6787b18b333ad9b08e7941fbb4f4b4527fc63 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Thu, 22 Feb 2024 12:23:27 -0500 Subject: [PATCH 12/28] added my name to contributors --- docs/CONTRIBUTORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CONTRIBUTORS.rst b/docs/CONTRIBUTORS.rst index 03d4fbb..0af939d 100644 --- a/docs/CONTRIBUTORS.rst +++ b/docs/CONTRIBUTORS.rst @@ -26,6 +26,7 @@ Current developers (*italics denote project administrators*) are: | Andrew Walker | Meilin Yan | Xiaoguang Yang + | Elisabeth Drakatos Acknowledgements ================ From 2e1cb809dc6e9ab9097862a789bb6af35a5f9f7f Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 20 May 2024 14:48:44 -0400 Subject: [PATCH 13/28] added bind to create_all --- examples/scripts/CreateDBsabrs.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/scripts/CreateDBsabrs.py b/examples/scripts/CreateDBsabrs.py index a3bed96..5e322ba 100644 --- a/examples/scripts/CreateDBsabrs.py +++ b/examples/scripts/CreateDBsabrs.py @@ -318,10 +318,7 @@ def createDB(self): # engine = create_engine('postgres:///' + self.filename, echo=False) # metadata.bind = engine - metadata.create_all(checkfirst=True) - # self.engine = engine - # self.metadata = metadata - + metadata.create_all(checkfirst=True, bind=engine) def addMission(self, filename): """utility to add a mission""" self.dbu = DButils.DButils(filename) From bde52aa32e535f96913f67947d01e7bab1817740 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 20 May 2024 15:11:55 -0400 Subject: [PATCH 14/28] removed bind from metadata --- examples/scripts/CreateDBsabrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scripts/CreateDBsabrs.py b/examples/scripts/CreateDBsabrs.py index 5e322ba..06f578f 100644 --- a/examples/scripts/CreateDBsabrs.py +++ b/examples/scripts/CreateDBsabrs.py @@ -37,7 +37,7 @@ def init_db(self, user, password, db, host='localhost', port=5432): url = "postgresql://{0}:{1}@{2}:{3}/{4}" url = url.format(user, password, host, port, db) self.engine = create_engine(url, echo=False, encoding='utf-8') - self.metadata = sqlalchemy.MetaData(bind=self.engine) + self.metadata = sqlalchemy.MetaData() self.metadata.reflect() def createDB(self): From aca843ceb4c71b127af135f6ff27fe86bfea8714 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 20 May 2024 15:54:30 -0400 Subject: [PATCH 15/28] scrubber: fix raw SQL syntax in version_number_check * fixes sqlalchemy 2.0 --- scripts/scrubber.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/scrubber.py b/scripts/scrubber.py index c415f1a..359b58f 100644 --- a/scripts/scrubber.py +++ b/scripts/scrubber.py @@ -24,7 +24,7 @@ def parents_are_newest(self): print(np.difference(n)) def version_number_check(self): - x = self.dbu.session.execute("SELECT max(interface_version), max(quality_version), max(revision_version) FROM file").fetchone() + x = self.dbu.session.execute(sqlalchemy.sql.text("SELECT max(interface_version), max(quality_version), max(revision_version) FROM file")).fetchone() if x[0] >= 1000: print("A interface version is too large") if x[1] >= 1000: From 8002ce065b0b0bec0e4adc1166099826ea441fec Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 20 May 2024 16:05:25 -0400 Subject: [PATCH 16/28] clean_test_db: fix raw SQL syntax in vacuum * fixes sqlalchemy 2.0 --- developer/scripts/clean_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/scripts/clean_test_db.py b/developer/scripts/clean_test_db.py index 43e731a..c99579b 100644 --- a/developer/scripts/clean_test_db.py +++ b/developer/scripts/clean_test_db.py @@ -116,5 +116,5 @@ def find_related_products(dbu, prod_ids, outputs=False): for file_id in delme: dbu._purgeFileFromDB(file_id, trust_id=True, commit=False) dbu.commitDB() -dbu.session.execute('VACUUM') +dbu.session.execute(sqlalchemy.sql.text("VACUUM")) dbu.commitDB() From 34a04abd3bce264194ee3e176cdc24d388633b21 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 21 May 2024 11:37:43 -0400 Subject: [PATCH 17/28] removed extra parentheses --- dbprocessing/DButils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbprocessing/DButils.py b/dbprocessing/DButils.py index ae0be12..6103d99 100644 --- a/dbprocessing/DButils.py +++ b/dbprocessing/DButils.py @@ -3566,7 +3566,7 @@ def getEntry(self, table, args): if ('get' + table + 'ID') in dir(self): cmd = 'get' + table + 'ID' pk = getattr(self, cmd)(args) - retval = self.session.get((getattr(self, table)),pk) + retval = self.session.get(getattr(self, table),pk) # This code will make it consistently raise DBNoData if nothing is found, # but codebase needs to be scrubbed for callers that expect None instead. # else: From edc8b6a01386e04a72650185979fe85fe2e67778 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Wed, 18 Sep 2024 11:05:20 -0400 Subject: [PATCH 18/28] sqlalchemy.sql was used without it imported --- scripts/scrubber.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/scrubber.py b/scripts/scrubber.py index 359b58f..bec0c6d 100644 --- a/scripts/scrubber.py +++ b/scripts/scrubber.py @@ -2,6 +2,8 @@ import argparse +import sqlalchemy.sql + from dbprocessing import DButils class scrubber(object): From 80c5eab0fe43f7dca0f61bfe3351a22dd208d68f Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Thu, 9 Jan 2025 12:54:54 -0500 Subject: [PATCH 19/28] move the bind from Metadata constructor to table creation (to reflect call) --- examples/scripts/CreateDBsabrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scripts/CreateDBsabrs.py b/examples/scripts/CreateDBsabrs.py index 06f578f..6dd4fa5 100644 --- a/examples/scripts/CreateDBsabrs.py +++ b/examples/scripts/CreateDBsabrs.py @@ -38,7 +38,7 @@ def init_db(self, user, password, db, host='localhost', port=5432): url = url.format(user, password, host, port, db) self.engine = create_engine(url, echo=False, encoding='utf-8') self.metadata = sqlalchemy.MetaData() - self.metadata.reflect() + self.metadata.reflect(bind=self.engine) def createDB(self): """ From ac70274292ec392da3a08ebf26d4a16602165383 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Fri, 17 Jan 2025 14:04:06 -0500 Subject: [PATCH 20/28] execute no longer taking direct SQL string --- developer/scripts/clean_test_db.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/developer/scripts/clean_test_db.py b/developer/scripts/clean_test_db.py index c99579b..13a6bf2 100644 --- a/developer/scripts/clean_test_db.py +++ b/developer/scripts/clean_test_db.py @@ -15,7 +15,7 @@ import datetime import dbprocessing.DButils - +import sqlalchemy def find_related_products(dbu, prod_ids, outputs=False): """Find all input/output products for a list of products @@ -67,7 +67,6 @@ def find_related_products(dbu, prod_ids, outputs=False): for prod_id in range(1, 190): if prod_id in keep_products: continue - files = [rec.file_id for rec in dbu.getFiles(product=prod_id)] for file_id in files: dbu._purgeFileFromDB(file_id, trust_id=True, commit=False) @@ -76,13 +75,11 @@ def find_related_products(dbu, prod_ids, outputs=False): .filter_by(product_id=prod_id) for ll in list(sq): dbu.session.delete(ll) - sq = dbu.session.query(dbu.Productprocesslink)\ .filter_by(input_product_id=prod_id) results = list(sq) for ll in results: dbu.session.delete(ll) - sq = dbu.session.query(dbu.Process).filter_by(output_product=prod_id) results = list(sq) for ll in results: @@ -99,7 +96,6 @@ def find_related_products(dbu, prod_ids, outputs=False): results = list(sq) for ll in results: dbu.session.delete(ll) - dbu.delProduct(prod_id) # performs commit # Only keep a few dates From e05685aff83b2f1108174b3f0e1ad5b0fb89c1fc Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 20 Jan 2025 07:32:39 -0500 Subject: [PATCH 21/28] adding self to bind argument --- examples/scripts/CreateDBsabrs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scripts/CreateDBsabrs.py b/examples/scripts/CreateDBsabrs.py index 6dd4fa5..62b20eb 100644 --- a/examples/scripts/CreateDBsabrs.py +++ b/examples/scripts/CreateDBsabrs.py @@ -318,7 +318,7 @@ def createDB(self): # engine = create_engine('postgres:///' + self.filename, echo=False) # metadata.bind = engine - metadata.create_all(checkfirst=True, bind=engine) + metadata.create_all(checkfirst=True, bind=self.engine) def addMission(self, filename): """utility to add a mission""" self.dbu = DButils.DButils(filename) From ba5c4bf01cb8b35a9ffe479a394ea17d9bce08e1 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Fri, 7 Feb 2025 12:33:39 -0500 Subject: [PATCH 22/28] Creates error directory if it doesn't exist --- dbprocessing/runMe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbprocessing/runMe.py b/dbprocessing/runMe.py index 1614987..0678dab 100644 --- a/dbprocessing/runMe.py +++ b/dbprocessing/runMe.py @@ -723,6 +723,7 @@ def moveToError(self, fname): DBlogging.dblogger.debug("Entered moveToError: {0}".format(fname)) path = self.dbu.getErrorPath() + os.makedirs(path, exist_ok=True) # Creates error directory if doesn't exist if os.path.isfile(os.path.join(path, os.path.basename(fname) ) ): #TODO do I really want to remove old version:? os.remove( os.path.join(path, os.path.basename(fname) ) ) From 403d1c7af71444b00f21f573a2a70a774776a472 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Fri, 7 Feb 2025 12:37:04 -0500 Subject: [PATCH 23/28] Updating python print syntax to a function call --- functional_test/scripts/run_rot13_L0toL1.py | 4 ++-- functional_test/scripts/run_rot13_L1toL2.py | 9 +++++---- functional_test/scripts/run_rot13_RUN_timebase.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/functional_test/scripts/run_rot13_L0toL1.py b/functional_test/scripts/run_rot13_L0toL1.py index 1954ddb..3c25ef7 100755 --- a/functional_test/scripts/run_rot13_L0toL1.py +++ b/functional_test/scripts/run_rot13_L0toL1.py @@ -19,6 +19,6 @@ def doProcess(infiles, outfile): infiles = sorted(args[:-1]) outfile = args[-1] - print "infiles", infiles - print "outfile", outfile + print("infiles + ", infiles) + print("outfile + ", outfile) doProcess(infiles, outfile) diff --git a/functional_test/scripts/run_rot13_L1toL2.py b/functional_test/scripts/run_rot13_L1toL2.py index d1b6290..f0e3e04 100755 --- a/functional_test/scripts/run_rot13_L1toL2.py +++ b/functional_test/scripts/run_rot13_L1toL2.py @@ -1,11 +1,12 @@ #!/usr/bin/env python +import codecs from optparse import OptionParser def doProcess(infile, outfile): with open(outfile, 'w') as output: with open(infile) as infile: - output.write(infile.read().encode('rot13')) - + output.write(codecs.encode(infile.read(), 'rot_13')) + if __name__ == '__main__': usage = "usage: %prog infile outfile" parser = OptionParser(usage=usage) @@ -18,6 +19,6 @@ def doProcess(infile, outfile): infile = args[0] outfile = args[-1] - print "infile", infile - print "outfile", outfile + print("infile + ", infile) + print("outfile + ", outfile) doProcess(infile, outfile) diff --git a/functional_test/scripts/run_rot13_RUN_timebase.py b/functional_test/scripts/run_rot13_RUN_timebase.py index 8dcbfed..37198ab 100755 --- a/functional_test/scripts/run_rot13_RUN_timebase.py +++ b/functional_test/scripts/run_rot13_RUN_timebase.py @@ -16,5 +16,5 @@ def doProcess(infile): infile = args[0] - print "infile", infile + print("infile + ", infile) doProcess(infile) From 587b029672977df88e7a62069b68566d94d2a82f Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Mon, 10 Feb 2025 11:55:44 -0500 Subject: [PATCH 24/28] Updated imp to importlib for load_source past Python 3.4 --- dbprocessing/dbprocessing.py | 12 ++++++++-- unit_tests/test_Inspector.py | 45 ++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/dbprocessing/dbprocessing.py b/dbprocessing/dbprocessing.py index e35c38c..190b5ef 100644 --- a/dbprocessing/dbprocessing.py +++ b/dbprocessing/dbprocessing.py @@ -5,7 +5,6 @@ from __future__ import print_function import datetime -import imp import os import shutil import sys @@ -259,7 +258,16 @@ def figureProduct(self, filename=None): claimed = [] for code, desc, arg, product in act_insp: try: - inspect = imp.load_source('inspect', code) + if sys.version_info < (3,4): + import imp # imp is deprecated at python 3.4 + inspect = imp.load_source('inspect', code) + else: + import importlib.util # used in python 3.4 and above + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('inspect', code) + spec = importlib.util.spec_from_file_location('inspect', code, loader=loader) + inspect = importlib.util.module_from_spec(spec) + loader.exec_module(inspect) except IOError as msg: DBlogging.dblogger.error('Inspector: "{0}" not found: {1}'.format(code, msg)) if os.path.isfile(code + ' '): diff --git a/unit_tests/test_Inspector.py b/unit_tests/test_Inspector.py index 44ac7d7..1741f93 100755 --- a/unit_tests/test_Inspector.py +++ b/unit_tests/test_Inspector.py @@ -4,7 +4,7 @@ import datetime import unittest import tempfile -import imp +import sys import warnings import os @@ -59,9 +59,17 @@ def setUp(self): self.makeTestDB() self.loadData(os.path.join(dbp_testing.testsdir, 'data', 'db_dumps', 'testDB_dump.json')) - self.inspect = imp.load_source('inspect', os.path.join( - dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) - + if sys.version_info < (3, 4): + import imp + self.inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) + else: + import importlib.util + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) + spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py'), loader=loader) + self.inspect = importlib.util.module_from_spec(spec) + loader.exec_module(self.inspect) + def tearDown(self): super(InspectorClass, self).tearDown() self.removeTestDB() @@ -80,10 +88,18 @@ def test_inspector(self): self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(self.inspect.Inspector(goodfile, self.dbu, 1,)())) #self.assertEqual(None, self.inspect.Inspector(goodfile, self.dbu, 1,).extract_YYYYMMDD()) # This inspector sets the data_level - not allowed - inspect = imp.load_source('inspect', os.path.join( - dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')) - with warnings.catch_warnings(record=True) as w: - self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(self.inspect.Inspector(goodfile, self.dbu, 1,)())) + if sys.version_info < (3, 4): + import imp # Depracated in Python 3.4 + inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')) + else: + import importlib.util + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')) + spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py'), loader=loader) + inspect = importlib.util.module_from_spec(spec) + loader.exec_module(inspect) + with warnings.catch_warnings(record=True) as w: + self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(inspect.Inspector(goodfile, self.dbu, 1,)())) self.assertEqual(len(w), 1) self.assertTrue(isinstance(w[0].message, UserWarning)) self.assertEqual('Inspector rot13_L1_dlevel.py: set level to 2.0, ' @@ -93,8 +109,17 @@ def test_inspector(self): # The file doesn't match the inspector pattern... badfile = os.path.join( dbp_testing.testsdir, 'inspector', 'testDB_01_first.raw') - inspect = imp.load_source('inspect', os.path.join( - dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) + if sys.version_info < (3, 4): + import imp # Depracated in Python 3.4 + inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) + else: + import importlib.util + import importlib.machinery + loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) + spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py'), loader=loader) + inspect = importlib.util.module_from_spec(spec) + loader.exec_module(inspect) + self.assertEqual(None, inspect.Inspector(badfile, self.dbu, 1,)()) def test_inspector_regex(self): From f92c10cf9460c91bc9b71df26120020f0454f179 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 11 Mar 2025 09:42:42 -0400 Subject: [PATCH 25/28] Updated error message checking to comply with Python 3.12 syntax --- unit_tests/test_DBRunner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/unit_tests/test_DBRunner.py b/unit_tests/test_DBRunner.py index 5255c05..a96850c 100644 --- a/unit_tests/test_DBRunner.py +++ b/unit_tests/test_DBRunner.py @@ -86,10 +86,8 @@ def test_parse_dbrunner_args_bad(self): finally: sys.stderr.close() sys.stderr = oldstderr - self.assertEqual( - '{}: error: {}'.format(os.path.basename(sys.argv[0]), msg), err) - - + running_script = os.path.basename(sys.argv[0]) + self.assertTrue(err.startswith("{}: error: argument".format(running_script))) class DBRunnerCalcRunmeTests(unittest.TestCase, dbp_testing.AddtoDBMixin): """DBRunner tests of calc_runme""" From c862aed04bc570d5ffa0a7664003a0310701e283 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 1 Apr 2025 08:31:04 -0400 Subject: [PATCH 26/28] Updating concatanation --- functional_test/scripts/run_rot13_L0toL1.py | 4 ++-- functional_test/scripts/run_rot13_L1toL2.py | 7 +++---- functional_test/scripts/run_rot13_RUN_timebase.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/functional_test/scripts/run_rot13_L0toL1.py b/functional_test/scripts/run_rot13_L0toL1.py index 3c25ef7..903ad6c 100755 --- a/functional_test/scripts/run_rot13_L0toL1.py +++ b/functional_test/scripts/run_rot13_L0toL1.py @@ -19,6 +19,6 @@ def doProcess(infiles, outfile): infiles = sorted(args[:-1]) outfile = args[-1] - print("infiles + ", infiles) - print("outfile + ", outfile) + print("infiles ", infiles) + print("outfile ", outfile) doProcess(infiles, outfile) diff --git a/functional_test/scripts/run_rot13_L1toL2.py b/functional_test/scripts/run_rot13_L1toL2.py index f0e3e04..f81f2bb 100755 --- a/functional_test/scripts/run_rot13_L1toL2.py +++ b/functional_test/scripts/run_rot13_L1toL2.py @@ -5,8 +5,7 @@ def doProcess(infile, outfile): with open(outfile, 'w') as output: with open(infile) as infile: - output.write(codecs.encode(infile.read(), 'rot_13')) - + output.write(codecs.encode(infile.read(), 'rot_13')) if __name__ == '__main__': usage = "usage: %prog infile outfile" parser = OptionParser(usage=usage) @@ -19,6 +18,6 @@ def doProcess(infile, outfile): infile = args[0] outfile = args[-1] - print("infile + ", infile) - print("outfile + ", outfile) + print("infile ", infile) + print("outfile ", outfile) doProcess(infile, outfile) diff --git a/functional_test/scripts/run_rot13_RUN_timebase.py b/functional_test/scripts/run_rot13_RUN_timebase.py index 37198ab..662db9e 100755 --- a/functional_test/scripts/run_rot13_RUN_timebase.py +++ b/functional_test/scripts/run_rot13_RUN_timebase.py @@ -16,5 +16,5 @@ def doProcess(infile): infile = args[0] - print("infile + ", infile) + print("infile ", infile) doProcess(infile) From 62ed2ee68a250c413fa61c1ed63b5b6ff92c8cd0 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 1 Apr 2025 11:47:30 -0400 Subject: [PATCH 27/28] Creating function to handle the update of imp to importlib --- dbprocessing/Utils.py | 15 +++++++++++++ dbprocessing/dbprocessing.py | 13 +++-------- unit_tests/test_Inspector.py | 42 +++++++++--------------------------- 3 files changed, 28 insertions(+), 42 deletions(-) diff --git a/dbprocessing/Utils.py b/dbprocessing/Utils.py index 499fa84..8a37a79 100644 --- a/dbprocessing/Utils.py +++ b/dbprocessing/Utils.py @@ -560,3 +560,18 @@ def readconfig(config_filepath): else: ans[section][item] = (ans[section][item], 0, 0) return ans +def load_source(modname, filepath, module): + """ + The imp module was removed in Python 3.4, thus, adaptations were made so the imp.load_source feature can be used for later Python versions + """ + try: + import importlib.util + import importlib.machinery + loader = importlib.machinery.SourceFileLoader(modname, filepath) + spec = importlib.util.spec_from_file_location(modname, filepath, loader=loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + except ImportError: + import imp # Depracated in Python 3.4 + module = imp.load_source(modname, filepath) + return module diff --git a/dbprocessing/dbprocessing.py b/dbprocessing/dbprocessing.py index 190b5ef..37d759d 100644 --- a/dbprocessing/dbprocessing.py +++ b/dbprocessing/dbprocessing.py @@ -258,16 +258,9 @@ def figureProduct(self, filename=None): claimed = [] for code, desc, arg, product in act_insp: try: - if sys.version_info < (3,4): - import imp # imp is deprecated at python 3.4 - inspect = imp.load_source('inspect', code) - else: - import importlib.util # used in python 3.4 and above - import importlib.machinery - loader = importlib.machinery.SourceFileLoader('inspect', code) - spec = importlib.util.spec_from_file_location('inspect', code, loader=loader) - inspect = importlib.util.module_from_spec(spec) - loader.exec_module(inspect) + fname = code + inspect = None + inspect = Utils.load_source('inspect',fname, inspect) except IOError as msg: DBlogging.dblogger.error('Inspector: "{0}" not found: {1}'.format(code, msg)) if os.path.isfile(code + ' '): diff --git a/unit_tests/test_Inspector.py b/unit_tests/test_Inspector.py index 1741f93..9ea4d5e 100755 --- a/unit_tests/test_Inspector.py +++ b/unit_tests/test_Inspector.py @@ -14,6 +14,7 @@ from dbprocessing import Version from dbprocessing import DButils from dbprocessing import Diskfile +from dbprocessing import Utils class InspectorFunctions(unittest.TestCase): """Tests of the inspector functions""" @@ -59,17 +60,9 @@ def setUp(self): self.makeTestDB() self.loadData(os.path.join(dbp_testing.testsdir, 'data', 'db_dumps', 'testDB_dump.json')) - if sys.version_info < (3, 4): - import imp - self.inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) - else: - import importlib.util - import importlib.machinery - loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) - spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py'), loader=loader) - self.inspect = importlib.util.module_from_spec(spec) - loader.exec_module(self.inspect) - + filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py') + self.inspect = None + self.inspect = Utils.load_source('inspect', filename, self.inspect) def tearDown(self): super(InspectorClass, self).tearDown() self.removeTestDB() @@ -88,16 +81,9 @@ def test_inspector(self): self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(self.inspect.Inspector(goodfile, self.dbu, 1,)())) #self.assertEqual(None, self.inspect.Inspector(goodfile, self.dbu, 1,).extract_YYYYMMDD()) # This inspector sets the data_level - not allowed - if sys.version_info < (3, 4): - import imp # Depracated in Python 3.4 - inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')) - else: - import importlib.util - import importlib.machinery - loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')) - spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py'), loader=loader) - inspect = importlib.util.module_from_spec(spec) - loader.exec_module(inspect) + filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py') + inspect = None + inspect = Utils.load_source('inspect', filename, inspect) with warnings.catch_warnings(record=True) as w: self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(inspect.Inspector(goodfile, self.dbu, 1,)())) self.assertEqual(len(w), 1) @@ -109,17 +95,9 @@ def test_inspector(self): # The file doesn't match the inspector pattern... badfile = os.path.join( dbp_testing.testsdir, 'inspector', 'testDB_01_first.raw') - if sys.version_info < (3, 4): - import imp # Depracated in Python 3.4 - inspect = imp.load_source('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) - else: - import importlib.util - import importlib.machinery - loader = importlib.machinery.SourceFileLoader('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')) - spec = importlib.util.spec_from_file_location('inspect', os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py'), loader=loader) - inspect = importlib.util.module_from_spec(spec) - loader.exec_module(inspect) - + filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py') + inspect = None + inspect = Utils.load_source('inspect', filename, inspect) self.assertEqual(None, inspect.Inspector(badfile, self.dbu, 1,)()) def test_inspector_regex(self): From d60fac1195417358be8acc2943c0059d7224ee45 Mon Sep 17 00:00:00 2001 From: ezd1000 Date: Tue, 1 Apr 2025 14:23:20 -0400 Subject: [PATCH 28/28] Test for load_source function --- unit_tests/test_Utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_Utils.py b/unit_tests/test_Utils.py index 652cd4c..b8034a6 100755 --- a/unit_tests/test_Utils.py +++ b/unit_tests/test_Utils.py @@ -304,7 +304,14 @@ def test_toDatetime(self): self.assertEqual( datetime.datetime(2010, 1, 1, 23, 59, 59, 999999), Utils.toDatetime(datetime.date(2010, 1, 1), end=True)) - - + def test_load_source(self): + """Testing load_source in Utils.py""" + filename = "temp_testfile.py" + with open(filename, "w") as f: + f.write("def hello():\n return 'Hello, world!'\n") + inspect = None + module = Utils.load_source("temp_testfile", filename, inspect) + self.assertEqual(module.hello(), 'Hello, world!') + if __name__ == "__main__": unittest.main()