updated dbprocessing code for sqlalchemy 2.0

.circleci/config.yml

@@ -20,7 +20,7 @@ jobs:
              pip install --user python-dateutil
              sudo apt update
              sudo apt install python3-pip
-             pip3 install --user "sqlalchemy<2.0"
+             pip3 install --user sqlalchemy
              pip3 install --user python-dateutil
        - persist_to_workspace:
            root: ~/.local

dbprocessing/Utils.py

@@ -560,3 +560,18 @@ def readconfig(config_filepath):
                 else:
                     ans[section][item] = (ans[section][item], 0, 0)
     return ans
+def load_source(modname, filepath, module):
+    """
+    The imp module was removed in Python 3.4, thus, adaptations were made so the imp.load_source feature can be used for later Python versions
+    """
+    try:
+        import importlib.util
+        import importlib.machinery
+        loader = importlib.machinery.SourceFileLoader(modname, filepath)
+        spec = importlib.util.spec_from_file_location(modname, filepath, loader=loader)
+        module = importlib.util.module_from_spec(spec)
+        loader.exec_module(module)
+    except ImportError:
+        import imp # Depracated in Python 3.4
+        module = imp.load_source(modname, filepath)
+    return module

dbprocessing/dbprocessing.py

@@ -5,7 +5,6 @@
 from __future__ import print_function
 
 import datetime
-import imp
 import os
 import shutil
 import sys
@@ -259,7 +258,9 @@ def figureProduct(self, filename=None):
         claimed = []
         for code, desc, arg, product in act_insp:
             try:
-                inspect = imp.load_source('inspect', code)
+                fname = code
+                inspect = None
+                inspect = Utils.load_source('inspect',fname, inspect)
             except IOError as msg:
                 DBlogging.dblogger.error('Inspector: "{0}" not found: {1}'.format(code, msg))
                 if os.path.isfile(code + ' '):

dbprocessing/runMe.py

@@ -723,6 +723,7 @@ def moveToError(self, fname):
         DBlogging.dblogger.debug("Entered moveToError: {0}".format(fname))
 
         path = self.dbu.getErrorPath()
+        os.makedirs(path, exist_ok=True) # Creates error directory if doesn't exist 
         if os.path.isfile(os.path.join(path, os.path.basename(fname) ) ):
         #TODO do I really want to remove old version:?
             os.remove( os.path.join(path, os.path.basename(fname) ) )

developer/scripts/clean_test_db.py

@@ -15,7 +15,7 @@
 
 import datetime
 import dbprocessing.DButils
-
+import sqlalchemy
 
 def find_related_products(dbu, prod_ids, outputs=False):
     """Find all input/output products for a list of products
@@ -67,7 +67,6 @@ def find_related_products(dbu, prod_ids, outputs=False):
 for prod_id in range(1, 190):
     if prod_id in keep_products:
         continue
-
     files = [rec.file_id for rec in dbu.getFiles(product=prod_id)]
     for file_id in files:
         dbu._purgeFileFromDB(file_id, trust_id=True, commit=False)
@@ -76,13 +75,11 @@ def find_related_products(dbu, prod_ids, outputs=False):
         .filter_by(product_id=prod_id)
     for ll in list(sq):
         dbu.session.delete(ll)
-
     sq = dbu.session.query(dbu.Productprocesslink)\
         .filter_by(input_product_id=prod_id)
     results = list(sq)
     for ll in results:
         dbu.session.delete(ll)
-
     sq = dbu.session.query(dbu.Process).filter_by(output_product=prod_id)
     results = list(sq)
     for ll in results:
@@ -99,7 +96,6 @@ def find_related_products(dbu, prod_ids, outputs=False):
     results = list(sq)
     for ll in results:
         dbu.session.delete(ll)
-
     dbu.delProduct(prod_id)  # performs commit
 
 # Only keep a few dates
@@ -116,5 +112,5 @@ def find_related_products(dbu, prod_ids, outputs=False):
 for file_id in delme:
     dbu._purgeFileFromDB(file_id, trust_id=True, commit=False)
 dbu.commitDB()
-dbu.session.execute('VACUUM')
+dbu.session.execute(sqlalchemy.sql.text("VACUUM"))
 dbu.commitDB()

docs/CONTRIBUTORS.rst

@@ -26,6 +26,7 @@ Current developers (*italics denote project administrators*) are:
    | Andrew Walker
    | Meilin Yan
    | Xiaoguang Yang
+   | Elisabeth Drakatos   
 
 Acknowledgements
 ================

examples/scripts/CreateDBsabrs.py

@@ -37,8 +37,8 @@ def init_db(self, user, password, db, host='localhost', port=5432):
         url = "postgresql://{0}:{1}@{2}:{3}/{4}"
         url = url.format(user, password, host, port, db)
         self.engine = create_engine(url, echo=False, encoding='utf-8')
-        self.metadata = sqlalchemy.MetaData(bind=self.engine)
-        self.metadata.reflect()
+        self.metadata = sqlalchemy.MetaData()
+        self.metadata.reflect(bind=self.engine)
 
     def createDB(self):
         """
@@ -318,10 +318,7 @@ def createDB(self):
         # engine = create_engine('postgres:///' + self.filename, echo=False)
         # metadata.bind = engine
 
-        metadata.create_all(checkfirst=True)
-        # self.engine = engine
-        # self.metadata = metadata
-
+        metadata.create_all(checkfirst=True, bind=self.engine)
     def addMission(self, filename):
         """utility to add a mission"""
         self.dbu = DButils.DButils(filename)

functional_test/scripts/run_rot13_L0toL1.py

@@ -19,6 +19,6 @@ def doProcess(infiles, outfile):
     infiles = sorted(args[:-1])
     outfile = args[-1]
 
-    print "infiles", infiles
-    print "outfile", outfile
+    print("infiles ", infiles)
+    print("outfile ", outfile)
     doProcess(infiles, outfile)

functional_test/scripts/run_rot13_L1toL2.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python
+import codecs
 from optparse import OptionParser
 
 def doProcess(infile, outfile):
     with open(outfile, 'w') as output:
         with open(infile) as infile:
-            output.write(infile.read().encode('rot13'))
-
+            output.write(codecs.encode(infile.read(), 'rot_13'))            
 if __name__ == '__main__':
     usage = "usage: %prog infile outfile"
     parser = OptionParser(usage=usage)
@@ -18,6 +18,6 @@ def doProcess(infile, outfile):
     infile = args[0]
     outfile = args[-1]
 
-    print "infile", infile
-    print "outfile", outfile
+    print("infile ", infile)
+    print("outfile ", outfile)
     doProcess(infile, outfile)

functional_test/scripts/run_rot13_RUN_timebase.py

@@ -16,5 +16,5 @@ def doProcess(infile):
 
 	infile = args[0]
 
-	print "infile", infile
+	print("infile ", infile)
 	doProcess(infile)

scripts/scrubber.py

@@ -2,6 +2,8 @@
 
 import argparse
 
+import sqlalchemy.sql
+
 from dbprocessing import DButils
 
 class scrubber(object):
@@ -24,7 +26,7 @@ def parents_are_newest(self):
             print(np.difference(n))
 
     def version_number_check(self):
-        x = self.dbu.session.execute("SELECT max(interface_version), max(quality_version), max(revision_version) FROM file").fetchone()
+        x = self.dbu.session.execute(sqlalchemy.sql.text("SELECT max(interface_version), max(quality_version), max(revision_version) FROM file")).fetchone()
         if x[0] >= 1000:
             print("A interface version is too large")
         if x[1] >= 1000:

unit_tests/dbp_testing.py

@@ -229,7 +229,7 @@ def removeTestDB(self):
         """
         if self.pg:
             self.dbu.session.close()
-            self.dbu.metadata.drop_all()
+            self.dbu.metadata.drop_all(bind=self.dbu.engine)
         self.dbu.closeDB() # Before the database is removed...
         del self.dbu
         shutil.rmtree(self.td)
@@ -282,7 +282,7 @@ def loadData(self, filename):
             # persist_selectable added 1.3 (mapped_table deprecated)
             tbl = insp.persist_selectable\
                   if hasattr(insp, 'persist_selectable') else insp.mapped_table
-            tbl.drop()
+            tbl.drop(bind=self.dbu.engine)
             self.dbu.metadata.remove(tbl)
             del self.dbu.Unixtime
         if data['productprocesslink']\
@@ -306,7 +306,7 @@ def loadData(self, filename):
                 sel = "SELECT pg_catalog.setval(pg_get_serial_sequence("\
                       "'{table}', '{column}'), {maxid})".format(
                           table=t, column=idcolumn, maxid=maxid)
-                self.dbu.session.execute(sel)
+                self.dbu.session.execute(sqlalchemy.sql.text(sel))
         self.dbu.commitDB()
         # Re-reference directories since new data loaded
         self.dbu.MissionDirectory = self.dbu.getMissionDirectory()

unit_tests/test_CreateDB.py

@@ -26,7 +26,7 @@ def test1(self):
             dbu = DButils.DButils(testdb)
             if pg:
                 dbu.session.close()
-                dbu.metadata.drop_all()
+                dbu.metadata.drop_all(bind=dbu.engine)
             del dbu
         finally:
             shutil.rmtree(td)

unit_tests/test_DBRunner.py

@@ -86,10 +86,8 @@ def test_parse_dbrunner_args_bad(self):
             finally:
                 sys.stderr.close()
                 sys.stderr = oldstderr
-            self.assertEqual(
-                '{}: error: {}'.format(os.path.basename(sys.argv[0]), msg), err)
-
-
+            running_script = os.path.basename(sys.argv[0])
+            self.assertTrue(err.startswith("{}: error: argument".format(running_script)))
 class DBRunnerCalcRunmeTests(unittest.TestCase, dbp_testing.AddtoDBMixin):
     """DBRunner tests of calc_runme"""
 

unit_tests/test_DButils.py

@@ -60,7 +60,7 @@ def tearDown(self):
         super(DBUtilsEmptyTests, self).tearDown()
         if self.pg:
             self.dbu.session.close()
-            self.dbu.metadata.drop_all()
+            self.dbu.metadata.drop_all(bind=self.dbu.engine)
         self.dbu.closeDB()
         del self.dbu
         shutil.rmtree(self.td)

unit_tests/test_Inspector.py

@@ -4,7 +4,7 @@
 import datetime
 import unittest
 import tempfile
-import imp
+import sys
 import warnings
 import os
 
@@ -14,6 +14,7 @@
 from dbprocessing import Version
 from dbprocessing import DButils
 from dbprocessing import Diskfile
+from dbprocessing import Utils
 
 class InspectorFunctions(unittest.TestCase):
     """Tests of the inspector functions"""
@@ -59,9 +60,9 @@ def setUp(self):
         self.makeTestDB()
         self.loadData(os.path.join(dbp_testing.testsdir, 'data', 'db_dumps',
                                    'testDB_dump.json'))
-        self.inspect = imp.load_source('inspect', os.path.join(
-            dbp_testing.testsdir, 'inspector', 'rot13_L1.py'))
-
+        filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')
+        self.inspect = None
+        self.inspect = Utils.load_source('inspect', filename, self.inspect)
     def tearDown(self):
         super(InspectorClass, self).tearDown()
         self.removeTestDB()
@@ -80,10 +81,11 @@ def test_inspector(self):
         self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(self.inspect.Inspector(goodfile, self.dbu, 1,)()))
         #self.assertEqual(None, self.inspect.Inspector(goodfile, self.dbu, 1,).extract_YYYYMMDD())
         # This inspector sets the data_level - not allowed
-        inspect = imp.load_source('inspect', os.path.join(
-            dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py'))
-        with warnings.catch_warnings(record=True) as w:
-            self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(self.inspect.Inspector(goodfile, self.dbu, 1,)()))
+        filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1_dlevel.py')
+        inspect = None
+        inspect = Utils.load_source('inspect', filename, inspect)
+        with warnings.catch_warnings(record=True) as w: 
+            self.assertEqual(repr(Diskfile.Diskfile(goodfile, self.dbu)), repr(inspect.Inspector(goodfile, self.dbu, 1,)()))
         self.assertEqual(len(w), 1)
         self.assertTrue(isinstance(w[0].message, UserWarning))
         self.assertEqual('Inspector rot13_L1_dlevel.py:  set level to 2.0, '
@@ -93,8 +95,9 @@ def test_inspector(self):
         # The file doesn't match the inspector pattern...
         badfile =  os.path.join(
             dbp_testing.testsdir, 'inspector', 'testDB_01_first.raw')
-        inspect = imp.load_source('inspect', os.path.join(
-            dbp_testing.testsdir, 'inspector', 'rot13_L1.py'))
+        filename = os.path.join(dbp_testing.testsdir, 'inspector', 'rot13_L1.py')
+        inspect = None
+        inspect = Utils.load_source('inspect', filename, inspect)    
         self.assertEqual(None, inspect.Inspector(badfile, self.dbu, 1,)())
 
     def test_inspector_regex(self):

unit_tests/test_Utils.py

@@ -304,7 +304,14 @@ def test_toDatetime(self):
         self.assertEqual(
             datetime.datetime(2010, 1, 1, 23, 59, 59, 999999),
             Utils.toDatetime(datetime.date(2010, 1, 1), end=True))
-
-
+    def test_load_source(self):
+        """Testing load_source in Utils.py"""
+        filename = "temp_testfile.py"
+        with open(filename, "w") as f:
+            f.write("def hello():\n    return 'Hello, world!'\n")
+        inspect = None
+        module = Utils.load_source("temp_testfile", filename, inspect)
+        self.assertEqual(module.hello(), 'Hello, world!')
+
 if __name__ == "__main__":
     unittest.main()

unit_tests/test_tables.py

@@ -30,7 +30,7 @@ def setUp(self):
         self.engine = sqlalchemy.create_engine(
             'sqlite:///{}'.format(os.path.join(self.td, 'test.sqlite')),
             echo=False)
-        self.metadata = sqlalchemy.schema.MetaData(bind=self.engine)
+        self.metadata = sqlalchemy.schema.MetaData()
 
     def tearDown(self):
         """Delete test database"""
@@ -54,7 +54,7 @@ def makeTables(self, *tables):
             name: sqlalchemy.schema.Table(
                 name, self.metadata, *dbprocessing.tables.definition(name))
             for name in tables}
-        self.metadata.create_all()
+        self.metadata.create_all(bind=self.engine)
         actual = sqlalchemy.inspection.inspect(self.engine)\
                  .get_table_names()
         self.assertEqual(sorted(tables), sorted(actual))