Don't track columns as separate attribute, use the CSV header when copying (columns copied can change between HDF tables for a single SQL table)

Author: bleonard33

pandas_to_postgres/_base_copy.py

@@ -33,7 +33,6 @@ def __init__(
         """
 
         self.rows = 0
-        self.columns = None
         self.csv_chunksize = csv_chunksize
 
         if not defer_sql_objs:
@@ -112,8 +111,9 @@ def copy_from_file(self, file_object: StringIO):
         file_object: CSV formatted data to COPY from DataFrame to PostgreSQL
         """
         cur = self.conn.connection.cursor()
-        cols = ", ".join([f"{col}" for col in self.columns])
-        sql = f"COPY {self.sql_table} ({cols}) FROM STDIN WITH CSV HEADER FREEZE"
+        file_object.seek(0)
+        columns = file_object.readline()
+        sql = f"COPY {self.sql_table} ({columns}) FROM STDIN WITH CSV FREEZE"
         cur.copy_expert(sql=sql, file=file_object)
 
     def data_formatting(self, df: DataFrame, functions: List[Callable] = [], **kwargs):

pandas_to_postgres/copy_df.py

@@ -20,7 +20,6 @@ def __init__(
 
         self.df = df
         self.levels = levels
-        self.columns = self.df.columns
         self.rows = self.df.shape[0]
 
     def copy(self, functions=[cast_pandas]):

pandas_to_postgres/copy_hdf.py

@@ -62,15 +62,11 @@ def hdf_to_pg(self, data_formatters=[cast_pandas], data_formatter_kwargs={}):
             self.rows += len(df)
 
             data_formatter_kwargs["hdf_table"] = hdf_table
-
             logger.info("Formatting data")
             df = self.data_formatting(
                 df, functions=data_formatters, **data_formatter_kwargs
             )
 
-            if self.columns is None:
-                self.columns = df.columns
-
             logger.info("Creating generator for chunking dataframe")
             for chunk in df_generator(df, self.csv_chunksize):
 
@@ -122,9 +118,6 @@ def hdf_to_pg(self, data_formatters=[cast_pandas], data_formatter_kwargs={}):
                 df, functions=data_formatters, **data_formatter_kwargs
             )
 
-            if self.columns is None:
-                self.columns = df.columns
-
             logger.info("Creating CSV in memory")
             fo = create_file_object(df)
 
@@ -189,9 +182,6 @@ def hdf_to_pg(self, data_formatters=[cast_pandas], data_formatter_kwargs={}):
                     df, functions=data_formatters, **data_formatter_kwargs
                 )
 
-                if self.columns is None:
-                    self.columns = df.columns
-
                 logger.info("Creating generator for chunking dataframe")
                 for chunk in df_generator(df, self.csv_chunksize):
                     logger.info("Creating CSV in memory")