diff --git a/cdflib/xarray/cdf_to_xarray.py b/cdflib/xarray/cdf_to_xarray.py index ab16c5f..9552bdb 100644 --- a/cdflib/xarray/cdf_to_xarray.py +++ b/cdflib/xarray/cdf_to_xarray.py @@ -99,12 +99,10 @@ def _convert_cdf_time_types( def _convert_cdf_to_dicts( - filename: Union[str, Path], to_datetime: bool = False, to_unixtime: bool = False + cdf_file: CDF, to_datetime: bool = False, to_unixtime: bool = False ) -> Tuple[Dict[str, List[Union[str, np.ndarray]]], Dict[str, Any], Dict[str, npt.NDArray], Dict[str, VDRInfo]]: - # Open the CDF file # Converts the entire CDF file into python dictionary objects - cdf_file = CDF(filename, string_encoding="latin-1") cdf_info = cdf_file.cdf_info() all_cdf_variables = cdf_info.rVariables + cdf_info.zVariables @@ -685,12 +683,12 @@ def _verify_dimension_sizes(created_data_vars: Dict[str, xr.Variable], created_c ) -def cdf_to_xarray(filename: str, to_datetime: bool = True, to_unixtime: bool = False, fillval_to_nan: bool = False) -> xr.Dataset: +def cdf_to_xarray(cdf_file: CDF, to_datetime: bool = True, to_unixtime: bool = False, fillval_to_nan: bool = False) -> xr.Dataset: """ This function converts CDF files into XArray Dataset Objects. Parameters: - filename (str): The path to the CDF file to read + cdf_file (CDF): The CDF file to convert to_datetime (bool, optional): Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to datetime64, or leave them as is to_unixtime (bool, optional): Whether or not to convert CDF_EPOCH/EPOCH_16/TT2000 to unixtime, or leave them as is fillval_to_nan (bool, optional): If True, any data values that match the FILLVAL attribute for a variable will be set to NaN @@ -767,7 +765,7 @@ def cdf_to_xarray(filename: str, to_datetime: bool = True, to_unixtime: bool = F # Convert the CDF file into a series of dicts, so we don't need to keep reading the file global_attributes, all_variable_attributes, all_variable_data, all_variable_properties = _convert_cdf_to_dicts( - filename, to_datetime=to_datetime, to_unixtime=to_unixtime + cdf_file, to_datetime=to_datetime, to_unixtime=to_unixtime ) created_vars, depend_dimensions = _generate_xarray_data_variables( diff --git a/tests/test_cdfwrite.py b/tests/test_cdfwrite.py index 9c69ad0..b1b5548 100755 --- a/tests/test_cdfwrite.py +++ b/tests/test_cdfwrite.py @@ -4,7 +4,7 @@ import numpy as np import pytest -from cdflib import cdfread, cdfwrite +from cdflib import cdfread, cdfwrite, CDF from cdflib.xarray import cdf_to_xarray R = Path(__file__).parent @@ -660,7 +660,7 @@ def test_string_input_but_number_type(tmp_path): cdf.close() # Reading it back in would cause an error - cdf_to_xarray(tmp_path / "test.cdf") + cdf_to_xarray(CDF(tmp_path / "test.cdf")) def test_array_string_input_but_number_type(tmp_path): @@ -681,4 +681,4 @@ def test_array_string_input_but_number_type(tmp_path): cdf.close() # Reading it back in would cause an error - cdf_to_xarray(tmp_path / "test.cdf") + cdf_to_xarray(CDF(tmp_path / "test.cdf")) diff --git a/tests/test_xarray_reader_writer.py b/tests/test_xarray_reader_writer.py index 61c4f96..b7c5019 100644 --- a/tests/test_xarray_reader_writer.py +++ b/tests/test_xarray_reader_writer.py @@ -5,6 +5,7 @@ import pytest import xarray as xr +from cdflib import CDF from cdflib.xarray import cdf_to_xarray, xarray_to_cdf # To run these tests use `pytest --remote-data` @@ -43,10 +44,12 @@ def test_xarray_read_write(tmp_path, cdf_fname, nc_fname): if not os.path.exists(cdf_fname): urllib.request.urlretrieve(url, cdf_fname) - a = cdf_to_xarray(cdf_fname, fillval_to_nan=True) + with CDF(cdf_fname) as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, tmp_path / cdf_fname) - b = cdf_to_xarray(tmp_path / cdf_fname, fillval_to_nan=True) + with CDF(tmp_path / cdf_fname) as cdf_file: + b = cdf_to_xarray(cdf_file, fillval_to_nan=True) url = f"https://lasp.colorado.edu/maven/sdc/public/data/sdc/web/cdflib_testing/{nc_fname}" if not os.path.exists(nc_fname): @@ -54,7 +57,8 @@ def test_xarray_read_write(tmp_path, cdf_fname, nc_fname): c = xr.load_dataset(nc_fname) xarray_to_cdf(c, tmp_path / ("nc_" + cdf_fname)) - d = cdf_to_xarray(tmp_path / ("nc_" + cdf_fname), fillval_to_nan=True) + with CDF(tmp_path / ("nc_" + cdf_fname)) as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) @pytest.mark.remote_data @@ -73,7 +77,8 @@ def test_MGITM_model(): c["altitude"].attrs["VAR_TYPE"] = "support_data" xarray_to_cdf(c, "MGITM_LS180_F130_150615-created-from-netcdf-input.cdf") - d = cdf_to_xarray("MGITM_LS180_F130_150615-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("MGITM_LS180_F130_150615-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("MGITM_LS180_F130_150615-created-from-netcdf-input.cdf") os.remove("MGITM_LS180_F130_150615.nc") @@ -92,7 +97,8 @@ def test_goes_mag(): c["time"].attrs["VAR_TYPE"] = "support_data" c["time_orbit"].attrs["VAR_TYPE"] = "support_data" xarray_to_cdf(c, "dn_magn-l2-hires_g17_d20211219_v1-0-1-created-from-netcdf-input.cdf") - d = cdf_to_xarray("dn_magn-l2-hires_g17_d20211219_v1-0-1-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("dn_magn-l2-hires_g17_d20211219_v1-0-1-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("dn_magn-l2-hires_g17_d20211219_v1-0-1-created-from-netcdf-input.cdf") os.remove("dn_magn-l2-hires_g17_d20211219_v1-0-1.nc") @@ -112,7 +118,8 @@ def test_saber(): c["sclongitude"].attrs["VAR_TYPE"] = "support_data" c["scaltitude"].attrs["VAR_TYPE"] = "support_data" xarray_to_cdf(c, "SABER_L2B_2021020_103692_02.07-created-from-netcdf-input.cdf") - d = cdf_to_xarray("SABER_L2B_2021020_103692_02.07-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("SABER_L2B_2021020_103692_02.07-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("SABER_L2B_2021020_103692_02.07-created-from-netcdf-input.cdf") os.remove("SABER_L2B_2021020_103692_02.07.nc") @@ -124,9 +131,11 @@ def test_euv(): if not os.path.exists(fname): urllib.request.urlretrieve(url, fname) - a = cdf_to_xarray("mvn_euv_l3_minute_20201130_v14_r02.cdf", fillval_to_nan=True) + with CDF("mvn_euv_l3_minute_20201130_v14_r02.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, "mvn_euv_l3_minute_20201130_v14_r02-created-from-cdf-input.cdf") - b = cdf_to_xarray("mvn_euv_l3_minute_20201130_v14_r02-created-from-cdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_euv_l3_minute_20201130_v14_r02-created-from-cdf-input.cdf") as cdf_file: + b = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_euv_l3_minute_20201130_v14_r02-created-from-cdf-input.cdf") os.remove("mvn_euv_l3_minute_20201130_v14_r02.cdf") @@ -138,9 +147,11 @@ def test_sep_anc(): if not os.path.exists(fname): urllib.request.urlretrieve(url, fname) - a = cdf_to_xarray("mvn_sep_l2_anc_20210501_v06_r00.cdf", fillval_to_nan=True) + with CDF("mvn_sep_l2_anc_20210501_v06_r00.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, "mvn_sep_l2_anc_20210501_v06_r00-created-from-cdf-input.cdf") - a = cdf_to_xarray("mvn_sep_l2_anc_20210501_v06_r00-created-from-cdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_sep_l2_anc_20210501_v06_r00-created-from-cdf-input.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_sep_l2_anc_20210501_v06_r00-created-from-cdf-input.cdf") os.remove("mvn_sep_l2_anc_20210501_v06_r00.cdf") @@ -152,9 +163,11 @@ def test_sep_svy(): if not os.path.exists(fname): urllib.request.urlretrieve(url, fname) - a = cdf_to_xarray("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05.cdf", fillval_to_nan=True) + with CDF("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, "mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05-created-from-cdf-input.cdf") - b = cdf_to_xarray("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05-created-from-cdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05-created-from-cdf-input.cdf") as cdf_file: + b = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05-created-from-cdf-input.cdf") os.remove("mvn_sep_l2_s2-raw-svy-full_20191231_v04_r05.cdf") @@ -187,9 +200,11 @@ def test_swe_arc3d(): if not os.path.exists(fname): urllib.request.urlretrieve(url, fname) - a = cdf_to_xarray("mvn_swe_l2_arc3d_20180717_v04_r02.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_arc3d_20180717_v04_r02.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, "mvn_swe_l2_arc3d_20180717_v04_r02-created-from-cdf-input.cdf") - b = cdf_to_xarray("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-cdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-cdf-input.cdf") as cdf_file: + b = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-cdf-input.cdf") os.remove("mvn_swe_l2_arc3d_20180717_v04_r02.cdf") @@ -200,7 +215,8 @@ def test_swe_arc3d(): c = xr.load_dataset("mvn_swe_l2_arc3d_20180717_v04_r02.nc") xarray_to_cdf(c, "mvn_swe_l2_arc3d_20180717_v04_r02-created-from-netcdf-input.cdf") - d = cdf_to_xarray("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_swe_l2_arc3d_20180717_v04_r02-created-from-netcdf-input.cdf") os.remove("mvn_swe_l2_arc3d_20180717_v04_r02.nc") @@ -212,9 +228,11 @@ def test_swe_svyspec(): if not os.path.exists(fname): urllib.request.urlretrieve(url, fname) - a = cdf_to_xarray("mvn_swe_l2_svyspec_20180718_v04_r04.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_svyspec_20180718_v04_r04.cdf") as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, "mvn_swe_l2_svyspec_20180718_v04_r04-created-from-cdf-input.cdf") - b = cdf_to_xarray("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-cdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-cdf-input.cdf") as cdf_file: + b = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-cdf-input.cdf") os.remove("mvn_swe_l2_svyspec_20180718_v04_r04.cdf") @@ -225,7 +243,8 @@ def test_swe_svyspec(): c = xr.load_dataset("mvn_swe_l2_svyspec_20180718_v04_r04.nc") xarray_to_cdf(c, "mvn_swe_l2_svyspec_20180718_v04_r04-created-from-netcdf-input.cdf") - d = cdf_to_xarray("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("mvn_swe_l2_svyspec_20180718_v04_r04-created-from-netcdf-input.cdf") os.remove("mvn_swe_l2_svyspec_20180718_v04_r04.nc") @@ -239,7 +258,8 @@ def test_raids(): c = xr.load_dataset("raids_nirs_20100823_v1.1.nc") xarray_to_cdf(c, "raids_nirs_20100823_v1.1-created-from-netcdf-input.cdf") - d = cdf_to_xarray("raids_nirs_20100823_v1.1-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("raids_nirs_20100823_v1.1-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("raids_nirs_20100823_v1.1-created-from-netcdf-input.cdf") os.remove("raids_nirs_20100823_v1.1.nc") @@ -274,7 +294,8 @@ def test_see_l3(): c = xr.load_dataset("see__L3_2021009_012_01.ncdf") xarray_to_cdf(c, "see__L3_2021009_012_01.ncdfhello2.cdf") - d = cdf_to_xarray("see__L3_2021009_012_01.ncdfhello2.cdf", fillval_to_nan=True) + with CDF("see__L3_2021009_012_01.ncdfhello2.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("see__L3_2021009_012_01.ncdfhello2.cdf") os.remove("see__L3_2021009_012_01.ncdf") @@ -288,7 +309,8 @@ def test_see_l2a(): c = xr.load_dataset("see__xps_L2A_2021006_012_02.ncdf") xarray_to_cdf(c, "see__xps_L2A_2021006_012_02.ncdfhello2.cdf") - d = cdf_to_xarray("see__xps_L2A_2021006_012_02.ncdfhello2.cdf", fillval_to_nan=True) + with CDF("see__xps_L2A_2021006_012_02.ncdfhello2.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("see__xps_L2A_2021006_012_02.ncdfhello2.cdf") os.remove("see__xps_L2A_2021006_012_02.ncdf") @@ -302,7 +324,8 @@ def test_something(): c = xr.load_dataset("sgpsondewnpnC1.nc") xarray_to_cdf(c, "sgpsondewnpnC1-created-from-netcdf-input.cdf") - d = cdf_to_xarray("sgpsondewnpnC1-created-from-netcdf-input.cdf", fillval_to_nan=True) + with CDF("sgpsondewnpnC1-created-from-netcdf-input.cdf") as cdf_file: + d = cdf_to_xarray(cdf_file, fillval_to_nan=True) os.remove("sgpsondewnpnC1-created-from-netcdf-input.cdf") os.remove("sgpsondewnpnC1.nc") @@ -348,7 +371,8 @@ def test_build_from_scratch(): def test_smoke(cdf_path, tmp_path): - a = cdf_to_xarray(cdf_path, fillval_to_nan=True) + with CDF(cdf_path) as cdf_file: + a = cdf_to_xarray(cdf_file, fillval_to_nan=True) xarray_to_cdf(a, tmp_path / cdf_path.name) @@ -363,7 +387,8 @@ def test_datetime64_conversion(): epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf") - x = cdf_to_xarray("hello.cdf", to_datetime=True) + with CDF("hello.cdf") as cdf_file: + x = cdf_to_xarray(cdf_file, to_datetime=True) assert x["epoch"][0] == np.datetime64("1970-01-01T00:00:01") os.remove("hello.cdf") @@ -380,7 +405,8 @@ def test_datetime64_conversion_odd_units(): epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf") - x = cdf_to_xarray("hello.cdf", to_datetime=True) + with CDF("hello.cdf") as cdf_file: + x = cdf_to_xarray(cdf_file, to_datetime=True) assert x["epoch"][1] == np.datetime64("2000-01-02") os.remove("hello.cdf") @@ -397,6 +423,7 @@ def test_numpy_string_array(): epoch = xr.Variable(epoch_dims, epoch_data) ds = xr.Dataset(data_vars={"data": data, "epoch": epoch}) xarray_to_cdf(ds, "hello.cdf") - x = cdf_to_xarray("hello.cdf", to_datetime=True) + with CDF("hello.cdf") as cdf_file: + x = cdf_to_xarray(cdf_file, to_datetime=True) assert x["data"][2] == "c" os.remove("hello.cdf")