diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f967db..9f9078d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,11 +36,3 @@ repos: - types-python-dateutil - types-requests - "pydantic>=2.1" - - - repo: https://github.com/PyCQA/pydocstyle - rev: "6.3.0" - hooks: - - id: pydocstyle - # https://github.com/PyCQA/pydocstyle/pull/608#issuecomment-1381168417 - additional_dependencies: ['.[toml]'] - exclude: tests diff --git a/conda-env.yml b/conda-env.yml index 3aa5d7e..6c490f3 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -24,6 +24,7 @@ dependencies: # - shapely>=1.8 - isce3>=0.14.0 - compass>=0.4.1 + - backoff - dask - dolphin>=0.5.1 - gdal @@ -38,5 +39,5 @@ dependencies: - rich - rioxarray - sardem - - sentineleof + - sentineleof>=0.9.5 - shapely diff --git a/pyproject.toml b/pyproject.toml index a5f32a4..8240e4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,24 +45,36 @@ write_to = "src/sweets/_version.py" # https://github.com/pypa/setuptools_scm#version-number-construction version_scheme = "no-guess-dev" # Will not guess the next version +[tool.ruff] +ignore = [ + "D100", # Missing docstring in public module + "D104", # Missing docstring in public package + "D105", # Missing docstring in magic method + "D107", # Missing docstring in __init__ + "D203", # 1 blank line required before class docstring + "D212", # Multi-line docstring summary should start at the first line + "PLR", # Pylint Refactor +] + +[tool.ruff.lint] +# Enable the isort rules. +extend-select = ["I"] [tool.black] -target-version = ["py38", "py39", "py310", "py310"] +target-version = ["py39", "py310", "py311", "py312"] preview = true [tool.isort] profile = "black" -known_first_party = ["sweets"] +known_first_party = ["dolphin"] [tool.mypy] -python_version = "3.8" +python_version = "3.10" ignore_missing_imports = true plugins = ["pydantic.mypy"] -[tool.pydocstyle] -ignore = "D100,D102,D104,D105,D106,D107,D203,D204,D213,D413" - +[tool.ruff.per-file-ignores] +"**/__init__.py" = ["F401"] +"test/**" = ["D"] [tool.pytest.ini_options] -filterwarnings = [ - # "error", -] +filterwarnings = ["error"] diff --git a/requirements.txt b/requirements.txt index 11a0809..e5dc3e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,5 @@ pyproj>=3.2 requests>=2.10 rich>=12.0 sardem>=0.11.1 -sentineleof>=0.6.5 +sentineleof>=0.9.5 shapely>=1.8 diff --git a/scripts/prep_mintpy.py b/scripts/prep_mintpy.py index 3d23639..0ba41b5 100755 --- a/scripts/prep_mintpy.py +++ b/scripts/prep_mintpy.py @@ -18,10 +18,10 @@ import numpy as np import pyproj from dolphin import io -from dolphin.utils import full_suffix, get_dates +from dolphin.utils import full_suffix from mintpy.utils import arg_utils, ptime, readfile, writefile from mintpy.utils.utils0 import calc_azimuth_from_east_north_obs -from opera_utils import OPERA_DATASET_ROOT +from opera_utils import OPERA_DATASET_ROOT, get_dates #################################################################################### EXAMPLE = """example: diff --git a/src/sweets/_geocode_slcs.py b/src/sweets/_geocode_slcs.py index 4519c2a..eb8613e 100644 --- a/src/sweets/_geocode_slcs.py +++ b/src/sweets/_geocode_slcs.py @@ -101,6 +101,7 @@ def create_config_files( burst_db_file: Filename, dem_file: Filename, orbit_dir: Filename, + tec_dir: Optional[Filename] = None, bbox: Optional[Tuple[float, ...]] = None, x_posting: float = 5, y_posting: float = 10, @@ -108,6 +109,7 @@ def create_config_files( out_dir: Filename = Path("gslcs"), overwrite: bool = False, using_zipped: bool = False, + enable_corrections: bool = True, ) -> List[Path]: """Create the geocoding config files for a stack of SLCs. @@ -121,6 +123,9 @@ def create_config_files( File containing the DEM orbit_dir : Filename Directory containing the orbit files + tec_dir : Filename, optional + Directory containing the ionosphere TEC files. + If None, skips correction. bbox : Optional[Tuple[float, ...]], optional [lon_min, lat_min, lon_max, lat_max] to limit the processing. Note that this does not change each burst's bounding box, but @@ -139,6 +144,9 @@ def create_config_files( using_zipped : bool, optional If true, will search for. zip files instead of unzipped .SAFE directories. By default False. + enable_corrections : bool, default = True + If true, computes model-based corrections to geocoding. + See `compass` for more details on all corrections. Returns ------- @@ -167,5 +175,7 @@ def create_config_files( x_spac=x_posting, y_spac=y_posting, using_zipped=using_zipped, + enable_corrections=enable_corrections, + tec_dir=tec_dir, ) return sorted((Path(out_dir) / "runconfigs").glob("*")) diff --git a/src/sweets/_log.py b/src/sweets/_log.py index 387c00d..3da47e1 100644 --- a/src/sweets/_log.py +++ b/src/sweets/_log.py @@ -17,10 +17,8 @@ """ import logging -import time -from collections.abc import Callable -from functools import wraps +from dolphin._log import log_runtime from rich.console import Console from rich.logging import RichHandler @@ -81,34 +79,3 @@ def format_log(logger: logging.Logger, debug: bool = False) -> logging.Logger: logger.setLevel(debug) return logger - - -def log_runtime(f: Callable) -> Callable: - """Decorate a function to time how long it takes to run. - - Usage - ----- - @log_runtime - def test_func(): - return 2 + 4 - """ - logger = get_log(__name__) - - @wraps(f) - def wrapper(*args, **kwargs): - t1 = time.time() - - result = f(*args, **kwargs) - - t2 = time.time() - elapsed_seconds = t2 - t1 - elapsed_minutes = elapsed_seconds / 60.0 - time_string = ( - f"Total elapsed time for {f.__module__}.{f.__name__} : " - f"{elapsed_minutes:.2f} minutes ({elapsed_seconds:.2f} seconds)" - ) - - logger.info(time_string) - return result - - return wrapper diff --git a/src/sweets/core.py b/src/sweets/core.py index dba36df..8ca6051 100644 --- a/src/sweets/core.py +++ b/src/sweets/core.py @@ -7,18 +7,18 @@ import h5py import numpy as np -from dolphin import io, stitching, unwrap -from dolphin._dates import group_by_date +from dolphin import stitching, unwrap from dolphin.interferogram import Network -from dolphin.utils import set_num_threads +from dolphin.utils import _format_date_pair, set_num_threads from dolphin.workflows.config import YamlModel -from opera_utils import group_by_burst +from opera_utils import group_by_burst, group_by_date from pydantic import ConfigDict, Field, field_validator, model_validator from shapely import geometry, wkt from ._burst_db import get_burst_db from ._geocode_slcs import create_config_files, run_geocode, run_static_layers from ._geometry import stitch_geometry +from ._ionosphere import download_ionosphere from ._log import get_log, log_runtime from ._netrc import setup_nasa_netrc from ._orbit import download_orbits @@ -190,12 +190,13 @@ def load(cls, config_file: Filename = "sweets_config.yaml"): logger.info(f"Loading config from {config_file}") return cls.from_yaml(config_file) - # Override the constructor to allow recursively construct without validation @classmethod - def construct(cls, **kwargs): + def model_construct(cls, **kwargs): + """Override the constructor to allow recursively construct without validation.""" + if "asf_query" not in kwargs: kwargs["asf_query"] = ASFQuery._construct_empty() - return super().construct( + return super().model_construct( **kwargs, ) @@ -205,6 +206,7 @@ def __init__(self, **data: Any) -> None: self.log_dir = self.work_dir / "logs" self.gslc_dir = self.work_dir / "gslcs" self.geom_dir = self.work_dir / "geometry" + self.iono_dir = self.work_dir / "ionosphere" self.ifg_dir = self.work_dir / "interferograms" self.stitched_ifg_dir = self.ifg_dir / "stitched" self.unw_dir = self.ifg_dir / "unwrapped" @@ -294,6 +296,8 @@ def _geocode_slcs(self, slc_files, dem_file, burst_db_file): out_dir=self.gslc_dir, overwrite=self.overwrite, using_zipped=not self.asf_query.unzip, + # enable_corrections=False, + tec_dir=self.iono_dir, ) def cfg_to_filename(cfg_path: Path) -> str: @@ -351,6 +355,14 @@ def cfg_to_static_filename(cfg_path: Path) -> str: with ProcessPoolExecutor(max_workers=self.n_workers) as _client: new_files = _client.map(run_func, todo_static) + def _download_ionosphere(self, slc_files) -> list[Path]: + """Download one IONEX file per SLC.""" + self.iono_dir.mkdir(parents=True, exist_ok=True) + outputs = [] + for slc_file in slc_files: + outputs.append(download_ionosphere(slc_file, self.iono_dir)) + return outputs + @log_runtime def _stitch_geometry(self, geom_path_list): return stitch_geometry( @@ -425,7 +437,7 @@ def _stitch_interferograms(self, ifg_path_list): stitched_ifg_files = [] for dates, cur_images in grouped_images.items(): logger.info(f"{dates}: Stitching {len(cur_images)} images.") - outfile = self.stitched_ifg_dir / (io._format_date_pair(*dates) + ".int") + outfile = self.stitched_ifg_dir / (_format_date_pair(*dates) + ".int") stitched_ifg_files.append(outfile) stitching.merge_images( @@ -517,6 +529,7 @@ def run(self, starting_step: int = 1): burst_db_file = get_burst_db() download_orbits(self.asf_query.out_dir, self.orbit_dir) rslc_files = self._get_existing_rslcs() + self._download_ionosphere(rslc_files) self._geocode_slcs(rslc_files, self._dem_filename, burst_db_file) geom_path_list = self._get_burst_static_layers() diff --git a/src/sweets/interferogram.py b/src/sweets/interferogram.py index 3ab24a3..594f232 100644 --- a/src/sweets/interferogram.py +++ b/src/sweets/interferogram.py @@ -13,9 +13,8 @@ import rioxarray from compass.utils.helpers import bbox_to_utm from dask.distributed import Client -from dolphin import utils from dolphin.io import DEFAULT_HDF5_OPTIONS, get_raster_xysize, load_gdal, write_arr -from opera_utils import OPERA_DATASET_NAME +from opera_utils import OPERA_DATASET_NAME, get_dates from pydantic import BaseModel, Field, model_validator from rich.progress import track @@ -229,8 +228,8 @@ def _form_ifg_name(slc1: Filename, slc2: Filename, out_dir: Filename) -> Path: Path to the interferogram file. """ - date1 = utils.get_dates(slc1)[0] - date2 = utils.get_dates(slc2)[0] + date1 = get_dates(slc1)[0] + date2 = get_dates(slc2)[0] fmt = "%Y%m%d" ifg_name = f"{date1.strftime(fmt)}_{date2.strftime(fmt)}.h5" return Path(out_dir) / ifg_name