diff --git a/sea-level-rise-s3.ipynb b/sea-level-rise-s3.ipynb new file mode 100644 index 0000000..bcdc49f --- /dev/null +++ b/sea-level-rise-s3.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8f552380-cc0a-4fb8-b4b3-276a6564b239", + "metadata": {}, + "source": [ + "# Analyzing Sea Level Rise in the Cloud with Coiled and Earthaccess\n", + "\n", + "_This notebook was adapted from [this NASA Earthdata Cloud Cookbook example](https://nasa-openscapes.github.io/earthdata-cloud-cookbook/tutorials/Sea_Level_Rise.html)_" + ] + }, + { + "cell_type": "markdown", + "id": "4a30aadb-41d4-4c08-a6fa-5ea60c6bd695", + "metadata": {}, + "source": [ + "## Get data files with `earthaccess`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2d48b80-6e72-4ad0-a71d-915d96f1ce03", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "import earthaccess\n", + "\n", + "# Retrieve data files for the dataset I'm interested in\n", + "granules = earthaccess.search_data(\n", + " short_name=\"SEA_SURFACE_HEIGHT_ALT_GRIDS_L4_2SATS_5DAY_6THDEG_V_JPL2205\",\n", + " temporal=(\"2000\", \"2019\"),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "64df6beb-e1e9-4691-b6ec-18dfa6803f53", + "metadata": {}, + "source": [ + "## Define processing function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5cb0793-5273-4ede-9b8d-080a79ab8228", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import coiled\n", + "import fsspec\n", + "import xarray as xr\n", + "\n", + "@coiled.function(region=\"us-west-2\") # Same region as data\n", + "def process(granule, fs):\n", + " files = []\n", + " for file in granule.data_links(\"direct\"):\n", + " ds = xr.open_dataset(fs.open(file))\n", + " ds = ds.sel(Latitude=slice(23, 50), Longitude=slice(270, 330))\n", + " ds = ds.SLA.where((ds.SLA >= 0) & (ds.SLA < 10))\n", + " # Write processed data to S3\n", + " outfile = f\"s3://openscapes-scratch/agu-2023/{os.path.basename(file)}\"\n", + " with fsspec.open(f\"simplecache::{outfile}\", mode=\"wb\") as out:\n", + " ds.to_netcdf(out, engine=\"h5netcdf\")\n", + " files.append(outfile)\n", + " return files\n", + "\n", + "\n", + "@coiled.function(region=\"us-west-2\") # Same region as data\n", + "def plot(files):\n", + " fs = fsspec.filesystem(\"s3\")\n", + " ds = xr.open_mfdataset(\n", + " [fs.open(f, mode=\"rb\") for f in files],\n", + " concat_dim=\"Time\",\n", + " combine=\"nested\",\n", + " parallel=True,\n", + " )\n", + " return ds.SLA.std(\"Time\").plot(x=\"Longitude\", y=\"Latitude\", figsize=(14, 6))" + ] + }, + { + "cell_type": "markdown", + "id": "fd853bd7-244e-4fbb-9cee-5e7921d082b1", + "metadata": {}, + "source": [ + "## Process Granules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abde80ff-a03d-4bfd-975a-17fcda39ce90", + "metadata": {}, + "outputs": [], + "source": [ + "fs = earthaccess.get_s3fs_session(results=granules)" + ] + }, + { + "cell_type": "markdown", + "id": "86988643-b344-43c5-9ad7-51c17c24b4d7", + "metadata": {}, + "source": [ + "### Single file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b268d9e1-b056-444e-9b99-1171e2f5b075", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "files = process(granules[0], fs=fs)\n", + "plot(files);" + ] + }, + { + "cell_type": "markdown", + "id": "cbe32079-17f9-45a8-9601-c0fe21980183", + "metadata": {}, + "source": [ + "### All files in parallel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ea9c91a-7ab6-40aa-bf6c-a75bb2afdb04", + "metadata": {}, + "outputs": [], + "source": [ + "%%time\n", + "\n", + "files = process.map(granules, fs=fs)\n", + "files = sum(files, []) # Flatten list of files\n", + "plot(files);" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}