From cf13adc54445057bec5a11b57af8204a101d502c Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Fri, 6 Mar 2020 13:29:21 +0100 Subject: [PATCH 1/6] debug --- main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 55a63af..400e8ed 100644 --- a/main.py +++ b/main.py @@ -17,14 +17,14 @@ # You can also change the variables namely default_XXXXXX in the Utilities.py file # In this case, you do not need to set these variables when you call functions in Utilities.py # Set environment -ut.gdal_osgeo_dir = r'C:\Users\ChengY\AppData\Roaming\Python\Python37\site-packages\osgeo' +ut.gdal_osgeo_dir = r'C:\Python37\Anaconda3\Lib\site-packages\osgeo' # Set work directory -ut.work_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD' +ut.work_dir = r'D:\IMAGES\PlanetScope\Kapiti-Kenya' # Set folders for saving different outputs ut.output_dirs = {'raw': 'raw', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge', 'clear prob': 'clear_prob', 'NDVI': 'NDVI', 'clip clear perc': 'bomas'} -ut.api_key = "9cada8bc134546fe9c1b8bce5b71860f" +ut.api_key = "e912e7e40d4d4c92bd6cbdffef89b76c" #"9cada8bc134546fe9c1b8bce5b71860f" ut.satellite = 'PS' ut.proj_code = 32737 ut.dpi = 90 @@ -32,17 +32,16 @@ # Filter settings ut.filter_items = ['date', 'cloud_cover', 'aoi'] ut.item_types = ["PSScene4Band"] -ut.process_level = '3B' ut.asset_types = ['analytic_sr', 'udm2'] # Set filter ut.start_date = '2020-02-05' ut.end_date = '2020-02-11' ut.cloud_cover = 1 -ut.aoi_shp = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\shp\Kapiti\Kapiti_Jun18_v2_prj.shp' +ut.aoi_shp = r'D:\DATA\Kapiti\Kapiti_Jun18_v2_prj.shp' # Settings for raster visualization ut.rgb_composition = {'red': 4, 'green': 3, 'blue': 2} # False color composition for PlanetScope images ut.percentile = [2, 98] -ut.remove_latest = True # Set as True only when you killed the previous run and want to rerun it. In this case, +ut.remove_latest = False # Set as True only when you killed the previous run and want to rerun it. In this case, # the lasted file will be removed in case it is not a complete file... From 22289eb672c8cfa71229bb62453c22221b30f6de Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Sun, 15 Mar 2020 19:30:27 +0100 Subject: [PATCH 2/6] debug --- Utilities.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Utilities.py b/Utilities.py index 52bbf1c..007984d 100644 --- a/Utilities.py +++ b/Utilities.py @@ -859,13 +859,13 @@ def band_algebra(self, output_type, file_list=None): # Check existing clipped images and remove the latest file, in case it is not complete file_list_exist = glob('{}\\*.tif'.format(clear_prob_dir)) if file_list_exist: - item_id_list_exist = [file.split('\\')[0].split('_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] + item_id_list_exist = [file.split('\\')[-1].split('_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] for file in file_list_exist] if self.remove_latest is True: latest_file = max(file_list_exist, key=os.path.getctime) os.remove(latest_file) file_list_exist.remove(latest_file) - file_list = [file for file in file_list if file.split('\\')[0].split( + file_list = [input_dir + file for file in file_list if file.split('\\')[-1].split( '_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] not in item_id_list_exist] for udm2_path in file_list: @@ -957,13 +957,13 @@ def clip_clear_perc(self, shapefile_path, clear_perc_min, save_rgb=True, save_cl # if save_clip is True: # file_list_exist = glob('{}\\*.tif'.format(output_dir)) # if file_list_exist: - # item_id_list_exist = [file.split('\\')[0].split('_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] + # item_id_list_exist = [file.split('\\')[-1].split('_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] # for file in file_list_exist] # if self.remove_latest is True: # latest_file = max(file_list_exist, key=os.path.getctime) # os.remove(latest_file) # file_list_exist.remove(latest_file) - # file_list = [file for file in file_list if file.split('\\')[0].split( + # file_list = [input_dir + file for file in file_list if file.split('\\')[-1].split( # '_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] not in item_id_list_exist] asset_id_list = [] From 2ee47313b8938202aeb1a7138fd8a222f7a62a4c Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Sun, 15 Mar 2020 19:40:48 +0100 Subject: [PATCH 3/6] debug --- Utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities.py b/Utilities.py index 007984d..978541e 100644 --- a/Utilities.py +++ b/Utilities.py @@ -972,7 +972,7 @@ def clip_clear_perc(self, shapefile_path, clear_perc_min, save_rgb=True, save_cl asset_id = asset_name.split('_udm2')[0] # Clip udm2 to the extent of bomas AOI.shp and save in memory vsimem_path = '/vsimem/' + asset_name + '.tif' - print(vsimem_path) + # print(vsimem_path) self.gdal_clip(file, pixel_res, shapefile_path, vsimem_path, data_type='Byte') raster = gdal.Open(vsimem_path) # Convert the first band (clear) to numpy array From a853528ab8b4d01dd18bb6d7108024e5feaebd5c Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Mon, 17 Aug 2020 08:51:25 +0200 Subject: [PATCH 4/6] debug merge images from the same satellite --- Utilities.py | 12 ++++++++---- main.py | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Utilities.py b/Utilities.py index 978541e..26d702b 100644 --- a/Utilities.py +++ b/Utilities.py @@ -698,13 +698,17 @@ def merge(self, file_list=None): file_list_new = glob("{}\\{}*{}_setnull.tif" .format(input_dir, date, self.asset_attrs(asset_type)['suffix'])) else: - file_list_new = glob("{}\\{}*{}.tif".format(input_dir, date, self.asset_attrs(asset_type)['suffix'])) - input_path = ' '.join(str(i) for i in file_list_new) + file_list_new = glob("{}\\{}*{}.tif".format(input_dir, date, + self.asset_attrs(asset_type)['suffix'])) + # input_path = ' '.join(str(i) for i in file_list_new) satellite_id_list = list(set([x.split('\\')[-1].split('_{}_'.format(self.process_level))[0] .split('_')[-1] for x in file_list_new])) for satellite_id in satellite_id_list: - output_path = output_dir + '\\' + date + '_' + \ - satellite_id + '_{}.tif'.format(self.asset_attrs(asset_type)['suffix']) + input_file_list = glob("{}\\{}*{}_{}*.tif".format(input_dir, date, satellite_id, + self.process_level)) + input_path = ' '.join(str(i) for i in input_file_list) + output_path = output_dir + '\\' + date + '_' + satellite_id + '_{}.tif'.format( + self.asset_attrs(asset_type)['suffix']) self.gdal_merge(input_path, output_path, self.asset_attrs(asset_type)['data type']) time_str = datetime.now().strftime("%Y%m%d-%H%M%S") diff --git a/main.py b/main.py index 400e8ed..463c7eb 100644 --- a/main.py +++ b/main.py @@ -20,9 +20,9 @@ ut.gdal_osgeo_dir = r'C:\Python37\Anaconda3\Lib\site-packages\osgeo' # Set work directory -ut.work_dir = r'D:\IMAGES\PlanetScope\Kapiti-Kenya' +ut.work_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD' # Set folders for saving different outputs -ut.output_dirs = {'raw': 'raw', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge', +ut.output_dirs = {'raw': 'raw', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge_test', 'clear prob': 'clear_prob', 'NDVI': 'NDVI', 'clip clear perc': 'bomas'} ut.api_key = "e912e7e40d4d4c92bd6cbdffef89b76c" #"9cada8bc134546fe9c1b8bce5b71860f" ut.satellite = 'PS' @@ -35,9 +35,9 @@ ut.asset_types = ['analytic_sr', 'udm2'] # Set filter ut.start_date = '2020-02-05' -ut.end_date = '2020-02-11' +ut.end_date = '2020-02-6' ut.cloud_cover = 1 -ut.aoi_shp = r'D:\DATA\Kapiti\Kapiti_Jun18_v2_prj.shp' +ut.aoi_shp = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\shp\Kapiti\Kapiti_Jun18_v2_prj.shp' # Settings for raster visualization ut.rgb_composition = {'red': 4, 'green': 3, 'blue': 2} # False color composition for PlanetScope images ut.percentile = [2, 98] @@ -73,12 +73,12 @@ # # # # =================================== Merge ======================================# -# # Set input directory that includes all data to be merged -# # the default directory is [..\raw], which is the automatically created folder for saving all downloaded images -# input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw' -# file_list = glob("{}\\*udm2.tif".format(input_dir)) # only for udm2 -# # file_list = glob("{}\\*.tif".format(input_dir)) # for all tif -# ut.merge(file_list=file_list) +# Set input directory that includes all data to be merged +# the default directory is [..\raw], which is the automatically created folder for saving all downloaded images +input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw' +file_list = glob("{}\\*udm2.tif".format(input_dir)) # only for udm2 +# file_list = glob("{}\\*.tif".format(input_dir)) # for all tif +ut.merge(file_list=file_list) # # # # =================================== Clip ======================================# From eafe431d75f7855eb5d7db6f548dfd879d9947a2 Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Mon, 17 Aug 2020 09:13:12 +0200 Subject: [PATCH 5/6] debug merge images from the same satellite --- Utilities.py | 8 +++++--- main.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Utilities.py b/Utilities.py index 26d702b..3ba1df5 100644 --- a/Utilities.py +++ b/Utilities.py @@ -700,13 +700,15 @@ def merge(self, file_list=None): else: file_list_new = glob("{}\\{}*{}.tif".format(input_dir, date, self.asset_attrs(asset_type)['suffix'])) - # input_path = ' '.join(str(i) for i in file_list_new) + satellite_id_list = list(set([x.split('\\')[-1].split('_{}_'.format(self.process_level))[0] .split('_')[-1] for x in file_list_new])) + for satellite_id in satellite_id_list: - input_file_list = glob("{}\\{}*{}_{}*.tif".format(input_dir, date, satellite_id, - self.process_level)) + input_file_list = [file_path for file_path in file_list_new + if '_{}_{}_'.format(satellite_id, self.process_level) in file_path] input_path = ' '.join(str(i) for i in input_file_list) + # print(input_path) output_path = output_dir + '\\' + date + '_' + satellite_id + '_{}.tif'.format( self.asset_attrs(asset_type)['suffix']) self.gdal_merge(input_path, output_path, self.asset_attrs(asset_type)['data type']) diff --git a/main.py b/main.py index 463c7eb..a3a3f21 100644 --- a/main.py +++ b/main.py @@ -76,7 +76,7 @@ # Set input directory that includes all data to be merged # the default directory is [..\raw], which is the automatically created folder for saving all downloaded images input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw' -file_list = glob("{}\\*udm2.tif".format(input_dir)) # only for udm2 +file_list = glob("{}\\*AnalyticMS_SR.tif".format(input_dir)) # only for udm2 # file_list = glob("{}\\*.tif".format(input_dir)) # for all tif ut.merge(file_list=file_list) # From a9ef9e37d0fdaa6f909e53f9e9a4c2d77b10d5f9 Mon Sep 17 00:00:00 2001 From: YanCheng-go <230409111cY> Date: Wed, 2 Sep 2020 00:14:04 +0200 Subject: [PATCH 6/6] updates for processing online clipped files --- Utilities.py | 43 +++++++++++++++++++++++++++---------------- main.py | 10 +++++----- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/Utilities.py b/Utilities.py index 3ba1df5..394f797 100644 --- a/Utilities.py +++ b/Utilities.py @@ -183,6 +183,9 @@ def __init__(self, gdal_osgeo_dir=default_gdal_osgeo_dir, work_dir=default_work_ self.records_path = None # File path of execution track document self.id_list_download = None # a list of item id which will be downloaded + self.analytic_sr_name = [x for x in self.asset_types if 'analytic_sr' in x] + self.udm2_name = [x for x in self.asset_types if 'udm2' in x] + def shp_to_json(self): ''' Convert AOI shapefile to json format that is required for retrieve imagery for specific location @@ -218,7 +221,15 @@ def asset_attrs(asset_type): 'udm2': { 'suffix': 'udm2', 'data type': 'Byte' - } + }, + 'analytic_sr_clip': { + 'suffix': 'AnalyticMS_SR_clip', + 'data type': 'UInt16' + }, + 'udm2_clip': { + 'suffix': 'udm2_clip', + 'data type': 'Byte' + } # clipped ones need to be downloaded from the plant explorer manually } return switch.get(asset_type, 'None') @@ -603,7 +614,7 @@ def udm2_setnull(self, file_list=None): if file_list is None: file_list = [] for i in self.id_list_download: - a = glob("{}\\{}*udm2.tif".format(input_dir, i)) + a = glob("{}\\{}*{}.tif".format(input_dir, i, self.udm2_name[0])) for j in a: file_list.append(j) else: @@ -615,7 +626,7 @@ def udm2_setnull(self, file_list=None): for file in file_list])) exist_setnull = list( set([file.split('\\')[-1].split('_{}_'.format(self.process_level))[0] - for file in file_list if 'setnull' in file])) + for file in file_list if 'setnull' in file])) # when the user defined file list does not contain _setnull file, this existing checking does not work... new_setnull = [i for i in item_id_list if i not in exist_setnull] if new_setnull: if exist_setnull: @@ -694,7 +705,7 @@ def merge(self, file_list=None): date_list = [date for date in date_list if date not in date_list_exist] for date in tqdm(date_list, total=len(date_list), unit="item", desc='Merging images'): - if asset_type == 'udm2': + if 'udm2' in asset_type: file_list_new = glob("{}\\{}*{}_setnull.tif" .format(input_dir, date, self.asset_attrs(asset_type)['suffix'])) else: @@ -798,10 +809,10 @@ def clip(self, file_list=None): for input_path in tqdm(file_list, total=len(file_list), unit="item", desc='Clipping images'): output_name = input_path.split('\\')[-1] output_path = output_dir + '\\' + output_name - if self.asset_attrs('udm2')['suffix'] in input_path: - data_type = self.asset_attrs('udm2')['data type'] - if self.asset_attrs('analytic_sr')['suffix'] in input_path: - data_type = self.asset_attrs('analytic_sr')['data type'] + if self.asset_attrs(self.udm2_name[0])['suffix'] in input_path: + data_type = self.asset_attrs(self.udm2_name[0])['data type'] + if self.asset_attrs(self.analytic_sr_name[0])['suffix'] in input_path: + data_type = self.asset_attrs(self.analytic_sr_name[0])['data type'] self.gdal_clip(input_path, self.pixel_res(self.satellite), self.aoi_shp, output_path, data_type) time_str = datetime.now().strftime("%Y%m%d-%H%M%S") @@ -851,16 +862,16 @@ def band_algebra(self, output_type, file_list=None): input_dir = self.work_dir + '\\' + self.output_dirs['clip'] if output_type == 'clear prob': - asset_type = 'udm2' + asset_type = self.udm2_name[0] clear_prob_dir = self.work_dir + '\\' + self.output_dirs['clear prob'] if file_list is None: file_list = [] for i in self.id_list_download: - a = glob("{}\\{}*{}.tif".format(input_dir, i, self.asset_attrs('udm2')['suffix'])) + a = glob("{}\\{}*{}.tif".format(input_dir, i, self.asset_attrs(asset_type)['suffix'])) for j in a: file_list.append(j) else: - file_list = [file for file in file_list if self.asset_attrs('udm2')['suffix'] in file] + file_list = [file for file in file_list if self.asset_attrs(asset_type)['suffix'] in file] # Check existing clipped images and remove the latest file, in case it is not complete file_list_exist = glob('{}\\*.tif'.format(clear_prob_dir)) @@ -875,7 +886,7 @@ def band_algebra(self, output_type, file_list=None): '_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] not in item_id_list_exist] for udm2_path in file_list: - clear_prob_path = clear_prob_dir + '\\' + udm2_path.split('\\')[-1].split('_{}'.format(self.asset_attrs('udm2')['suffix']))[0] + '_clearprob.tif' + clear_prob_path = clear_prob_dir + '\\' + udm2_path.split('\\')[-1].split('_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] + '_clearprob.tif' self.gdal_calc_clear_prob(input_path=udm2_path, output_path=clear_prob_path) print('Finish GDAL Calculation :)') print('The outputs have been saved in this directory: ' + clear_prob_dir) @@ -885,14 +896,14 @@ def band_algebra(self, output_type, file_list=None): if file_list is None: file_list = [] for i in self.id_list_download: - a = glob("{}\\{}*SR.tif".format(input_dir, i)) + a = glob("{}\\{}*_{}.tif".format(input_dir, i, self.asset_attrs(self.analytic_sr_name[0])['suffix'])) for j in a: file_list.append(j) else: - file_list = [file for file in file_list if 'SR' in file] + file_list = [file for file in file_list if self.asset_attrs(self.analytic_sr_name[0])['suffix'] in file] for sr_path in file_list: ndvi_path = ndvi_dir + '\\' + sr_path.split('\\')[-1].split( - '_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] + '_ndvi.tif' + '_{}'.format(self.asset_attrs(self.analytic_sr_name[0])['suffix']))[0] + '_ndvi.tif' self.gdal_calc_ndvi(input_path=sr_path, output_path=ndvi_path) time_str = datetime.now().strftime("%Y%m%d-%H%M%S") print('Finish GDAL Calculation :)') @@ -955,7 +966,7 @@ def clip_clear_perc(self, shapefile_path, clear_perc_min, save_rgb=True, save_cl shp_name = shapefile_path.split('\\')[-1].split('.shp')[0] # List merged and clipped udm2 file path if file_list is None: - file_list = glob('{}\\{}\\*udm2.tif'.format(self.work_dir, self.output_dirs['clip'])) + file_list = glob('{}\\{}\\*_{}.tif'.format(self.work_dir, self.output_dirs['clip'], self.udm2_name[0])) else: file_list = [file for file in file_list if 'udm2' in file] diff --git a/main.py b/main.py index a3a3f21..0886b36 100644 --- a/main.py +++ b/main.py @@ -22,7 +22,7 @@ # Set work directory ut.work_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD' # Set folders for saving different outputs -ut.output_dirs = {'raw': 'raw', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge_test', +ut.output_dirs = {'raw': 'raw_clip_test', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge_clip_test', 'clear prob': 'clear_prob', 'NDVI': 'NDVI', 'clip clear perc': 'bomas'} ut.api_key = "e912e7e40d4d4c92bd6cbdffef89b76c" #"9cada8bc134546fe9c1b8bce5b71860f" ut.satellite = 'PS' @@ -32,7 +32,7 @@ # Filter settings ut.filter_items = ['date', 'cloud_cover', 'aoi'] ut.item_types = ["PSScene4Band"] -ut.asset_types = ['analytic_sr', 'udm2'] +ut.asset_types = ['analytic_sr_clip', 'udm2_clip'] # download() cannot be used for on-the-fly clipped assets (with asset_type named as ['analytic_sr_clip', 'udm2_clip']) # Set filter ut.start_date = '2020-02-05' ut.end_date = '2020-02-6' @@ -75,9 +75,9 @@ # # =================================== Merge ======================================# # Set input directory that includes all data to be merged # the default directory is [..\raw], which is the automatically created folder for saving all downloaded images -input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw' -file_list = glob("{}\\*AnalyticMS_SR.tif".format(input_dir)) # only for udm2 -# file_list = glob("{}\\*.tif".format(input_dir)) # for all tif +input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw_online' +file_list = glob("{}\\*udm2_clip*.tif".format(input_dir)) #### The * after udm2_clip can avoid repeating set_null process if it has been down +# only for udm2_clip # for checking existing setnull file, better to write as "udm2_clip*" instead of udm2_clip... I mean include set_null file as well ut.merge(file_list=file_list) # #