From c6d00da591ca3c7117dc4c65c6ee7a6542ae4715 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Mon, 12 Aug 2024 21:22:45 -0800 Subject: [PATCH 01/81] New workflow for multi bursts --- environment.yml | 1 + pyproject.toml | 2 + src/hyp3_isce2/__main__.py | 2 +- src/hyp3_isce2/insar_tops.py | 9 +- src/hyp3_isce2/insar_tops_multi_bursts.py | 485 ++++++++++++++++++++++ src/hyp3_isce2/slc.py | 17 +- 6 files changed, 512 insertions(+), 4 deletions(-) create mode 100644 src/hyp3_isce2/insar_tops_multi_bursts.py diff --git a/environment.yml b/environment.yml index 46ba920b..eaff6208 100644 --- a/environment.yml +++ b/environment.yml @@ -29,3 +29,4 @@ dependencies: # For running - hyp3lib>=3,<4 - s1_orbits + - burst2safe diff --git a/pyproject.toml b/pyproject.toml index 7533f21e..8d7002b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,12 +55,14 @@ insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" +insar_tops_multi_bursts = "hyp3_isce2.insar_tops_multi_bursts:main" [project.entry-points.hyp3] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" +insar_tops_multi_bursts = "hyp3_isce2.insar_tops_multi_bursts:main" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/hyp3_isce2/__main__.py b/src/hyp3_isce2/__main__.py index 9d50385e..2fa21782 100644 --- a/src/hyp3_isce2/__main__.py +++ b/src/hyp3_isce2/__main__.py @@ -19,7 +19,7 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '++process', - choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts'], + choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts', 'insar_tops_multi_bursts'], default='insar_tops_burst', help='Select the HyP3 entrypoint to use', # HyP3 entrypoints are specified in `pyproject.toml` ) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 4feb81dd..f2bce38e 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -26,6 +26,7 @@ def insar_tops( polarization: str = 'VV', azimuth_looks: int = 4, range_looks: int = 20, + download: bool = True, ) -> Path: """Create a full-SLC interferogram @@ -44,8 +45,12 @@ def insar_tops( aux_cal_dir = Path('aux_cal') dem_dir = Path('dem') - ref_dir = slc.get_granule(reference_scene) - sec_dir = slc.get_granule(secondary_scene) + if download: + ref_dir = slc.get_granule(reference_scene) + sec_dir = slc.get_granule(secondary_scene) + else: + ref_dir = Path(reference_scene+'.SAFE') + sec_dir = Path(secondary_scene+'.SAFE') roi = slc.get_dem_bounds(ref_dir, sec_dir) log.info(f'DEM ROI: {roi}') diff --git a/src/hyp3_isce2/insar_tops_multi_bursts.py b/src/hyp3_isce2/insar_tops_multi_bursts.py new file mode 100644 index 00000000..d1c729f2 --- /dev/null +++ b/src/hyp3_isce2/insar_tops_multi_bursts.py @@ -0,0 +1,485 @@ +"""A workflow for merging standard burst InSAR products.""" +import argparse +import copy +import datetime +import logging +import os +import shutil +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from datetime import datetime, timezone +from itertools import combinations +from lxml import etree +from pathlib import Path +from secrets import token_hex +from shutil import make_archive +from tempfile import TemporaryDirectory +from typing import Iterable, List, Optional, Tuple + +import asf_search +import isce +import isceobj +import lxml.etree as ET +import numpy as np +from burst2safe import utils +from burst2safe.burst2safe import burst2safe +from burst2safe.safe import Safe +from burst2safe.search import download_bursts, find_bursts +from contrib.Snaphu.Snaphu import Snaphu +from hyp3lib.aws import upload_file_to_s3 +from hyp3lib.image import create_thumbnail +from hyp3lib.util import string_is_true +from isceobj.Orbit.Orbit import Orbit +from isceobj.Planet.Planet import Planet +from isceobj.Sensor.TOPS.Sentinel1 import Sentinel1 +from isceobj.TopsProc.runIon import maskUnwrap +from isceobj.TopsProc.runMergeBursts import mergeBox, mergeBursts2 +from iscesys.Component import createTraitSeq +from iscesys.Component.ProductManager import ProductManager +from mroipac.filter.Filter import Filter +from mroipac.icu.Icu import Icu +from osgeo import gdal, gdalconst +from shapely import geometry +from stdproc.rectify.geocode.Geocodable import Geocodable +from zerodop.geozero import createGeozero + +import hyp3_isce2 +import hyp3_isce2.burst as burst_utils +from hyp3_isce2.insar_tops_burst import find_product, get_pixel_size, convert_raster_from_isce2_gdal +from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.insar_tops import insar_tops +from hyp3_isce2.utils import ( + ParameterFile, + create_image, + image_math, + load_product, + make_browse_image, + read_product_metadata, + resample_to_radar_io, + utm_from_lon_lat, +) +from hyp3_isce2.water_mask import create_water_mask + + +log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +logging.basicConfig(stream=sys.stdout, format=log_format, level=logging.INFO, force=True) +log = logging.getLogger(__name__) + + +@dataclass +class ISCE2Dataset: + name: str + suffix: str + band: Iterable[int] + dtype: Optional[int] = gdalconst.GDT_Float32 + + +def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: int) -> str: + """Get the name of the interferogram product. + + Args: + reference_scene: The reference burst name. + secondary_scene: The secondary burst name. + pixel_spacing: The spacing of the pixels in the output image. + + Returns: + The name of the interferogram product. + """ + + reference_split = reference_scene.split('_') + secondary_split = secondary_scene.split('_') + + platform = reference_split[0][0:2] + reference_date = reference_split[5][0:8] + secondary_date = secondary_split[5][0:8] + product_type = 'INT' + pixel_spacing = str(int(pixel_spacing)) + product_id = token_hex(2).upper() + + return '_'.join( + [ + platform, + reference_date, + secondary_date, + product_type + pixel_spacing, + product_id, + ] + ) + + +def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False, use_multilooked=False) -> None: + """Translate ISCE outputs to a standard GTiff format with a UTM projection. + Assume you are in the top level of an ISCE run directory + + Args: + product_name: Name of the product + pixel_size: Pixel size + include_radar: Flag to include the full resolution radar geometry products in the output + """ + + src_ds = gdal.Open('merged/filt_topophase.unw.geo') + src_geotransform = src_ds.GetGeoTransform() + src_projection = src_ds.GetProjection() + + target_ds = gdal.Open('merged/dem.crop', gdal.GA_Update) + target_ds.SetGeoTransform(src_geotransform) + target_ds.SetProjection(src_projection) + + del src_ds, target_ds + + datasets = [ + ISCE2Dataset('merged/filt_topophase.unw.geo', 'unw_phase', [2]), + ISCE2Dataset('merged/phsig.cor.geo', 'corr', [1]), + ISCE2Dataset('merged/dem.crop', 'dem', [1]), + ISCE2Dataset('merged/filt_topophase.unw.conncomp.geo', 'conncomp', [1]), + ] + + if use_multilooked: + suffix = '.multilooked' + + rdr_datasets = [ + ISCE2Dataset( + find_product(f'merged/filt_topophase.flat.vrt'), + 'wrapped_phase_rdr', + [1], + gdalconst.GDT_CFloat32, + ), + ISCE2Dataset(find_product(f'merged/lat.rdr.full.vrt'), 'lat_rdr', [1]), + ISCE2Dataset(find_product(f'merged/lon.rdr.full.vrt'), 'lon_rdr', [1]), + ISCE2Dataset(find_product(f'merged/los.rdr.full.vrt'), 'los_rdr', [1, 2]), + ] + if include_radar: + datasets += rdr_datasets + + for dataset in datasets: + out_file = str(Path(product_name) / f'{product_name}_{dataset.suffix}.tif') + gdal.Translate( + destName=out_file, + srcDS=dataset.name, + bandList=dataset.band, + format='GTiff', + outputType=dataset.dtype, + noData=0, + creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], + ) + + # Use numpy.angle to extract the phase component of the complex wrapped interferogram + wrapped_phase = ISCE2Dataset('filt_topophase.flat.geo', 'wrapped_phase', 1) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{wrapped_phase.suffix}.tif ' + f'-A merged/{wrapped_phase.name} --A_band={wrapped_phase.band} ' + '--calc angle(A) --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + ds = gdal.Open('merged/los.rdr.geo', gdal.GA_Update) + ds.GetRasterBand(1).SetNoDataValue(0) + ds.GetRasterBand(2).SetNoDataValue(0) + del ds + + # Performs the inverse of the operation performed by MintPy: + # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L732-L737 + # First subtract the incidence angle from ninety degrees to go from sensor-to-ground to ground-to-sensor, + # then convert to radians + incidence_angle = ISCE2Dataset('los.rdr.geo', 'lv_theta', 1) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{incidence_angle.suffix}.tif ' + f'-A merged/{incidence_angle.name} --A_band={incidence_angle.band} ' + '--calc (90-A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + # Performs the inverse of the operation performed by MintPy: + # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L739-L745 + # First add ninety degrees to the azimuth angle to go from angle-from-east to angle-from-north, + # then convert to radians + azimuth_angle = ISCE2Dataset('los.rdr.geo', 'lv_phi', 2) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{azimuth_angle.suffix}.tif ' + f'-A merged/{azimuth_angle.name} --A_band={azimuth_angle.band} ' + '--calc (90+A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + ds = gdal.Open('merged/filt_topophase.unw.geo') + geotransform = ds.GetGeoTransform() + del ds + + epsg = utm_from_lon_lat(geotransform[0], geotransform[3]) + files = [str(path) for path in Path(product_name).glob('*.tif') if not path.name.endswith('rdr.tif')] + for file in files: + gdal.Warp( + file, + file, + dstSRS=f'epsg:{epsg}', + creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], + xRes=pixel_size, + yRes=pixel_size, + targetAlignedPixels=True, + ) + + +def make_parameter_file( + out_path: Path, + reference_scene: str, + secondary_scene: str, + azimuth_looks: int, + range_looks: int, + apply_water_mask: bool, + dem_name: str = 'GLO_30', + dem_resolution: int = 30, +) -> None: + """Create a parameter file for the output product + + Args: + out_path: path to output the parameter file + reference_scene: Reference burst name + secondary_scene: Secondary burst name + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + dem_name: Name of the DEM that is use + dem_resolution: Resolution of the DEM + + returns: + None + """ + SPEED_OF_LIGHT = 299792458.0 + SPACECRAFT_HEIGHT = 693000.0 + EARTH_RADIUS = 6337286.638938101 + + parser = etree.XMLParser(encoding='utf-8', recover=True) + + ref_tag = reference_scene[-4::] + sec_tag = secondary_scene[-4::] + print(ref_tag, sec_tag) + reference_safe = [file for file in os.listdir('.') if file.endswith(f'{ref_tag}.SAFE')][0] + secondary_safe = [file for file in os.listdir('.') if file.endswith(f'{sec_tag}.SAFE')][0] + + ref_annotation_path = f'{reference_safe}/annotation/' + ref_annotation = [file for file in os.listdir(ref_annotation_path) if os.path.isfile(ref_annotation_path + file)][0] + + ref_manifest_xml = etree.parse(f'{reference_safe}/manifest.safe', parser) + sec_manifest_xml = etree.parse(f'{secondary_safe}/manifest.safe', parser) + ref_annotation_xml = etree.parse(f'{ref_annotation_path}{ref_annotation}', parser) + topsProc_xml = etree.parse('topsProc.xml', parser) + topsApp_xml = etree.parse('topsApp.xml', parser) + + safe = '{http://www.esa.int/safe/sentinel-1.0}' + s1 = '{http://www.esa.int/safe/sentinel-1.0/sentinel-1}' + metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' + orbit_number_query = metadata_path + safe + 'orbitNumber' + orbit_direction_query = metadata_path + safe + 'extension//' + s1 + 'pass' + + ref_orbit_number = ref_manifest_xml.find(orbit_number_query).text + ref_orbit_direction = ref_manifest_xml.find(orbit_direction_query).text + sec_orbit_number = sec_manifest_xml.find(orbit_number_query).text + sec_orbit_direction = sec_manifest_xml.find(orbit_direction_query).text + ref_heading = float(ref_annotation_xml.find('.//platformHeading').text) + ref_time = ref_annotation_xml.find('.//productFirstLineUtcTime').text + slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) + range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) + number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) + baseline_perp = topsProc_xml.find(f'.//IW-2_Bperp_at_midrange_for_first_common_burst').text + unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text + phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text + + slant_range_near = float(slant_range_time) * SPEED_OF_LIGHT / 2 + range_pixel_spacing = SPEED_OF_LIGHT / (2 * range_sampling_rate) + slant_range_far = slant_range_near + (number_samples - 1) * range_pixel_spacing + slant_range_center = (slant_range_near + slant_range_far) / 2 + + s = ref_time.split('T')[1].split(':') + utc_time = ((int(s[0]) * 60 + int(s[1])) * 60) + float(s[2]) + + parameter_file = ParameterFile( + reference_granule=reference_scene, + secondary_granule=secondary_scene, + reference_orbit_direction=ref_orbit_direction, + reference_orbit_number=ref_orbit_number, + secondary_orbit_direction=sec_orbit_direction, + secondary_orbit_number=sec_orbit_number, + baseline=float(baseline_perp), + utc_time=utc_time, + heading=ref_heading, + spacecraft_height=SPACECRAFT_HEIGHT, + earth_radius_at_nadir=EARTH_RADIUS, + slant_range_near=slant_range_near, + slant_range_center=slant_range_center, + slant_range_far=slant_range_far, + range_looks=int(range_looks), + azimuth_looks=int(azimuth_looks), + insar_phase_filter=True, + phase_filter_parameter=float(phase_filter_strength), + range_bandpass_filter=False, + azimuth_bandpass_filter=False, + dem_source=dem_name, + dem_resolution=dem_resolution, + unwrapping_type=unwrapper_type, + speckle_filter=True, + water_mask=apply_water_mask, + ) + parameter_file.write(out_path) + + +def make_readme( + product_dir: Path, + product_name: str, + reference_scene: str, + secondary_scene: str, + range_looks: int, + azimuth_looks: int, + apply_water_mask: bool, +) -> None: + wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' + info = gdal.Info(str(wrapped_phase_path), format='json') + secondary_granule_datetime_str = secondary_scene.split('_')[5] + + payload = { + 'processing_date': datetime.now(timezone.utc), + 'plugin_name': hyp3_isce2.__name__, + 'plugin_version': hyp3_isce2.__version__, + 'processor_name': isce.__name__.upper(), + 'processor_version': isce.__version__, + 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'pixel_spacing': info['geoTransform'][1], + 'product_name': product_name, + 'reference_burst_name': reference_scene, + 'secondary_burst_name': secondary_scene, + 'range_looks': range_looks, + 'azimuth_looks': azimuth_looks, + 'secondary_granule_date': datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'), + 'dem_name': 'GLO-30', + 'dem_pixel_spacing': '30 m', + 'apply_water_mask': apply_water_mask, + } + content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_readme.md.txt.j2', payload) + + output_file = product_dir / f'{product_name}_README.md.txt' + with open(output_file, 'w') as f: + f.write(content) + + +def main(): + """HyP3 entrypoint for the TOPS burst merging workflow""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') + parser.add_argument('--reference', nargs='*', help='List of granules for the reference bursts') + parser.add_argument('--secondary', nargs='*', help='List of granules for the secondary bursts') + parser.add_argument( + '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' + ) + parser.add_argument( + '--apply-water-mask', + type=string_is_true, + default=False, + help='Apply a water body mask before unwrapping.', + ) + args = parser.parse_args() + granules_ref = list(set(args.reference)) + granules_sec = list(set(args.secondary)) + + ids_ref = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_ref] + ids_sec = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_sec] + + print(set(ids_ref)) + print(set(ids_sec)) + print(list(set(ids_ref)-set(ids_sec)),list(set(ids_sec)-set(ids_ref))) + if len(list(set(ids_ref)-set(ids_sec)))>0: + raise Exception('The reference bursts '+', '.join(list(set(ids_ref)-set(ids_sec)))+' do not have the correspondant bursts in the secondary granules') + elif len(list(set(ids_sec)-set(ids_ref)))>0: + raise Exception('The secondary bursts '+', '.join(list(set(ids_sec)-set(ids_ref)))+' do not have the correspondant bursts in the reference granules') + + work_dir = utils.optional_wd(None) + + if not granules_ref[0].split('_')[4] == granules_sec[0].split('_')[4]: + raise Exception('The secondary and reference granules do not have the same polarization') + + if granules_ref[0].split('_')[3] > granules_sec[0].split('_')[3]: + log.info('The secondary granules have a later date than the reference granules.') + temp=copy.copy(granules_ref) + granules_ref=copy.copy(granules_sec) + granules_sec=temp + + swaths=list(set([int(granule.split('_')[2][2]) for granule in granules_ref])) + + #reference_scene=burst2safe(granules_ref) + #reference_scene = os.path.basename(reference_scene).split('.')[0] + reference_scene = 'S1A_IW_SLC__1SSV_20230212T025529_20230212T025534_047197_05A9B2_971A' + + #secondary_scene=burst2safe(granules_sec) + #secondary_scene = os.path.basename(secondary_scene).split('.')[0] + secondary_scene = 'S1A_IW_SLC__1SSV_20230916T025538_20230916T025542_050347_060FBD_1EFB' + + polarization=granules_ref[0].split('_')[4] + + range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] + apply_water_mask = args.apply_water_mask + + #insar_tops(reference_scene, secondary_scene, swaths, polarization) + + pixel_size = get_pixel_size(args.looks) + product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) + + product_dir = Path(product_name) + product_dir.mkdir(parents=True, exist_ok=True) + + translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) + + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' + water_mask = f'{product_name}/{product_name}_water_mask.tif' + + if apply_water_mask: + convert_raster_from_isce2_gdal('water_mask.wgs84', unwrapped_phase, water_mask) + cmd = ( + 'gdal_calc.py ' + f'--outfile {unwrapped_phase} ' + f'-A {unwrapped_phase} -B {water_mask} ' + '--calc A*B ' + '--overwrite ' + '--NoDataValue 0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + + make_readme( + product_dir=product_dir, + product_name=product_name, + reference_scene=reference_scene, + secondary_scene=secondary_scene, + range_looks=range_looks, + azimuth_looks=azimuth_looks, + apply_water_mask=apply_water_mask, + ) + make_parameter_file( + Path(f'{product_name}/{product_name}.txt'), + reference_scene=reference_scene, + secondary_scene=secondary_scene, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + ) + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + + if args.bucket: + for browse in product_dir.glob('*.png'): + create_thumbnail(browse, output_dir=product_dir) + + upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix) + + for product_file in product_dir.iterdir(): + upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) + + +if __name__ == '__main__': + main() diff --git a/src/hyp3_isce2/slc.py b/src/hyp3_isce2/slc.py index 422e4846..74f4d41b 100644 --- a/src/hyp3_isce2/slc.py +++ b/src/hyp3_isce2/slc.py @@ -4,6 +4,7 @@ from subprocess import PIPE, run from zipfile import ZipFile +import lxml.etree as ET from hyp3lib.fetch import download_file from hyp3lib.scene import get_download_url from shapely import geometry @@ -19,6 +20,10 @@ def get_granule(granule: str) -> Path: Returns: The path to the unzipped granule """ + if Path(f'{granule}.SAFE').exists(): + print('SAFE file already exists, skipping download.') + return Path.cwd() / f'{granule}.SAFE' + download_url = get_download_url(granule) zip_file = download_file(download_url, chunk_size=10485760) safe_dir = unzip_granule(zip_file, remove=True) @@ -41,6 +46,16 @@ def get_geometry_from_kml(kml_file: str) -> Polygon: return geometry.shape(geojson) +def get_geometry_from_manifest(manifest_path: Path): + manifest = ET.parse(manifest_path).getroot() + frame_element = [x for x in manifest.findall('.//metadataObject') if x.get('ID') == 'measurementFrameSet'][0] + frame_string = frame_element.find('.//{http://www.opengis.net/gml}coordinates').text + coord_strings = [pair.split(',') for pair in frame_string.split(' ')] + coords = [(float(lon), float(lat)) for lat, lon in coord_strings] + footprint = Polygon(coords) + return footprint + + def get_dem_bounds(reference_granule: Path, secondary_granule: Path) -> tuple: """Get the bounds of the DEM to use in processing from SAFE KML files @@ -53,7 +68,7 @@ def get_dem_bounds(reference_granule: Path, secondary_granule: Path) -> tuple: """ bboxs = [] for granule in (reference_granule, secondary_granule): - footprint = get_geometry_from_kml(str(granule / 'preview' / 'map-overlay.kml')) + footprint = get_geometry_from_manifest(granule / 'manifest.safe') bbox = geometry.box(*footprint.bounds) bboxs.append(bbox) From 347eb3dfb3385998cb03bde81fca2b1a4caf63c3 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Mon, 12 Aug 2024 21:47:49 -0800 Subject: [PATCH 02/81] Small changes --- src/hyp3_isce2/insar_tops_multi_bursts.py | 99 +++++++---------------- src/hyp3_isce2/slc.py | 2 +- 2 files changed, 30 insertions(+), 71 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_multi_bursts.py b/src/hyp3_isce2/insar_tops_multi_bursts.py index d1c729f2..27a57062 100644 --- a/src/hyp3_isce2/insar_tops_multi_bursts.py +++ b/src/hyp3_isce2/insar_tops_multi_bursts.py @@ -1,66 +1,33 @@ """A workflow for merging standard burst InSAR products.""" import argparse import copy -import datetime import logging import os -import shutil import subprocess import sys -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime, timezone -from itertools import combinations -from lxml import etree from pathlib import Path from secrets import token_hex from shutil import make_archive -from tempfile import TemporaryDirectory -from typing import Iterable, List, Optional, Tuple +from typing import Iterable, Optional -import asf_search import isce -import isceobj -import lxml.etree as ET -import numpy as np -from burst2safe import utils from burst2safe.burst2safe import burst2safe -from burst2safe.safe import Safe -from burst2safe.search import download_bursts, find_bursts -from contrib.Snaphu.Snaphu import Snaphu from hyp3lib.aws import upload_file_to_s3 from hyp3lib.image import create_thumbnail from hyp3lib.util import string_is_true -from isceobj.Orbit.Orbit import Orbit -from isceobj.Planet.Planet import Planet -from isceobj.Sensor.TOPS.Sentinel1 import Sentinel1 -from isceobj.TopsProc.runIon import maskUnwrap -from isceobj.TopsProc.runMergeBursts import mergeBox, mergeBursts2 -from iscesys.Component import createTraitSeq -from iscesys.Component.ProductManager import ProductManager -from mroipac.filter.Filter import Filter -from mroipac.icu.Icu import Icu +from lxml import etree from osgeo import gdal, gdalconst -from shapely import geometry -from stdproc.rectify.geocode.Geocodable import Geocodable -from zerodop.geozero import createGeozero import hyp3_isce2 -import hyp3_isce2.burst as burst_utils -from hyp3_isce2.insar_tops_burst import find_product, get_pixel_size, convert_raster_from_isce2_gdal -from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.insar_tops import insar_tops +from hyp3_isce2.insar_tops_burst import convert_raster_from_isce2_gdal, find_product, get_pixel_size from hyp3_isce2.utils import ( ParameterFile, - create_image, - image_math, - load_product, make_browse_image, - read_product_metadata, - resample_to_radar_io, utm_from_lon_lat, ) -from hyp3_isce2.water_mask import create_water_mask log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' @@ -109,7 +76,7 @@ def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: ) -def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False, use_multilooked=False) -> None: +def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False) -> None: """Translate ISCE outputs to a standard GTiff format with a UTM projection. Assume you are in the top level of an ISCE run directory @@ -136,19 +103,16 @@ def translate_outputs(product_name: str, pixel_size: float, include_radar: bool ISCE2Dataset('merged/filt_topophase.unw.conncomp.geo', 'conncomp', [1]), ] - if use_multilooked: - suffix = '.multilooked' - rdr_datasets = [ ISCE2Dataset( - find_product(f'merged/filt_topophase.flat.vrt'), + find_product('merged/filt_topophase.flat.vrt'), 'wrapped_phase_rdr', [1], gdalconst.GDT_CFloat32, ), - ISCE2Dataset(find_product(f'merged/lat.rdr.full.vrt'), 'lat_rdr', [1]), - ISCE2Dataset(find_product(f'merged/lon.rdr.full.vrt'), 'lon_rdr', [1]), - ISCE2Dataset(find_product(f'merged/los.rdr.full.vrt'), 'los_rdr', [1, 2]), + ISCE2Dataset(find_product('merged/lat.rdr.full.vrt'), 'lat_rdr', [1]), + ISCE2Dataset(find_product('merged/lon.rdr.full.vrt'), 'lon_rdr', [1]), + ISCE2Dataset(find_product('merged/los.rdr.full.vrt'), 'los_rdr', [1, 2]), ] if include_radar: datasets += rdr_datasets @@ -287,7 +251,7 @@ def make_parameter_file( slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) - baseline_perp = topsProc_xml.find(f'.//IW-2_Bperp_at_midrange_for_first_common_burst').text + baseline_perp = topsProc_xml.find('.//IW-2_Bperp_at_midrange_for_first_common_burst').text unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text @@ -386,45 +350,40 @@ def main(): args = parser.parse_args() granules_ref = list(set(args.reference)) granules_sec = list(set(args.secondary)) - + ids_ref = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_ref] ids_sec = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_sec] - print(set(ids_ref)) - print(set(ids_sec)) - print(list(set(ids_ref)-set(ids_sec)),list(set(ids_sec)-set(ids_ref))) - if len(list(set(ids_ref)-set(ids_sec)))>0: - raise Exception('The reference bursts '+', '.join(list(set(ids_ref)-set(ids_sec)))+' do not have the correspondant bursts in the secondary granules') - elif len(list(set(ids_sec)-set(ids_ref)))>0: - raise Exception('The secondary bursts '+', '.join(list(set(ids_sec)-set(ids_ref)))+' do not have the correspondant bursts in the reference granules') - - work_dir = utils.optional_wd(None) + if len(list(set(ids_ref)-set(ids_sec))) > 0: + raise Exception('The reference bursts ' + ', '.join(list(set(ids_ref)-set(ids_sec))) + + ' do not have the correspondant bursts in the secondary granules') + elif len(list(set(ids_sec)-set(ids_ref))) > 0: + raise Exception('The secondary bursts ' + ', '.join(list(set(ids_sec)-set(ids_ref))) + + ' do not have the correspondant bursts in the reference granules') if not granules_ref[0].split('_')[4] == granules_sec[0].split('_')[4]: raise Exception('The secondary and reference granules do not have the same polarization') - + if granules_ref[0].split('_')[3] > granules_sec[0].split('_')[3]: log.info('The secondary granules have a later date than the reference granules.') - temp=copy.copy(granules_ref) - granules_ref=copy.copy(granules_sec) - granules_sec=temp + temp = copy.copy(granules_ref) + granules_ref = copy.copy(granules_sec) + granules_sec = temp - swaths=list(set([int(granule.split('_')[2][2]) for granule in granules_ref])) + swaths = list(set([int(granule.split('_')[2][2]) for granule in granules_ref])) - #reference_scene=burst2safe(granules_ref) - #reference_scene = os.path.basename(reference_scene).split('.')[0] - reference_scene = 'S1A_IW_SLC__1SSV_20230212T025529_20230212T025534_047197_05A9B2_971A' + reference_scene = burst2safe(granules_ref) + reference_scene = os.path.basename(reference_scene).split('.')[0] - #secondary_scene=burst2safe(granules_sec) - #secondary_scene = os.path.basename(secondary_scene).split('.')[0] - secondary_scene = 'S1A_IW_SLC__1SSV_20230916T025538_20230916T025542_050347_060FBD_1EFB' + secondary_scene = burst2safe(granules_sec) + secondary_scene = os.path.basename(secondary_scene).split('.')[0] - polarization=granules_ref[0].split('_')[4] + polarization = granules_ref[0].split('_')[4] range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] apply_water_mask = args.apply_water_mask - #insar_tops(reference_scene, secondary_scene, swaths, polarization) + insar_tops(reference_scene, secondary_scene, swaths, polarization) pixel_size = get_pixel_size(args.looks) product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) @@ -432,7 +391,7 @@ def main(): product_dir = Path(product_name) product_dir.mkdir(parents=True, exist_ok=True) - translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) + translate_outputs(product_name, pixel_size=pixel_size, include_radar=True) unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' water_mask = f'{product_name}/{product_name}_water_mask.tif' diff --git a/src/hyp3_isce2/slc.py b/src/hyp3_isce2/slc.py index 74f4d41b..b26893f4 100644 --- a/src/hyp3_isce2/slc.py +++ b/src/hyp3_isce2/slc.py @@ -54,7 +54,7 @@ def get_geometry_from_manifest(manifest_path: Path): coords = [(float(lon), float(lat)) for lat, lon in coord_strings] footprint = Polygon(coords) return footprint - + def get_dem_bounds(reference_granule: Path, secondary_granule: Path) -> tuple: """Get the bounds of the DEM to use in processing from SAFE KML files From a564e55fd997e0ef57058486d473b6a23afef9b9 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Mon, 12 Aug 2024 21:59:46 -0800 Subject: [PATCH 03/81] Small changes --- src/hyp3_isce2/insar_tops_multi_bursts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_bursts.py b/src/hyp3_isce2/insar_tops_multi_bursts.py index 27a57062..5666f170 100644 --- a/src/hyp3_isce2/insar_tops_multi_bursts.py +++ b/src/hyp3_isce2/insar_tops_multi_bursts.py @@ -383,7 +383,7 @@ def main(): range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] apply_water_mask = args.apply_water_mask - insar_tops(reference_scene, secondary_scene, swaths, polarization) + insar_tops(reference_scene, secondary_scene, swaths, polarization, download=False) pixel_size = get_pixel_size(args.looks) product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) From d84d810680c71c333f7de5beeb7b9e5fcc50f44c Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Aug 2024 09:36:15 -0500 Subject: [PATCH 04/81] morning work --- src/hyp3_isce2/burst.py | 42 +-- src/hyp3_isce2/insar_tops.py | 77 +++- src/hyp3_isce2/insar_tops_burst.py | 518 ++++++-------------------- src/hyp3_isce2/packaging.py | 561 +++++++++++++++++++++++++++++ src/hyp3_isce2/utils.py | 134 +------ 5 files changed, 739 insertions(+), 593 deletions(-) create mode 100644 src/hyp3_isce2/packaging.py diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 2ab7a3cd..2c5e37a5 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -7,7 +7,6 @@ from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path -from secrets import token_hex from typing import Iterator, List, Optional, Tuple, Union import asf_search @@ -345,45 +344,6 @@ def download_bursts(param_list: Iterator[BurstParams]) -> List[BurstMetadata]: return bursts -def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: int) -> str: - """Get the name of the interferogram product. - - Args: - reference_scene: The reference burst name. - secondary_scene: The secondary burst name. - pixel_spacing: The spacing of the pixels in the output image. - - Returns: - The name of the interferogram product. - """ - - reference_split = reference_scene.split('_') - secondary_split = secondary_scene.split('_') - - platform = reference_split[0] - burst_id = reference_split[1] - image_plus_swath = reference_split[2] - reference_date = reference_split[3][0:8] - secondary_date = secondary_split[3][0:8] - polarization = reference_split[4] - product_type = 'INT' - pixel_spacing = str(int(pixel_spacing)) - product_id = token_hex(2).upper() - - return '_'.join( - [ - platform, - burst_id, - image_plus_swath, - reference_date, - secondary_date, - polarization, - product_type + pixel_spacing, - product_id, - ] - ) - - def get_burst_params(scene_name: str) -> BurstParams: results = asf_search.search(product_list=[scene_name]) @@ -572,7 +532,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line: last_line, position.first_valid_sample: last_sample] = identity_value + mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value else: mask[:, :] = identity_value diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index f2bce38e..57e949aa 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -6,11 +6,9 @@ from pathlib import Path from shutil import copyfile, make_archive -from hyp3lib.aws import upload_file_to_s3 from s1_orbits import fetch_for_scene -from hyp3_isce2 import slc -from hyp3_isce2 import topsapp +from hyp3_isce2 import packaging, slc, topsapp from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal @@ -49,8 +47,8 @@ def insar_tops( ref_dir = slc.get_granule(reference_scene) sec_dir = slc.get_granule(secondary_scene) else: - ref_dir = Path(reference_scene+'.SAFE') - sec_dir = Path(secondary_scene+'.SAFE') + ref_dir = Path(reference_scene + '.SAFE') + sec_dir = Path(secondary_scene + '.SAFE') roi = slc.get_dem_bounds(ref_dir, sec_dir) log.info(f'DEM ROI: {roi}') @@ -85,6 +83,70 @@ def insar_tops( return Path('merged') +def insar_tops_packaged( + reference: str, + secondary: str, + swaths: list = [1, 2, 3], + polarization: str = 'VV', + azimuth_looks: int = 4, + range_looks: int = 20, + apply_water_mask: bool = True, + download: bool = True, +) -> Path: + """Create a full-SLC interferogram + + Args: + reference_scene: Reference SLC name + secondary_scene: Secondary SLC name + swaths: Swaths to process + polarization: Polarization to use + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + apply_water_mask: Apply water mask to unwrapped phase + download: Download the SLCs + + Returns: + Path to the output files + """ + pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + + log.info('Begin ISCE2 TopsApp run') + insar_tops(reference, secondary, download=False) + log.info('ISCE2 TopsApp run completed successfully') + + product_dir = Path(product_name) + product_dir.mkdir(parents=True, exist_ok=True) + + packaging.translate_outputs(product_name, pixel_size=pixel_size) + + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' + if apply_water_mask: + packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') + + packaging.make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + packaging.make_readme( + product_dir=product_dir, + product_name=product_name, + reference_scene=reference, + secondary_scene=secondary, + range_looks=range_looks, + azimuth_looks=azimuth_looks, + apply_water_mask=apply_water_mask, + ) + packaging.make_parameter_file( + Path(f'{product_name}/{product_name}.txt'), + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + # swath_number=swath_number, + # multilook_position=multilook_position, + apply_water_mask=apply_water_mask, + ) + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + + def main(): """HyP3 entrypoint for the SLC TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -95,10 +157,7 @@ def main(): parser.add_argument('--secondary-scene', type=str, required=True) parser.add_argument('--polarization', type=str, choices=['VV', 'HH'], default='VV') parser.add_argument( - '--looks', - choices=['20x4', '10x2', '5x1'], - default='20x4', - help='Number of looks to take in range and azimuth' + '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' ) args = parser.parse_args() diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index f76b35dd..08922e4d 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -2,49 +2,36 @@ import argparse import logging -import os -import subprocess import sys -from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path from shutil import copyfile, make_archive from typing import Iterable, Optional -import isce -from hyp3lib.aws import upload_file_to_s3 -from hyp3lib.image import create_thumbnail +import isce # noqa +from burst2safe import burst2safe from hyp3lib.util import string_is_true from isceobj.TopsProc.runMergeBursts import multilook -from lxml import etree -from osgeo import gdal, gdalconst -from pyproj import CRS +from osgeo import gdal from s1_orbits import fetch_for_scene -import hyp3_isce2 -import hyp3_isce2.metadata.util -from hyp3_isce2 import topsapp +from hyp3_isce2 import packaging, topsapp from hyp3_isce2.burst import ( - BurstPosition, download_bursts, get_burst_params, get_isce2_burst_bbox, - get_product_name, get_region_of_interest, multilook_radar_merge_inputs, validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.insar_tops import insar_tops from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( - ParameterFile, image_math, isce2_copy, - make_browse_image, oldest_granule_first, resample_to_radar_io, - utm_from_lon_lat, ) from hyp3_isce2.water_mask import create_water_mask @@ -54,14 +41,6 @@ log = logging.getLogger(__name__) -@dataclass -class ISCE2Dataset: - name: str - suffix: str - band: Iterable[int] - dtype: Optional[int] = gdalconst.GDT_Float32 - - def insar_tops_burst( reference_scene: str, secondary_scene: str, @@ -157,329 +136,109 @@ def insar_tops_burst( return Path('merged') -def make_readme( - product_dir: Path, - product_name: str, - reference_scene: str, - secondary_scene: str, - range_looks: int, - azimuth_looks: int, - apply_water_mask: bool, -) -> None: - wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' - info = gdal.Info(str(wrapped_phase_path), format='json') - secondary_granule_datetime_str = secondary_scene.split('_')[3] - - payload = { - 'processing_date': datetime.now(timezone.utc), - 'plugin_name': hyp3_isce2.__name__, - 'plugin_version': hyp3_isce2.__version__, - 'processor_name': isce.__name__.upper(), - 'processor_version': isce.__version__, - 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), - 'pixel_spacing': info['geoTransform'][1], - 'product_name': product_name, - 'reference_burst_name': reference_scene, - 'secondary_burst_name': secondary_scene, - 'range_looks': range_looks, - 'azimuth_looks': azimuth_looks, - 'secondary_granule_date': datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'), - 'dem_name': 'GLO-30', - 'dem_pixel_spacing': '30 m', - 'apply_water_mask': apply_water_mask, - } - content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_readme.md.txt.j2', payload) - - output_file = product_dir / f'{product_name}_README.md.txt' - with open(output_file, 'w') as f: - f.write(content) - - -def make_parameter_file( - out_path: Path, - reference_scene: str, - secondary_scene: str, - swath_number: int, - azimuth_looks: int, - range_looks: int, - multilook_position: BurstPosition, - apply_water_mask: bool, - dem_name: str = 'GLO_30', - dem_resolution: int = 30, -) -> None: - """Create a parameter file for the output product +def insar_tops_single_burst( + reference: str, + secondary: str, + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + reference, secondary = oldest_granule_first(reference, secondary) + validate_bursts(reference, secondary) + swath_number = int(reference[12]) + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] - Args: - out_path: path to output the parameter file - reference_scene: Reference burst name - secondary_scene: Secondary burst name - swath_number: Number of swath to grab bursts from (1, 2, or 3) for IW - azimuth_looks: Number of azimuth looks - range_looks: Number of range looks - dem_name: Name of the DEM that is use - dem_resolution: Resolution of the DEM + log.info('Begin ISCE2 TopsApp run') - returns: - None - """ - SPEED_OF_LIGHT = 299792458.0 - SPACECRAFT_HEIGHT = 693000.0 - EARTH_RADIUS = 6337286.638938101 - - parser = etree.XMLParser(encoding='utf-8', recover=True) - - ref_tag = reference_scene[-10:-6] - sec_tag = secondary_scene[-10:-6] - reference_safe = [file for file in os.listdir('.') if file.endswith(f'{ref_tag}.SAFE')][0] - secondary_safe = [file for file in os.listdir('.') if file.endswith(f'{sec_tag}.SAFE')][0] - - ref_annotation_path = f'{reference_safe}/annotation/' - ref_annotation = [file for file in os.listdir(ref_annotation_path) if os.path.isfile(ref_annotation_path + file)][0] - - ref_manifest_xml = etree.parse(f'{reference_safe}/manifest.safe', parser) - sec_manifest_xml = etree.parse(f'{secondary_safe}/manifest.safe', parser) - ref_annotation_xml = etree.parse(f'{ref_annotation_path}{ref_annotation}', parser) - topsProc_xml = etree.parse('topsProc.xml', parser) - topsApp_xml = etree.parse('topsApp.xml', parser) - - safe = '{http://www.esa.int/safe/sentinel-1.0}' - s1 = '{http://www.esa.int/safe/sentinel-1.0/sentinel-1}' - metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' - orbit_number_query = metadata_path + safe + 'orbitNumber' - orbit_direction_query = metadata_path + safe + 'extension//' + s1 + 'pass' - - ref_orbit_number = ref_manifest_xml.find(orbit_number_query).text - ref_orbit_direction = ref_manifest_xml.find(orbit_direction_query).text - sec_orbit_number = sec_manifest_xml.find(orbit_number_query).text - sec_orbit_direction = sec_manifest_xml.find(orbit_direction_query).text - ref_heading = float(ref_annotation_xml.find('.//platformHeading').text) - ref_time = ref_annotation_xml.find('.//productFirstLineUtcTime').text - slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) - range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) - number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) - baseline_perp = topsProc_xml.find(f'.//IW-{swath_number}_Bperp_at_midrange_for_first_common_burst').text - unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text - phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text - - slant_range_near = float(slant_range_time) * SPEED_OF_LIGHT / 2 - range_pixel_spacing = SPEED_OF_LIGHT / (2 * range_sampling_rate) - slant_range_far = slant_range_near + (number_samples - 1) * range_pixel_spacing - slant_range_center = (slant_range_near + slant_range_far) / 2 - - s = ref_time.split('T')[1].split(':') - utc_time = ((int(s[0]) * 60 + int(s[1])) * 60) + float(s[2]) - - parameter_file = ParameterFile( - reference_granule=reference_scene, - secondary_granule=secondary_scene, - reference_orbit_direction=ref_orbit_direction, - reference_orbit_number=ref_orbit_number, - secondary_orbit_direction=sec_orbit_direction, - secondary_orbit_number=sec_orbit_number, - baseline=float(baseline_perp), - utc_time=utc_time, - heading=ref_heading, - spacecraft_height=SPACECRAFT_HEIGHT, - earth_radius_at_nadir=EARTH_RADIUS, - slant_range_near=slant_range_near, - slant_range_center=slant_range_center, - slant_range_far=slant_range_far, - range_looks=int(range_looks), - azimuth_looks=int(azimuth_looks), - insar_phase_filter=True, - phase_filter_parameter=float(phase_filter_strength), - range_bandpass_filter=False, - azimuth_bandpass_filter=False, - dem_source=dem_name, - dem_resolution=dem_resolution, - unwrapping_type=unwrapper_type, - speckle_filter=True, - water_mask=apply_water_mask, - radar_n_lines=multilook_position.n_lines, - radar_n_samples=multilook_position.n_samples, - radar_first_valid_line=multilook_position.first_valid_line, - radar_n_valid_lines=multilook_position.n_valid_lines, - radar_first_valid_sample=multilook_position.first_valid_sample, - radar_n_valid_samples=multilook_position.n_valid_samples, - multilook_azimuth_time_interval=multilook_position.azimuth_time_interval, - multilook_range_pixel_size=multilook_position.range_pixel_size, - radar_sensing_stop=multilook_position.sensing_stop, + insar_tops_burst( + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + swath_number=swath_number, + apply_water_mask=apply_water_mask, ) - parameter_file.write(out_path) + log.info('ISCE2 TopsApp run completed successfully') -def find_product(pattern: str) -> str: - """Find a single file within the working directory's structure - - Args: - pattern: Glob pattern for file + multilook_position = multilook_radar_merge_inputs(swath_number, rg_looks=range_looks, az_looks=azimuth_looks) - Returns - Path to file - """ - search = Path.cwd().glob(pattern) - product = str(list(search)[0]) - return product + pixel_size = packaging.get_pixel_size(looks) + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + product_dir = Path(product_name) + product_dir.mkdir(parents=True, exist_ok=True) -def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False, use_multilooked=False) -> None: - """Translate ISCE outputs to a standard GTiff format with a UTM projection. - Assume you are in the top level of an ISCE run directory + packaging.translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) - Args: - product_name: Name of the product - pixel_size: Pixel size - include_radar: Flag to include the full resolution radar geometry products in the output - """ + if apply_water_mask: + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' + water_mask = f'{product_name}/{product_name}_water_mask.tif' + packaging.water_mask(unwrapped_phase, water_mask) - src_ds = gdal.Open('merged/filt_topophase.unw.geo') - src_geotransform = src_ds.GetGeoTransform() - src_projection = src_ds.GetProjection() - - target_ds = gdal.Open('merged/dem.crop', gdal.GA_Update) - target_ds.SetGeoTransform(src_geotransform) - target_ds.SetProjection(src_projection) - - del src_ds, target_ds - - datasets = [ - ISCE2Dataset('merged/filt_topophase.unw.geo', 'unw_phase', [2]), - ISCE2Dataset('merged/phsig.cor.geo', 'corr', [1]), - ISCE2Dataset('merged/dem.crop', 'dem', [1]), - ISCE2Dataset('merged/filt_topophase.unw.conncomp.geo', 'conncomp', [1]), - ] - - suffix = '01' - if use_multilooked: - suffix += '.multilooked' - - rdr_datasets = [ - ISCE2Dataset( - find_product(f'fine_interferogram/IW*/burst_{suffix}.int.vrt'), - 'wrapped_phase_rdr', - [1], - gdalconst.GDT_CFloat32, - ), - ISCE2Dataset(find_product(f'geom_reference/IW*/lat_{suffix}.rdr.vrt'), 'lat_rdr', [1]), - ISCE2Dataset(find_product(f'geom_reference/IW*/lon_{suffix}.rdr.vrt'), 'lon_rdr', [1]), - ISCE2Dataset(find_product(f'geom_reference/IW*/los_{suffix}.rdr.vrt'), 'los_rdr', [1, 2]), - ] - if include_radar: - datasets += rdr_datasets - - for dataset in datasets: - out_file = str(Path(product_name) / f'{product_name}_{dataset.suffix}.tif') - gdal.Translate( - destName=out_file, - srcDS=dataset.name, - bandList=dataset.band, - format='GTiff', - outputType=dataset.dtype, - noData=0, - creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], - ) + packaging.make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') - # Use numpy.angle to extract the phase component of the complex wrapped interferogram - wrapped_phase = ISCE2Dataset('filt_topophase.flat.geo', 'wrapped_phase', 1) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{wrapped_phase.suffix}.tif ' - f'-A merged/{wrapped_phase.name} --A_band={wrapped_phase.band} ' - '--calc angle(A) --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' - ) - subprocess.run(cmd.split(' '), check=True) - - ds = gdal.Open('merged/los.rdr.geo', gdal.GA_Update) - ds.GetRasterBand(1).SetNoDataValue(0) - ds.GetRasterBand(2).SetNoDataValue(0) - del ds - - # Performs the inverse of the operation performed by MintPy: - # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L732-L737 - # First subtract the incidence angle from ninety degrees to go from sensor-to-ground to ground-to-sensor, - # then convert to radians - incidence_angle = ISCE2Dataset('los.rdr.geo', 'lv_theta', 1) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{incidence_angle.suffix}.tif ' - f'-A merged/{incidence_angle.name} --A_band={incidence_angle.band} ' - '--calc (90-A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + packaging.make_readme( + product_dir=product_dir, + product_name=product_name, + reference_scene=reference, + secondary_scene=secondary, + range_looks=range_looks, + azimuth_looks=azimuth_looks, + apply_water_mask=apply_water_mask, ) - subprocess.run(cmd.split(' '), check=True) - - # Performs the inverse of the operation performed by MintPy: - # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L739-L745 - # First add ninety degrees to the azimuth angle to go from angle-from-east to angle-from-north, - # then convert to radians - azimuth_angle = ISCE2Dataset('los.rdr.geo', 'lv_phi', 2) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{azimuth_angle.suffix}.tif ' - f'-A merged/{azimuth_angle.name} --A_band={azimuth_angle.band} ' - '--calc (90+A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + packaging.make_parameter_file( + Path(f'{product_name}/{product_name}.txt'), + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + swath_number=swath_number, + multilook_position=multilook_position, + apply_water_mask=apply_water_mask, ) - subprocess.run(cmd.split(' '), check=True) - - ds = gdal.Open('merged/filt_topophase.unw.geo') - geotransform = ds.GetGeoTransform() - del ds - - epsg = utm_from_lon_lat(geotransform[0], geotransform[3]) - files = [str(path) for path in Path(product_name).glob('*.tif') if not path.name.endswith('rdr.tif')] - for file in files: - gdal.Warp( - file, - file, - dstSRS=f'epsg:{epsg}', - creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], - xRes=pixel_size, - yRes=pixel_size, - targetAlignedPixels=True, - ) - - -def get_pixel_size(looks: str) -> float: - return {'20x4': 80.0, '10x2': 40.0, '5x1': 20.0}[looks] - - -def convert_raster_from_isce2_gdal(input_image, ref_image, output_image): - """Convert the water mask in WGS84 to be the same projection and extent of the output product. - - Args: - input_image: dem file name - ref_image: output geotiff file name - output_image: water mask file name - """ - - ref_ds = gdal.Open(ref_image) - - gt = ref_ds.GetGeoTransform() - - pixel_size = gt[1] + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - minx = gt[0] - maxx = gt[0] + gt[1] * ref_ds.RasterXSize - maxy = gt[3] - miny = gt[3] + gt[5] * ref_ds.RasterYSize + if bucket: + packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) + + +def insar_tops_multi_burst( + reference: Iterable[str], + secondary: Iterable[str], + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + + if len(list(set(ref_ids) - set(sec_ids))) > 0: + raise Exception( + 'The reference bursts ' + + ', '.join(list(set(ref_ids) - set(sec_ids))) + + ' do not have the correspondant bursts in the secondary granules' + ) + elif len(list(set(sec_ids) - set(ref_ids))) > 0: + raise Exception( + 'The secondary bursts ' + + ', '.join(list(set(sec_ids) - set(ref_ids))) + + ' do not have the correspondant bursts in the reference granules' + ) - crs = ref_ds.GetSpatialRef() - epsg = CRS.from_wkt(crs.ExportToWkt()).to_epsg() + if not reference[0].split('_')[4] == secondary[0].split('_')[4]: + raise Exception('The secondary and reference granules do not have the same polarization') - del ref_ds + reference_safe_path = burst2safe(reference) + reference_safe = reference_safe_path.name.split('.')[0] + secondary_safe_path = burst2safe(secondary) + secondary_safe = secondary_safe_path.name.split('.')[0] - gdal.Warp( - output_image, - input_image, - dstSRS=f'epsg:{epsg}', - creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], - outputBounds=[minx, miny, maxx, maxy], - xRes=pixel_size, - yRes=pixel_size, - targetAlignedPixels=True, - ) + log.info('Begin ISCE2 TopsApp run') + insar_tops(reference_safe, secondary_safe, download=False) + log.info('ISCE2 TopsApp run completed successfully') def main(): @@ -500,90 +259,27 @@ def main(): # Allows granules to be given as a space-delimited list of strings (e.g. foo bar) or as a single # quoted string that contains spaces (e.g. "foo bar"). AWS Batch uses the latter format when # invoking the container command. - parser.add_argument('granules', type=str.split, nargs='+') + parser.add_argument('--reference', type=str.split, nargs='+', help='List of granules for the reference bursts') + parser.add_argument('--secondary', type=str.split, nargs='+', help='List of granules for the secondary bursts') args = parser.parse_args() - args.granules = [item for sublist in args.granules for item in sublist] - if len(args.granules) != 2: - parser.error('Must provide exactly two granules') + args.reference = [item for sublist in args.reference for item in sublist] + args.secondary = [item for sublist in args.secondary for item in sublist] + if len(args.reference) != len(args.secondary): + parser.error('Number of reference and secondary granules must be the same') configure_root_logger() log.debug(' '.join(sys.argv)) - log.info('Begin ISCE2 TopsApp run') - - reference_scene, secondary_scene = oldest_granule_first(args.granules[0], args.granules[1]) - validate_bursts(reference_scene, secondary_scene) - swath_number = int(reference_scene[12]) - range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] - apply_water_mask = args.apply_water_mask - - insar_tops_burst( - reference_scene=reference_scene, - secondary_scene=secondary_scene, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - swath_number=swath_number, - apply_water_mask=apply_water_mask, - ) - - log.info('ISCE2 TopsApp run completed successfully') - - multilook_position = multilook_radar_merge_inputs(swath_number, rg_looks=range_looks, az_looks=azimuth_looks) - - pixel_size = get_pixel_size(args.looks) - product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) - - product_dir = Path(product_name) - product_dir.mkdir(parents=True, exist_ok=True) - - translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) - - unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' - water_mask = f'{product_name}/{product_name}_water_mask.tif' - - if apply_water_mask: - convert_raster_from_isce2_gdal('water_mask.wgs84', unwrapped_phase, water_mask) - cmd = ( - 'gdal_calc.py ' - f'--outfile {unwrapped_phase} ' - f'-A {unwrapped_phase} -B {water_mask} ' - '--calc A*B ' - '--overwrite ' - '--NoDataValue 0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + if len(args.reference) == 1: + insar_tops_single_burst( + reference=args.reference[0], + secondary=args.secondary[0], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, ) - subprocess.run(cmd.split(' '), check=True) - - make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') - - make_readme( - product_dir=product_dir, - product_name=product_name, - reference_scene=reference_scene, - secondary_scene=secondary_scene, - range_looks=range_looks, - azimuth_looks=azimuth_looks, - apply_water_mask=apply_water_mask, - ) - make_parameter_file( - Path(f'{product_name}/{product_name}.txt'), - reference_scene=reference_scene, - secondary_scene=secondary_scene, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - swath_number=swath_number, - multilook_position=multilook_position, - apply_water_mask=apply_water_mask, - ) - output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - - if args.bucket: - for browse in product_dir.glob('*.png'): - create_thumbnail(browse, output_dir=product_dir) - - upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix) - - for product_file in product_dir.iterdir(): - upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) + else: + insar_tops_multi_burst() diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py new file mode 100644 index 00000000..e53ea3b6 --- /dev/null +++ b/src/hyp3_isce2/packaging.py @@ -0,0 +1,561 @@ +import os +import subprocess +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from secrets import token_hex +from typing import Iterable, Optional + +import isce +from hyp3lib.aws import upload_file_to_s3 +from hyp3lib.image import create_thumbnail +from lxml import etree +from osgeo import gdal, gdalconst +from pyproj import CRS + +import hyp3_isce2 +from hyp3_isce2.burst import BurstPosition +from hyp3_isce2.utils import utm_from_lon_lat + + +@dataclass +class ISCE2Dataset: + name: str + suffix: str + band: Iterable[int] + dtype: Optional[int] = gdalconst.GDT_Float32 + + +def get_pixel_size(looks: str) -> float: + return {'20x4': 80.0, '10x2': 40.0, '5x1': 20.0}[looks] + + +def find_product(pattern: str) -> str: + """Find a single file within the working directory's structure + + Args: + pattern: Glob pattern for file + + Returns + Path to file + """ + search = Path.cwd().glob(pattern) + product = str(list(search)[0]) + return product + + +def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: int, slc: bool = True) -> str: + """Get the name of the interferogram product. + + Args: + reference_scene: The reference burst name. + secondary_scene: The secondary burst name. + pixel_spacing: The spacing of the pixels in the output image. + slc: Whether the input scenes are SLCs or bursts. + + Returns: + The name of the interferogram product. + """ + + reference_split = reference_scene.split('_') + secondary_split = secondary_scene.split('_') + + if slc: + platform = reference_split[0] + reference_date = reference_split[5][0:8] + secondary_date = secondary_split[5][0:8] + polarization = reference_split[4] + # TODO: Remove hard code + polarization = 'VV' + name_parts = [platform] + else: + platform = reference_split[0] + burst_id = reference_split[1] + image_plus_swath = reference_split[2] + reference_date = reference_split[3][0:8] + secondary_date = secondary_split[3][0:8] + polarization = reference_split[4] + name_parts = [platform, burst_id, image_plus_swath] + + product_type = 'INT' + pixel_spacing = str(int(pixel_spacing)) + product_id = token_hex(2).upper() + product_name = '_'.join( + name_parts + + [ + reference_date, + secondary_date, + polarization, + product_type + pixel_spacing, + product_id, + ] + ) + + return product_name + + +def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False, use_multilooked=False) -> None: + """Translate ISCE outputs to a standard GTiff format with a UTM projection. + Assume you are in the top level of an ISCE run directory + + Args: + product_name: Name of the product + pixel_size: Pixel size + include_radar: Flag to include the full resolution radar geometry products in the output + use_multilooked: Flag to use multilooked versions of the radar geometry products + """ + + src_ds = gdal.Open('merged/filt_topophase.unw.geo') + src_geotransform = src_ds.GetGeoTransform() + src_projection = src_ds.GetProjection() + + target_ds = gdal.Open('merged/dem.crop', gdal.GA_Update) + target_ds.SetGeoTransform(src_geotransform) + target_ds.SetProjection(src_projection) + + del src_ds, target_ds + + datasets = [ + ISCE2Dataset('merged/filt_topophase.unw.geo', 'unw_phase', [2]), + ISCE2Dataset('merged/phsig.cor.geo', 'corr', [1]), + ISCE2Dataset('merged/dem.crop', 'dem', [1]), + ISCE2Dataset('merged/filt_topophase.unw.conncomp.geo', 'conncomp', [1]), + ] + + suffix = '01' + if use_multilooked: + suffix += '.multilooked' + + rdr_datasets = [ + ISCE2Dataset( + find_product(f'fine_interferogram/IW*/burst_{suffix}.int.vrt'), + 'wrapped_phase_rdr', + [1], + gdalconst.GDT_CFloat32, + ), + ISCE2Dataset(find_product(f'geom_reference/IW*/lat_{suffix}.rdr.vrt'), 'lat_rdr', [1]), + ISCE2Dataset(find_product(f'geom_reference/IW*/lon_{suffix}.rdr.vrt'), 'lon_rdr', [1]), + ISCE2Dataset(find_product(f'geom_reference/IW*/los_{suffix}.rdr.vrt'), 'los_rdr', [1, 2]), + ] + if include_radar: + datasets += rdr_datasets + + for dataset in datasets: + out_file = str(Path(product_name) / f'{product_name}_{dataset.suffix}.tif') + gdal.Translate( + destName=out_file, + srcDS=dataset.name, + bandList=dataset.band, + format='GTiff', + outputType=dataset.dtype, + noData=0, + creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], + ) + + # Use numpy.angle to extract the phase component of the complex wrapped interferogram + wrapped_phase = ISCE2Dataset('filt_topophase.flat.geo', 'wrapped_phase', 1) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{wrapped_phase.suffix}.tif ' + f'-A merged/{wrapped_phase.name} --A_band={wrapped_phase.band} ' + '--calc angle(A) --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + ds = gdal.Open('merged/los.rdr.geo', gdal.GA_Update) + ds.GetRasterBand(1).SetNoDataValue(0) + ds.GetRasterBand(2).SetNoDataValue(0) + del ds + + # Performs the inverse of the operation performed by MintPy: + # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L732-L737 + # First subtract the incidence angle from ninety degrees to go from sensor-to-ground to ground-to-sensor, + # then convert to radians + incidence_angle = ISCE2Dataset('los.rdr.geo', 'lv_theta', 1) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{incidence_angle.suffix}.tif ' + f'-A merged/{incidence_angle.name} --A_band={incidence_angle.band} ' + '--calc (90-A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + # Performs the inverse of the operation performed by MintPy: + # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L739-L745 + # First add ninety degrees to the azimuth angle to go from angle-from-east to angle-from-north, + # then convert to radians + azimuth_angle = ISCE2Dataset('los.rdr.geo', 'lv_phi', 2) + cmd = ( + 'gdal_calc.py ' + f'--outfile {product_name}/{product_name}_{azimuth_angle.suffix}.tif ' + f'-A merged/{azimuth_angle.name} --A_band={azimuth_angle.band} ' + '--calc (90+A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + ds = gdal.Open('merged/filt_topophase.unw.geo') + geotransform = ds.GetGeoTransform() + del ds + + epsg = utm_from_lon_lat(geotransform[0], geotransform[3]) + files = [str(path) for path in Path(product_name).glob('*.tif') if not path.name.endswith('rdr.tif')] + for file in files: + gdal.Warp( + file, + file, + dstSRS=f'epsg:{epsg}', + creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], + xRes=pixel_size, + yRes=pixel_size, + targetAlignedPixels=True, + ) + + +def convert_raster_from_isce2_gdal(input_image, ref_image, output_image): + """Convert the water mask in WGS84 to be the same projection and extent of the output product. + + Args: + input_image: dem file name + ref_image: output geotiff file name + output_image: water mask file name + """ + + ref_ds = gdal.Open(ref_image) + + gt = ref_ds.GetGeoTransform() + + pixel_size = gt[1] + + minx = gt[0] + maxx = gt[0] + gt[1] * ref_ds.RasterXSize + maxy = gt[3] + miny = gt[3] + gt[5] * ref_ds.RasterYSize + + crs = ref_ds.GetSpatialRef() + epsg = CRS.from_wkt(crs.ExportToWkt()).to_epsg() + + del ref_ds + + gdal.Warp( + output_image, + input_image, + dstSRS=f'epsg:{epsg}', + creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], + outputBounds=[minx, miny, maxx, maxy], + xRes=pixel_size, + yRes=pixel_size, + targetAlignedPixels=True, + ) + + +def water_mask(unwrapped_phase: str, water_mask: str) -> None: + """Apply the water mask to the unwrapped phase + + Args: + unwrapped_phase: The unwrapped phase file + water_mask: The water mask file + """ + + convert_raster_from_isce2_gdal('water_mask.wgs84', unwrapped_phase, water_mask) + cmd = ( + 'gdal_calc.py ' + f'--outfile {unwrapped_phase} ' + f'-A {unwrapped_phase} -B {water_mask} ' + '--calc A*B ' + '--overwrite ' + '--NoDataValue 0 ' + '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' + ) + subprocess.run(cmd.split(' '), check=True) + + +class GDALConfigManager: + """Context manager for setting GDAL config options temporarily""" + + def __init__(self, **options): + """ + Args: + **options: GDAL Config `option=value` keyword arguments. + """ + self.options = options.copy() + self._previous_options = {} + + def __enter__(self): + for key in self.options: + self._previous_options[key] = gdal.GetConfigOption(key) + + for key, value in self.options.items(): + gdal.SetConfigOption(key, value) + + def __exit__(self, exc_type, exc_val, exc_tb): + for key, value in self._previous_options.items(): + gdal.SetConfigOption(key, value) + + +def make_browse_image(input_tif: str, output_png: str) -> None: + with GDALConfigManager(GDAL_PAM_ENABLED='NO'): + stats = gdal.Info(input_tif, format='json', stats=True)['stac']['raster:bands'][0]['stats'] + gdal.Translate( + destName=output_png, + srcDS=input_tif, + format='png', + outputType=gdal.GDT_Byte, + width=2048, + strict=True, + scaleParams=[[stats['minimum'], stats['maximum']]], + ) + + +def make_readme( + product_dir: Path, + product_name: str, + reference_scene: str, + secondary_scene: str, + range_looks: int, + azimuth_looks: int, + apply_water_mask: bool, +) -> None: + wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' + info = gdal.Info(str(wrapped_phase_path), format='json') + secondary_granule_datetime_str = secondary_scene.split('_')[3] + + payload = { + 'processing_date': datetime.now(timezone.utc), + 'plugin_name': hyp3_isce2.__name__, + 'plugin_version': hyp3_isce2.__version__, + 'processor_name': isce.__name__.upper(), + 'processor_version': isce.__version__, + 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'pixel_spacing': info['geoTransform'][1], + 'product_name': product_name, + 'reference_burst_name': reference_scene, + 'secondary_burst_name': secondary_scene, + 'range_looks': range_looks, + 'azimuth_looks': azimuth_looks, + 'secondary_granule_date': datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'), + 'dem_name': 'GLO-30', + 'dem_pixel_spacing': '30 m', + 'apply_water_mask': apply_water_mask, + } + content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_readme.md.txt.j2', payload) + + output_file = product_dir / f'{product_name}_README.md.txt' + with open(output_file, 'w') as f: + f.write(content) + + +@dataclass +class ParameterFile: + reference_granule: str + secondary_granule: str + reference_orbit_direction: str + reference_orbit_number: str + secondary_orbit_direction: str + secondary_orbit_number: str + baseline: float + utc_time: float + heading: float + spacecraft_height: float + earth_radius_at_nadir: float + slant_range_near: float + slant_range_center: float + slant_range_far: float + range_looks: int + azimuth_looks: int + insar_phase_filter: bool + phase_filter_parameter: float + range_bandpass_filter: bool + azimuth_bandpass_filter: bool + dem_source: str + dem_resolution: int + unwrapping_type: str + speckle_filter: bool + water_mask: bool + radar_n_lines: Optional[int] = None + radar_n_samples: Optional[int] = None + radar_first_valid_line: Optional[int] = None + radar_n_valid_lines: Optional[int] = None + radar_first_valid_sample: Optional[int] = None + radar_n_valid_samples: Optional[int] = None + multilook_azimuth_time_interval: Optional[float] = None + multilook_range_pixel_size: Optional[float] = None + radar_sensing_stop: Optional[datetime] = None + + def __str__(self): + output_strings = [ + f'Reference Granule: {self.reference_granule}\n', + f'Secondary Granule: {self.secondary_granule}\n', + f'Reference Pass Direction: {self.reference_orbit_direction}\n', + f'Reference Orbit Number: {self.reference_orbit_number}\n', + f'Secondary Pass Direction: {self.secondary_orbit_direction}\n', + f'Secondary Orbit Number: {self.secondary_orbit_number}\n', + f'Baseline: {self.baseline}\n', + f'UTC time: {self.utc_time}\n', + f'Heading: {self.heading}\n', + f'Spacecraft height: {self.spacecraft_height}\n', + f'Earth radius at nadir: {self.earth_radius_at_nadir}\n', + f'Slant range near: {self.slant_range_near}\n', + f'Slant range center: {self.slant_range_center}\n', + f'Slant range far: {self.slant_range_far}\n', + f'Range looks: {self.range_looks}\n', + f'Azimuth looks: {self.azimuth_looks}\n', + f'INSAR phase filter: {"yes" if self.insar_phase_filter else "no"}\n', + f'Phase filter parameter: {self.phase_filter_parameter}\n', + f'Range bandpass filter: {"yes" if self.range_bandpass_filter else "no"}\n', + f'Azimuth bandpass filter: {"yes" if self.azimuth_bandpass_filter else "no"}\n', + f'DEM source: {self.dem_source}\n', + f'DEM resolution (m): {self.dem_resolution}\n', + f'Unwrapping type: {self.unwrapping_type}\n', + f'Speckle filter: {"yes" if self.speckle_filter else "no"}\n', + f'Water mask: {"yes" if self.water_mask else "no"}\n', + ] + + # TODO could use a more robust way to check if radar data is present + if self.radar_n_lines: + radar_data = [ + f'Radar n lines: {self.radar_n_lines}\n', + f'Radar n samples: {self.radar_n_samples}\n', + f'Radar first valid line: {self.radar_first_valid_line}\n', + f'Radar n valid lines: {self.radar_n_valid_lines}\n', + f'Radar first valid sample: {self.radar_first_valid_sample}\n', + f'Radar n valid samples: {self.radar_n_valid_samples}\n', + f'Multilook azimuth time interval: {self.multilook_azimuth_time_interval}\n', + f'Multilook range pixel size: {self.multilook_range_pixel_size}\n', + f'Radar sensing stop: {datetime.strftime(self.radar_sensing_stop, "%Y-%m-%dT%H:%M:%S.%f")}\n', + ] + output_strings += radar_data + + return ''.join(output_strings) + + def __repr__(self): + return self.__str__() + + def write(self, out_path: Path): + out_path.write_text(self.__str__()) + + +def make_parameter_file( + out_path: Path, + reference_scene: str, + secondary_scene: str, + azimuth_looks: int, + range_looks: int, + apply_water_mask: bool, + multilook_position: Optional[BurstPosition] = None, + swath_number: Optional[int] = None, + dem_name: str = 'GLO_30', + dem_resolution: int = 30, +) -> None: + """Create a parameter file for the output product + + Args: + out_path: path to output the parameter file + reference_scene: Reference burst name + secondary_scene: Secondary burst name + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + swath_number: Number of swath to grab bursts from (1, 2, or 3) for IW + multilook_position: Burst position for multilooked radar geometry products + dem_name: Name of the DEM that is use + dem_resolution: Resolution of the DEM + + returns: + None + """ + SPEED_OF_LIGHT = 299792458.0 + SPACECRAFT_HEIGHT = 693000.0 + EARTH_RADIUS = 6337286.638938101 + + parser = etree.XMLParser(encoding='utf-8', recover=True) + + ref_tag = reference_scene[-10:-6] + sec_tag = secondary_scene[-10:-6] + reference_safe = [file for file in os.listdir('.') if file.endswith(f'{ref_tag}.SAFE')][0] + secondary_safe = [file for file in os.listdir('.') if file.endswith(f'{sec_tag}.SAFE')][0] + + ref_annotation_path = f'{reference_safe}/annotation/' + ref_annotation = [file for file in os.listdir(ref_annotation_path) if os.path.isfile(ref_annotation_path + file)][0] + + ref_manifest_xml = etree.parse(f'{reference_safe}/manifest.safe', parser) + sec_manifest_xml = etree.parse(f'{secondary_safe}/manifest.safe', parser) + ref_annotation_xml = etree.parse(f'{ref_annotation_path}{ref_annotation}', parser) + topsProc_xml = etree.parse('topsProc.xml', parser) + topsApp_xml = etree.parse('topsApp.xml', parser) + + safe = '{http://www.esa.int/safe/sentinel-1.0}' + s1 = '{http://www.esa.int/safe/sentinel-1.0/sentinel-1}' + metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' + orbit_number_query = metadata_path + safe + 'orbitNumber' + orbit_direction_query = metadata_path + safe + 'extension//' + s1 + 'pass' + + ref_orbit_number = ref_manifest_xml.find(orbit_number_query).text + ref_orbit_direction = ref_manifest_xml.find(orbit_direction_query).text + sec_orbit_number = sec_manifest_xml.find(orbit_number_query).text + sec_orbit_direction = sec_manifest_xml.find(orbit_direction_query).text + ref_heading = float(ref_annotation_xml.find('.//platformHeading').text) + ref_time = ref_annotation_xml.find('.//productFirstLineUtcTime').text + slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) + range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) + number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) + baseline_perp = topsProc_xml.find(f'.//IW-{swath_number}_Bperp_at_midrange_for_first_common_burst').text + unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text + phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text + + slant_range_near = float(slant_range_time) * SPEED_OF_LIGHT / 2 + range_pixel_spacing = SPEED_OF_LIGHT / (2 * range_sampling_rate) + slant_range_far = slant_range_near + (number_samples - 1) * range_pixel_spacing + slant_range_center = (slant_range_near + slant_range_far) / 2 + + s = ref_time.split('T')[1].split(':') + utc_time = ((int(s[0]) * 60 + int(s[1])) * 60) + float(s[2]) + + parameter_file = ParameterFile( + reference_granule=reference_scene, + secondary_granule=secondary_scene, + reference_orbit_direction=ref_orbit_direction, + reference_orbit_number=ref_orbit_number, + secondary_orbit_direction=sec_orbit_direction, + secondary_orbit_number=sec_orbit_number, + baseline=float(baseline_perp), + utc_time=utc_time, + heading=ref_heading, + spacecraft_height=SPACECRAFT_HEIGHT, + earth_radius_at_nadir=EARTH_RADIUS, + slant_range_near=slant_range_near, + slant_range_center=slant_range_center, + slant_range_far=slant_range_far, + range_looks=int(range_looks), + azimuth_looks=int(azimuth_looks), + insar_phase_filter=True, + phase_filter_parameter=float(phase_filter_strength), + range_bandpass_filter=False, + azimuth_bandpass_filter=False, + dem_source=dem_name, + dem_resolution=dem_resolution, + unwrapping_type=unwrapper_type, + speckle_filter=True, + water_mask=apply_water_mask, + radar_n_lines=multilook_position.n_lines, + radar_n_samples=multilook_position.n_samples, + radar_first_valid_line=multilook_position.first_valid_line, + radar_n_valid_lines=multilook_position.n_valid_lines, + radar_first_valid_sample=multilook_position.first_valid_sample, + radar_n_valid_samples=multilook_position.n_valid_samples, + multilook_azimuth_time_interval=multilook_position.azimuth_time_interval, + multilook_range_pixel_size=multilook_position.range_pixel_size, + radar_sensing_stop=multilook_position.sensing_stop, + ) + parameter_file.write(out_path) + + +def upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix): + for browse in product_dir.glob('*.png'): + create_thumbnail(browse, output_dir=product_dir) + + upload_file_to_s3(Path(output_zip), bucket, bucket_prefix) + + for product_file in product_dir.iterdir(): + upload_file_to_s3(product_file, bucket, bucket_prefix) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index b06d2731..d80d9362 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -1,8 +1,5 @@ import shutil import subprocess -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path from typing import Optional import isceobj @@ -15,119 +12,6 @@ gdal.UseExceptions() -class GDALConfigManager: - """Context manager for setting GDAL config options temporarily""" - - def __init__(self, **options): - """ - Args: - **options: GDAL Config `option=value` keyword arguments. - """ - self.options = options.copy() - self._previous_options = {} - - def __enter__(self): - for key in self.options: - self._previous_options[key] = gdal.GetConfigOption(key) - - for key, value in self.options.items(): - gdal.SetConfigOption(key, value) - - def __exit__(self, exc_type, exc_val, exc_tb): - for key, value in self._previous_options.items(): - gdal.SetConfigOption(key, value) - - -@dataclass -class ParameterFile: - reference_granule: str - secondary_granule: str - reference_orbit_direction: str - reference_orbit_number: str - secondary_orbit_direction: str - secondary_orbit_number: str - baseline: float - utc_time: float - heading: float - spacecraft_height: float - earth_radius_at_nadir: float - slant_range_near: float - slant_range_center: float - slant_range_far: float - range_looks: int - azimuth_looks: int - insar_phase_filter: bool - phase_filter_parameter: float - range_bandpass_filter: bool - azimuth_bandpass_filter: bool - dem_source: str - dem_resolution: int - unwrapping_type: str - speckle_filter: bool - water_mask: bool - radar_n_lines: Optional[int] = None - radar_n_samples: Optional[int] = None - radar_first_valid_line: Optional[int] = None - radar_n_valid_lines: Optional[int] = None - radar_first_valid_sample: Optional[int] = None - radar_n_valid_samples: Optional[int] = None - multilook_azimuth_time_interval: Optional[float] = None - multilook_range_pixel_size: Optional[float] = None - radar_sensing_stop: Optional[datetime] = None - - def __str__(self): - output_strings = [ - f'Reference Granule: {self.reference_granule}\n', - f'Secondary Granule: {self.secondary_granule}\n', - f'Reference Pass Direction: {self.reference_orbit_direction}\n', - f'Reference Orbit Number: {self.reference_orbit_number}\n', - f'Secondary Pass Direction: {self.secondary_orbit_direction}\n', - f'Secondary Orbit Number: {self.secondary_orbit_number}\n', - f'Baseline: {self.baseline}\n', - f'UTC time: {self.utc_time}\n', - f'Heading: {self.heading}\n', - f'Spacecraft height: {self.spacecraft_height}\n', - f'Earth radius at nadir: {self.earth_radius_at_nadir}\n', - f'Slant range near: {self.slant_range_near}\n', - f'Slant range center: {self.slant_range_center}\n', - f'Slant range far: {self.slant_range_far}\n', - f'Range looks: {self.range_looks}\n', - f'Azimuth looks: {self.azimuth_looks}\n', - f'INSAR phase filter: {"yes" if self.insar_phase_filter else "no"}\n', - f'Phase filter parameter: {self.phase_filter_parameter}\n', - f'Range bandpass filter: {"yes" if self.range_bandpass_filter else "no"}\n', - f'Azimuth bandpass filter: {"yes" if self.azimuth_bandpass_filter else "no"}\n', - f'DEM source: {self.dem_source}\n', - f'DEM resolution (m): {self.dem_resolution}\n', - f'Unwrapping type: {self.unwrapping_type}\n', - f'Speckle filter: {"yes" if self.speckle_filter else "no"}\n', - f'Water mask: {"yes" if self.water_mask else "no"}\n', - ] - - # TODO could use a more robust way to check if radar data is present - if self.radar_n_lines: - radar_data = [ - f'Radar n lines: {self.radar_n_lines}\n', - f'Radar n samples: {self.radar_n_samples}\n', - f'Radar first valid line: {self.radar_first_valid_line}\n', - f'Radar n valid lines: {self.radar_n_valid_lines}\n', - f'Radar first valid sample: {self.radar_first_valid_sample}\n', - f'Radar n valid samples: {self.radar_n_valid_samples}\n', - f'Multilook azimuth time interval: {self.multilook_azimuth_time_interval}\n', - f'Multilook range pixel size: {self.multilook_range_pixel_size}\n', - f'Radar sensing stop: {datetime.strftime(self.radar_sensing_stop, "%Y-%m-%dT%H:%M:%S.%f")}\n', - ] - output_strings += radar_data - - return ''.join(output_strings) - - def __repr__(self): - return self.__str__() - - def write(self, out_path: Path): - out_path.write_text(self.__str__()) - - def utm_from_lon_lat(lon: float, lat: float) -> int: """Get the UTM zone EPSG code from a longitude and latitude. See https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system @@ -165,20 +49,6 @@ def extent_from_geotransform(geotransform: tuple, x_size: int, y_size: int) -> t return extent -def make_browse_image(input_tif: str, output_png: str) -> None: - with GDALConfigManager(GDAL_PAM_ENABLED='NO'): - stats = gdal.Info(input_tif, format='json', stats=True)['stac']['raster:bands'][0]['stats'] - gdal.Translate( - destName=output_png, - srcDS=input_tif, - format='png', - outputType=gdal.GDT_Byte, - width=2048, - strict=True, - scaleParams=[[stats['minimum'], stats['maximum']]], - ) - - def oldest_granule_first(g1, g2): if g1[14:29] <= g2[14:29]: return g1, g2 @@ -203,7 +73,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i:: image_obj.bands] + new_array[i, :, :] = array[i :: image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -368,7 +238,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i:: image_obj.bands] = array[i, :, :] + new_array[i :: image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') From 865e442e1f2f7097dd61797c7ab18ad934d6be75 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:34:16 +0000 Subject: [PATCH 05/81] update coverage image --- images/coverage.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 6c15cace..dfb99a19 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -9,13 +9,13 @@ - + coverage coverage - 75% - 75% + 67% + 67% From a34e832f29b77aff382d5282a88305c48ed437db Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Aug 2024 13:47:17 -0500 Subject: [PATCH 06/81] working single burst --- src/hyp3_isce2/insar_tops.py | 51 ++++++++++++--------- src/hyp3_isce2/insar_tops_burst.py | 28 ++++++++---- src/hyp3_isce2/insar_tops_multi_bursts.py | 34 ++++++++------ src/hyp3_isce2/merge_tops_bursts.py | 3 +- src/hyp3_isce2/metadata/util.py | 7 --- src/hyp3_isce2/packaging.py | 56 +++++++++++++++-------- src/hyp3_isce2/utils.py | 8 +++- 7 files changed, 114 insertions(+), 73 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 57e949aa..0e1fe26e 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -6,6 +6,7 @@ from pathlib import Path from shutil import copyfile, make_archive +from hyp3lib.util import string_is_true from s1_orbits import fetch_for_scene from hyp3_isce2 import packaging, slc, topsapp @@ -92,18 +93,22 @@ def insar_tops_packaged( range_looks: int = 20, apply_water_mask: bool = True, download: bool = True, + bucket: str = None, + bucket_prefix: str = '', ) -> Path: """Create a full-SLC interferogram Args: - reference_scene: Reference SLC name - secondary_scene: Secondary SLC name + reference: Reference SLC name + secondary: Secondary SLC name swaths: Swaths to process polarization: Polarization to use azimuth_looks: Number of azimuth looks range_looks: Number of range looks apply_water_mask: Apply water mask to unwrapped phase download: Download the SLCs + bucket: AWS S3 bucket to upload the final product to + bucket_prefix: Bucket prefix to prefix to use when uploading the final product Returns: Path to the output files @@ -140,46 +145,48 @@ def insar_tops_packaged( secondary_scene=secondary, azimuth_looks=azimuth_looks, range_looks=range_looks, - # swath_number=swath_number, - # multilook_position=multilook_position, apply_water_mask=apply_water_mask, ) output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + if bucket: + packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) def main(): """HyP3 entrypoint for the SLC TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - parser.add_argument('--reference-scene', type=str, required=True) - parser.add_argument('--secondary-scene', type=str, required=True) - parser.add_argument('--polarization', type=str, choices=['VV', 'HH'], default='VV') + parser.add_argument('--reference', type=str, help='Reference granule') + parser.add_argument('--secondary', type=str, help='Secondary granule') + parser.add_argument('--polarization', type=str, defualt='VV', help='Polarization to use') parser.add_argument( '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' ) + parser.add_argument( + '--apply-water-mask', + type=string_is_true, + default=False, + help='Apply a water body mask before unwrapping.', + ) + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') args = parser.parse_args() - configure_root_logger() log.debug(' '.join(sys.argv)) - log.info('Begin ISCE2 TopsApp run') - range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] - isce_output_dir = insar_tops( - reference_scene=args.reference_scene, - secondary_scene=args.secondary_scene, + if args.polarization not in ['VV', 'VH', 'HV', 'HH']: + raise ValueError('Polarization must be one of VV, VH, HV, or HH') + + insar_tops_packaged( + reference_scene=args.reference, + secondary_scene=args.secondary, polarization=args.polarization, azimuth_looks=azimuth_looks, range_looks=range_looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, ) log.info('ISCE2 TopsApp run completed successfully') - - product_name = f'{args.reference_scene}x{args.secondary_scene}' - output_zip = make_archive(base_name=product_name, format='zip', base_dir=isce_output_dir) - - if args.bucket: - upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 08922e4d..95abd517 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -24,7 +24,7 @@ validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.insar_tops import insar_tops +from hyp3_isce2.insar_tops import insar_tops_packaged from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( @@ -165,17 +165,16 @@ def insar_tops_single_burst( multilook_position = multilook_radar_merge_inputs(swath_number, rg_looks=range_looks, az_looks=azimuth_looks) pixel_size = packaging.get_pixel_size(looks) - product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size), slc=False) product_dir = Path(product_name) product_dir.mkdir(parents=True, exist_ok=True) packaging.translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' if apply_water_mask: - unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' - water_mask = f'{product_name}/{product_name}_water_mask.tif' - packaging.water_mask(unwrapped_phase, water_mask) + packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') packaging.make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') @@ -194,7 +193,6 @@ def insar_tops_single_burst( secondary_scene=secondary, azimuth_looks=azimuth_looks, range_looks=range_looks, - swath_number=swath_number, multilook_position=multilook_position, apply_water_mask=apply_water_mask, ) @@ -237,7 +235,14 @@ def insar_tops_multi_burst( secondary_safe = secondary_safe_path.name.split('.')[0] log.info('Begin ISCE2 TopsApp run') - insar_tops(reference_safe, secondary_safe, download=False) + insar_tops_packaged( + reference=reference_safe, + secondary=secondary_safe, + looks=looks, + apply_water_mask=apply_water_mask, + bucket=bucket, + bucket_prefix=bucket_prefix + ) log.info('ISCE2 TopsApp run completed successfully') @@ -282,4 +287,11 @@ def main(): bucket_prefix=args.bucket_prefix, ) else: - insar_tops_multi_burst() + insar_tops_multi_burst( + reference=args.reference[0], + secondary=args.secondary[0], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) diff --git a/src/hyp3_isce2/insar_tops_multi_bursts.py b/src/hyp3_isce2/insar_tops_multi_bursts.py index 5666f170..957c5ce2 100644 --- a/src/hyp3_isce2/insar_tops_multi_bursts.py +++ b/src/hyp3_isce2/insar_tops_multi_bursts.py @@ -23,11 +23,7 @@ import hyp3_isce2 from hyp3_isce2.insar_tops import insar_tops from hyp3_isce2.insar_tops_burst import convert_raster_from_isce2_gdal, find_product, get_pixel_size -from hyp3_isce2.utils import ( - ParameterFile, - make_browse_image, - utm_from_lon_lat, -) +from hyp3_isce2.utils import ParameterFile, get_projection, make_browse_image, utm_from_lon_lat log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' @@ -312,7 +308,7 @@ def make_readme( 'plugin_version': hyp3_isce2.__version__, 'processor_name': isce.__name__.upper(), 'processor_version': isce.__version__, - 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'projection': get_projection(info['coordinateSystem']['wkt']), 'pixel_spacing': info['geoTransform'][1], 'product_name': product_name, 'reference_burst_name': reference_scene, @@ -351,15 +347,25 @@ def main(): granules_ref = list(set(args.reference)) granules_sec = list(set(args.secondary)) - ids_ref = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_ref] - ids_sec = [granule.split('_')[1]+'_'+granule.split('_')[2]+'_'+granule.split('_')[4] for granule in granules_sec] + ids_ref = [ + granule.split('_')[1] + '_' + granule.split('_')[2] + '_' + granule.split('_')[4] for granule in granules_ref + ] + ids_sec = [ + granule.split('_')[1] + '_' + granule.split('_')[2] + '_' + granule.split('_')[4] for granule in granules_sec + ] - if len(list(set(ids_ref)-set(ids_sec))) > 0: - raise Exception('The reference bursts ' + ', '.join(list(set(ids_ref)-set(ids_sec))) + - ' do not have the correspondant bursts in the secondary granules') - elif len(list(set(ids_sec)-set(ids_ref))) > 0: - raise Exception('The secondary bursts ' + ', '.join(list(set(ids_sec)-set(ids_ref))) + - ' do not have the correspondant bursts in the reference granules') + if len(list(set(ids_ref) - set(ids_sec))) > 0: + raise Exception( + 'The reference bursts ' + + ', '.join(list(set(ids_ref) - set(ids_sec))) + + ' do not have the correspondant bursts in the secondary granules' + ) + elif len(list(set(ids_sec) - set(ids_ref))) > 0: + raise Exception( + 'The secondary bursts ' + + ', '.join(list(set(ids_sec) - set(ids_ref))) + + ' do not have the correspondant bursts in the reference granules' + ) if not granules_ref[0].split('_')[4] == granules_sec[0].split('_')[4]: raise Exception('The secondary and reference granules do not have the same polarization') diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index f3ef8457..b3c1f667 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -42,6 +42,7 @@ from hyp3_isce2.utils import ( ParameterFile, create_image, + get_projection, image_math, load_product, make_browse_image, @@ -1027,7 +1028,7 @@ def make_readme( 'plugin_version': hyp3_isce2.__version__, 'processor_name': isce.__name__.upper(), # noqa 'processor_version': isce.__version__, # noqa - 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'projection': get_projection(info['coordinateSystem']['wkt']), 'pixel_spacing': info['geoTransform'][1], 'product_name': product_name, 'reference_burst_name': ', '.join(reference_scenes), diff --git a/src/hyp3_isce2/metadata/util.py b/src/hyp3_isce2/metadata/util.py index d8ada9a7..1f238b0c 100644 --- a/src/hyp3_isce2/metadata/util.py +++ b/src/hyp3_isce2/metadata/util.py @@ -1,5 +1,4 @@ from jinja2 import Environment, PackageLoader, StrictUndefined, select_autoescape -from osgeo import osr def get_environment() -> Environment: @@ -19,9 +18,3 @@ def render_template(template: str, payload: dict) -> str: template = env.get_template(template) rendered = template.render(payload) return rendered - - -def get_projection(srs_wkt) -> str: - srs = osr.SpatialReference() - srs.ImportFromWkt(srs_wkt) - return srs.GetAttrValue('projcs') diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index e53ea3b6..394c49e3 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -14,8 +14,9 @@ from pyproj import CRS import hyp3_isce2 +import hyp3_isce2.metadata.util from hyp3_isce2.burst import BurstPosition -from hyp3_isce2.utils import utm_from_lon_lat +from hyp3_isce2.utils import get_projection, utm_from_lon_lat @dataclass @@ -44,21 +45,21 @@ def find_product(pattern: str) -> str: return product -def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: int, slc: bool = True) -> str: +def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bool = True) -> str: """Get the name of the interferogram product. Args: - reference_scene: The reference burst name. - secondary_scene: The secondary burst name. + reference: The reference burst name. + secondary: The secondary burst name. pixel_spacing: The spacing of the pixels in the output image. slc: Whether the input scenes are SLCs or bursts. Returns: The name of the interferogram product. """ - - reference_split = reference_scene.split('_') - secondary_split = secondary_scene.split('_') + breakpoint() + reference_split = reference.split('_') + secondary_split = secondary.split('_') if slc: platform = reference_split[0] @@ -328,7 +329,7 @@ def make_readme( 'plugin_version': hyp3_isce2.__version__, 'processor_name': isce.__name__.upper(), 'processor_version': isce.__version__, - 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'projection': get_projection(info['coordinateSystem']['wkt']), 'pixel_spacing': info['geoTransform'][1], 'product_name': product_name, 'reference_burst_name': reference_scene, @@ -437,6 +438,20 @@ def write(self, out_path: Path): out_path.write_text(self.__str__()) +def find_available_swaths(base_dir: Path | str) -> list[str]: + """Find the available swaths in the given directory + + Args: + base_dir: Path to the directory containing the swaths + + Returns: + List of available swaths + """ + geom_dir = Path(base_dir) / 'geom_reference' + swaths = sorted([file.name for file in geom_dir.iterdir() if file.is_dir()]) + return swaths + + def make_parameter_file( out_path: Path, reference_scene: str, @@ -445,7 +460,6 @@ def make_parameter_file( range_looks: int, apply_water_mask: bool, multilook_position: Optional[BurstPosition] = None, - swath_number: Optional[int] = None, dem_name: str = 'GLO_30', dem_resolution: int = 30, ) -> None: @@ -457,7 +471,6 @@ def make_parameter_file( secondary_scene: Secondary burst name azimuth_looks: Number of azimuth looks range_looks: Number of range looks - swath_number: Number of swath to grab bursts from (1, 2, or 3) for IW multilook_position: Burst position for multilooked radar geometry products dem_name: Name of the DEM that is use dem_resolution: Resolution of the DEM @@ -500,7 +513,8 @@ def make_parameter_file( slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) - baseline_perp = topsProc_xml.find(f'.//IW-{swath_number}_Bperp_at_midrange_for_first_common_burst').text + min_swath = find_available_swaths(Path.cwd())[0] + baseline_perp = topsProc_xml.find(f'.//IW-{int(min_swath[2])}_Bperp_at_midrange_for_first_common_burst').text unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text @@ -538,16 +552,18 @@ def make_parameter_file( unwrapping_type=unwrapper_type, speckle_filter=True, water_mask=apply_water_mask, - radar_n_lines=multilook_position.n_lines, - radar_n_samples=multilook_position.n_samples, - radar_first_valid_line=multilook_position.first_valid_line, - radar_n_valid_lines=multilook_position.n_valid_lines, - radar_first_valid_sample=multilook_position.first_valid_sample, - radar_n_valid_samples=multilook_position.n_valid_samples, - multilook_azimuth_time_interval=multilook_position.azimuth_time_interval, - multilook_range_pixel_size=multilook_position.range_pixel_size, - radar_sensing_stop=multilook_position.sensing_stop, ) + if multilook_position: + parameter_file.radar_n_lines = multilook_position.n_lines + parameter_file.radar_n_samples = multilook_position.n_samples + parameter_file.radar_first_valid_line = multilook_position.first_valid_line + parameter_file.radar_n_valid_lines = multilook_position.n_valid_lines + parameter_file.radar_first_valid_sample = multilook_position.first_valid_sample + parameter_file.radar_n_valid_samples = multilook_position.n_valid_samples + parameter_file.multilook_azimuth_time_interval = multilook_position.azimuth_time_interval + parameter_file.multilook_range_pixel_size = multilook_position.range_pixel_size + parameter_file.radar_sensing_stop = multilook_position.sensing_stop + parameter_file.write(out_path) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index d80d9362..798305da 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -6,7 +6,7 @@ import numpy as np from isceobj.Util.ImageUtil.ImageLib import loadImage from iscesys.Component.ProductManager import ProductManager -from osgeo import gdal +from osgeo import gdal, osr gdal.UseExceptions() @@ -315,3 +315,9 @@ def read_product_metadata(meta_file_path: str) -> dict: value = ':'.join(values) hyp3_meta[key] = value return hyp3_meta + + +def get_projection(srs_wkt) -> str: + srs = osr.SpatialReference() + srs.ImportFromWkt(srs_wkt) + return srs.GetAttrValue('projcs') From 683052d7d0525849f639bc3dfd163f1b9c81db14 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Aug 2024 14:07:14 -0500 Subject: [PATCH 07/81] should work but dependency issues --- environment.yml | 1 + pyproject.toml | 1 + src/hyp3_isce2/insar_tops_burst.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index eaff6208..39747d51 100644 --- a/environment.yml +++ b/environment.yml @@ -14,6 +14,7 @@ dependencies: - asf_search>=6.4.0 - gdal - opencv + - burst2safe>=1.0.0 # For packaging, and testing - flake8 - flake8-import-order diff --git a/pyproject.toml b/pyproject.toml index 8d7002b4..7be9bf0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "gdal", "hyp3lib>=3,<4", "s1_orbits", + "burst2safe>=1.0.0" # Conda-forge only dependencies are listed below # "opencv", # "isce2>=2.6.3", diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 95abd517..d416dac0 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -8,7 +8,7 @@ from typing import Iterable, Optional import isce # noqa -from burst2safe import burst2safe +from burst2safe.burst2safe import burst2safe from hyp3lib.util import string_is_true from isceobj.TopsProc.runMergeBursts import multilook from osgeo import gdal @@ -288,8 +288,8 @@ def main(): ) else: insar_tops_multi_burst( - reference=args.reference[0], - secondary=args.secondary[0], + reference=args.reference, + secondary=args.secondary, looks=args.looks, apply_water_mask=args.apply_water_mask, bucket=args.bucket, From 3d69568df31cf6742df794af5cc342cc04a36448 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Aug 2024 14:10:10 -0500 Subject: [PATCH 08/81] remove duplicated script --- src/hyp3_isce2/insar_tops_multi_bursts.py | 450 ---------------------- 1 file changed, 450 deletions(-) delete mode 100644 src/hyp3_isce2/insar_tops_multi_bursts.py diff --git a/src/hyp3_isce2/insar_tops_multi_bursts.py b/src/hyp3_isce2/insar_tops_multi_bursts.py deleted file mode 100644 index 957c5ce2..00000000 --- a/src/hyp3_isce2/insar_tops_multi_bursts.py +++ /dev/null @@ -1,450 +0,0 @@ -"""A workflow for merging standard burst InSAR products.""" -import argparse -import copy -import logging -import os -import subprocess -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from secrets import token_hex -from shutil import make_archive -from typing import Iterable, Optional - -import isce -from burst2safe.burst2safe import burst2safe -from hyp3lib.aws import upload_file_to_s3 -from hyp3lib.image import create_thumbnail -from hyp3lib.util import string_is_true -from lxml import etree -from osgeo import gdal, gdalconst - -import hyp3_isce2 -from hyp3_isce2.insar_tops import insar_tops -from hyp3_isce2.insar_tops_burst import convert_raster_from_isce2_gdal, find_product, get_pixel_size -from hyp3_isce2.utils import ParameterFile, get_projection, make_browse_image, utm_from_lon_lat - - -log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -logging.basicConfig(stream=sys.stdout, format=log_format, level=logging.INFO, force=True) -log = logging.getLogger(__name__) - - -@dataclass -class ISCE2Dataset: - name: str - suffix: str - band: Iterable[int] - dtype: Optional[int] = gdalconst.GDT_Float32 - - -def get_product_name(reference_scene: str, secondary_scene: str, pixel_spacing: int) -> str: - """Get the name of the interferogram product. - - Args: - reference_scene: The reference burst name. - secondary_scene: The secondary burst name. - pixel_spacing: The spacing of the pixels in the output image. - - Returns: - The name of the interferogram product. - """ - - reference_split = reference_scene.split('_') - secondary_split = secondary_scene.split('_') - - platform = reference_split[0][0:2] - reference_date = reference_split[5][0:8] - secondary_date = secondary_split[5][0:8] - product_type = 'INT' - pixel_spacing = str(int(pixel_spacing)) - product_id = token_hex(2).upper() - - return '_'.join( - [ - platform, - reference_date, - secondary_date, - product_type + pixel_spacing, - product_id, - ] - ) - - -def translate_outputs(product_name: str, pixel_size: float, include_radar: bool = False) -> None: - """Translate ISCE outputs to a standard GTiff format with a UTM projection. - Assume you are in the top level of an ISCE run directory - - Args: - product_name: Name of the product - pixel_size: Pixel size - include_radar: Flag to include the full resolution radar geometry products in the output - """ - - src_ds = gdal.Open('merged/filt_topophase.unw.geo') - src_geotransform = src_ds.GetGeoTransform() - src_projection = src_ds.GetProjection() - - target_ds = gdal.Open('merged/dem.crop', gdal.GA_Update) - target_ds.SetGeoTransform(src_geotransform) - target_ds.SetProjection(src_projection) - - del src_ds, target_ds - - datasets = [ - ISCE2Dataset('merged/filt_topophase.unw.geo', 'unw_phase', [2]), - ISCE2Dataset('merged/phsig.cor.geo', 'corr', [1]), - ISCE2Dataset('merged/dem.crop', 'dem', [1]), - ISCE2Dataset('merged/filt_topophase.unw.conncomp.geo', 'conncomp', [1]), - ] - - rdr_datasets = [ - ISCE2Dataset( - find_product('merged/filt_topophase.flat.vrt'), - 'wrapped_phase_rdr', - [1], - gdalconst.GDT_CFloat32, - ), - ISCE2Dataset(find_product('merged/lat.rdr.full.vrt'), 'lat_rdr', [1]), - ISCE2Dataset(find_product('merged/lon.rdr.full.vrt'), 'lon_rdr', [1]), - ISCE2Dataset(find_product('merged/los.rdr.full.vrt'), 'los_rdr', [1, 2]), - ] - if include_radar: - datasets += rdr_datasets - - for dataset in datasets: - out_file = str(Path(product_name) / f'{product_name}_{dataset.suffix}.tif') - gdal.Translate( - destName=out_file, - srcDS=dataset.name, - bandList=dataset.band, - format='GTiff', - outputType=dataset.dtype, - noData=0, - creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], - ) - - # Use numpy.angle to extract the phase component of the complex wrapped interferogram - wrapped_phase = ISCE2Dataset('filt_topophase.flat.geo', 'wrapped_phase', 1) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{wrapped_phase.suffix}.tif ' - f'-A merged/{wrapped_phase.name} --A_band={wrapped_phase.band} ' - '--calc angle(A) --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' - ) - subprocess.run(cmd.split(' '), check=True) - - ds = gdal.Open('merged/los.rdr.geo', gdal.GA_Update) - ds.GetRasterBand(1).SetNoDataValue(0) - ds.GetRasterBand(2).SetNoDataValue(0) - del ds - - # Performs the inverse of the operation performed by MintPy: - # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L732-L737 - # First subtract the incidence angle from ninety degrees to go from sensor-to-ground to ground-to-sensor, - # then convert to radians - incidence_angle = ISCE2Dataset('los.rdr.geo', 'lv_theta', 1) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{incidence_angle.suffix}.tif ' - f'-A merged/{incidence_angle.name} --A_band={incidence_angle.band} ' - '--calc (90-A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' - ) - subprocess.run(cmd.split(' '), check=True) - - # Performs the inverse of the operation performed by MintPy: - # https://github.com/insarlab/MintPy/blob/df96e0b73f13cc7e2b6bfa57d380963f140e3159/src/mintpy/objects/stackDict.py#L739-L745 - # First add ninety degrees to the azimuth angle to go from angle-from-east to angle-from-north, - # then convert to radians - azimuth_angle = ISCE2Dataset('los.rdr.geo', 'lv_phi', 2) - cmd = ( - 'gdal_calc.py ' - f'--outfile {product_name}/{product_name}_{azimuth_angle.suffix}.tif ' - f'-A merged/{azimuth_angle.name} --A_band={azimuth_angle.band} ' - '--calc (90+A)*pi/180 --type Float32 --format GTiff --NoDataValue=0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' - ) - subprocess.run(cmd.split(' '), check=True) - - ds = gdal.Open('merged/filt_topophase.unw.geo') - geotransform = ds.GetGeoTransform() - del ds - - epsg = utm_from_lon_lat(geotransform[0], geotransform[3]) - files = [str(path) for path in Path(product_name).glob('*.tif') if not path.name.endswith('rdr.tif')] - for file in files: - gdal.Warp( - file, - file, - dstSRS=f'epsg:{epsg}', - creationOptions=['TILED=YES', 'COMPRESS=LZW', 'NUM_THREADS=ALL_CPUS'], - xRes=pixel_size, - yRes=pixel_size, - targetAlignedPixels=True, - ) - - -def make_parameter_file( - out_path: Path, - reference_scene: str, - secondary_scene: str, - azimuth_looks: int, - range_looks: int, - apply_water_mask: bool, - dem_name: str = 'GLO_30', - dem_resolution: int = 30, -) -> None: - """Create a parameter file for the output product - - Args: - out_path: path to output the parameter file - reference_scene: Reference burst name - secondary_scene: Secondary burst name - azimuth_looks: Number of azimuth looks - range_looks: Number of range looks - dem_name: Name of the DEM that is use - dem_resolution: Resolution of the DEM - - returns: - None - """ - SPEED_OF_LIGHT = 299792458.0 - SPACECRAFT_HEIGHT = 693000.0 - EARTH_RADIUS = 6337286.638938101 - - parser = etree.XMLParser(encoding='utf-8', recover=True) - - ref_tag = reference_scene[-4::] - sec_tag = secondary_scene[-4::] - print(ref_tag, sec_tag) - reference_safe = [file for file in os.listdir('.') if file.endswith(f'{ref_tag}.SAFE')][0] - secondary_safe = [file for file in os.listdir('.') if file.endswith(f'{sec_tag}.SAFE')][0] - - ref_annotation_path = f'{reference_safe}/annotation/' - ref_annotation = [file for file in os.listdir(ref_annotation_path) if os.path.isfile(ref_annotation_path + file)][0] - - ref_manifest_xml = etree.parse(f'{reference_safe}/manifest.safe', parser) - sec_manifest_xml = etree.parse(f'{secondary_safe}/manifest.safe', parser) - ref_annotation_xml = etree.parse(f'{ref_annotation_path}{ref_annotation}', parser) - topsProc_xml = etree.parse('topsProc.xml', parser) - topsApp_xml = etree.parse('topsApp.xml', parser) - - safe = '{http://www.esa.int/safe/sentinel-1.0}' - s1 = '{http://www.esa.int/safe/sentinel-1.0/sentinel-1}' - metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' - orbit_number_query = metadata_path + safe + 'orbitNumber' - orbit_direction_query = metadata_path + safe + 'extension//' + s1 + 'pass' - - ref_orbit_number = ref_manifest_xml.find(orbit_number_query).text - ref_orbit_direction = ref_manifest_xml.find(orbit_direction_query).text - sec_orbit_number = sec_manifest_xml.find(orbit_number_query).text - sec_orbit_direction = sec_manifest_xml.find(orbit_direction_query).text - ref_heading = float(ref_annotation_xml.find('.//platformHeading').text) - ref_time = ref_annotation_xml.find('.//productFirstLineUtcTime').text - slant_range_time = float(ref_annotation_xml.find('.//slantRangeTime').text) - range_sampling_rate = float(ref_annotation_xml.find('.//rangeSamplingRate').text) - number_samples = int(ref_annotation_xml.find('.//swathTiming/samplesPerBurst').text) - baseline_perp = topsProc_xml.find('.//IW-2_Bperp_at_midrange_for_first_common_burst').text - unwrapper_type = topsApp_xml.find('.//property[@name="unwrapper name"]').text - phase_filter_strength = topsApp_xml.find('.//property[@name="filter strength"]').text - - slant_range_near = float(slant_range_time) * SPEED_OF_LIGHT / 2 - range_pixel_spacing = SPEED_OF_LIGHT / (2 * range_sampling_rate) - slant_range_far = slant_range_near + (number_samples - 1) * range_pixel_spacing - slant_range_center = (slant_range_near + slant_range_far) / 2 - - s = ref_time.split('T')[1].split(':') - utc_time = ((int(s[0]) * 60 + int(s[1])) * 60) + float(s[2]) - - parameter_file = ParameterFile( - reference_granule=reference_scene, - secondary_granule=secondary_scene, - reference_orbit_direction=ref_orbit_direction, - reference_orbit_number=ref_orbit_number, - secondary_orbit_direction=sec_orbit_direction, - secondary_orbit_number=sec_orbit_number, - baseline=float(baseline_perp), - utc_time=utc_time, - heading=ref_heading, - spacecraft_height=SPACECRAFT_HEIGHT, - earth_radius_at_nadir=EARTH_RADIUS, - slant_range_near=slant_range_near, - slant_range_center=slant_range_center, - slant_range_far=slant_range_far, - range_looks=int(range_looks), - azimuth_looks=int(azimuth_looks), - insar_phase_filter=True, - phase_filter_parameter=float(phase_filter_strength), - range_bandpass_filter=False, - azimuth_bandpass_filter=False, - dem_source=dem_name, - dem_resolution=dem_resolution, - unwrapping_type=unwrapper_type, - speckle_filter=True, - water_mask=apply_water_mask, - ) - parameter_file.write(out_path) - - -def make_readme( - product_dir: Path, - product_name: str, - reference_scene: str, - secondary_scene: str, - range_looks: int, - azimuth_looks: int, - apply_water_mask: bool, -) -> None: - wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' - info = gdal.Info(str(wrapped_phase_path), format='json') - secondary_granule_datetime_str = secondary_scene.split('_')[5] - - payload = { - 'processing_date': datetime.now(timezone.utc), - 'plugin_name': hyp3_isce2.__name__, - 'plugin_version': hyp3_isce2.__version__, - 'processor_name': isce.__name__.upper(), - 'processor_version': isce.__version__, - 'projection': get_projection(info['coordinateSystem']['wkt']), - 'pixel_spacing': info['geoTransform'][1], - 'product_name': product_name, - 'reference_burst_name': reference_scene, - 'secondary_burst_name': secondary_scene, - 'range_looks': range_looks, - 'azimuth_looks': azimuth_looks, - 'secondary_granule_date': datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'), - 'dem_name': 'GLO-30', - 'dem_pixel_spacing': '30 m', - 'apply_water_mask': apply_water_mask, - } - content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_readme.md.txt.j2', payload) - - output_file = product_dir / f'{product_name}_README.md.txt' - with open(output_file, 'w') as f: - f.write(content) - - -def main(): - """HyP3 entrypoint for the TOPS burst merging workflow""" - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - parser.add_argument('--reference', nargs='*', help='List of granules for the reference bursts') - parser.add_argument('--secondary', nargs='*', help='List of granules for the secondary bursts') - parser.add_argument( - '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' - ) - parser.add_argument( - '--apply-water-mask', - type=string_is_true, - default=False, - help='Apply a water body mask before unwrapping.', - ) - args = parser.parse_args() - granules_ref = list(set(args.reference)) - granules_sec = list(set(args.secondary)) - - ids_ref = [ - granule.split('_')[1] + '_' + granule.split('_')[2] + '_' + granule.split('_')[4] for granule in granules_ref - ] - ids_sec = [ - granule.split('_')[1] + '_' + granule.split('_')[2] + '_' + granule.split('_')[4] for granule in granules_sec - ] - - if len(list(set(ids_ref) - set(ids_sec))) > 0: - raise Exception( - 'The reference bursts ' - + ', '.join(list(set(ids_ref) - set(ids_sec))) - + ' do not have the correspondant bursts in the secondary granules' - ) - elif len(list(set(ids_sec) - set(ids_ref))) > 0: - raise Exception( - 'The secondary bursts ' - + ', '.join(list(set(ids_sec) - set(ids_ref))) - + ' do not have the correspondant bursts in the reference granules' - ) - - if not granules_ref[0].split('_')[4] == granules_sec[0].split('_')[4]: - raise Exception('The secondary and reference granules do not have the same polarization') - - if granules_ref[0].split('_')[3] > granules_sec[0].split('_')[3]: - log.info('The secondary granules have a later date than the reference granules.') - temp = copy.copy(granules_ref) - granules_ref = copy.copy(granules_sec) - granules_sec = temp - - swaths = list(set([int(granule.split('_')[2][2]) for granule in granules_ref])) - - reference_scene = burst2safe(granules_ref) - reference_scene = os.path.basename(reference_scene).split('.')[0] - - secondary_scene = burst2safe(granules_sec) - secondary_scene = os.path.basename(secondary_scene).split('.')[0] - - polarization = granules_ref[0].split('_')[4] - - range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] - apply_water_mask = args.apply_water_mask - - insar_tops(reference_scene, secondary_scene, swaths, polarization, download=False) - - pixel_size = get_pixel_size(args.looks) - product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) - - product_dir = Path(product_name) - product_dir.mkdir(parents=True, exist_ok=True) - - translate_outputs(product_name, pixel_size=pixel_size, include_radar=True) - - unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' - water_mask = f'{product_name}/{product_name}_water_mask.tif' - - if apply_water_mask: - convert_raster_from_isce2_gdal('water_mask.wgs84', unwrapped_phase, water_mask) - cmd = ( - 'gdal_calc.py ' - f'--outfile {unwrapped_phase} ' - f'-A {unwrapped_phase} -B {water_mask} ' - '--calc A*B ' - '--overwrite ' - '--NoDataValue 0 ' - '--creation-option TILED=YES --creation-option COMPRESS=LZW --creation-option NUM_THREADS=ALL_CPUS' - ) - subprocess.run(cmd.split(' '), check=True) - - make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') - - make_readme( - product_dir=product_dir, - product_name=product_name, - reference_scene=reference_scene, - secondary_scene=secondary_scene, - range_looks=range_looks, - azimuth_looks=azimuth_looks, - apply_water_mask=apply_water_mask, - ) - make_parameter_file( - Path(f'{product_name}/{product_name}.txt'), - reference_scene=reference_scene, - secondary_scene=secondary_scene, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=apply_water_mask, - ) - output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - - if args.bucket: - for browse in product_dir.glob('*.png'): - create_thumbnail(browse, output_dir=product_dir) - - upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix) - - for product_file in product_dir.iterdir(): - upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) - - -if __name__ == '__main__': - main() From 504e5485cbca6166021ba4c4bd4cbbba5fd45406 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Tue, 13 Aug 2024 17:58:57 -0800 Subject: [PATCH 09/81] Fixing small issues --- environment.yml | 5 ++--- src/hyp3_isce2/insar_tops.py | 2 +- src/hyp3_isce2/insar_tops_burst.py | 16 +++++++++++++--- src/hyp3_isce2/packaging.py | 12 ++++++++---- src/hyp3_isce2/slc.py | 12 ++++++++++++ 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/environment.yml b/environment.yml index 39747d51..dec70987 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: hyp3-isce2 +name: hyp3-isce2-forr channels: - conda-forge - nodefaults @@ -14,7 +14,7 @@ dependencies: - asf_search>=6.4.0 - gdal - opencv - - burst2safe>=1.0.0 + - burst2safe # For packaging, and testing - flake8 - flake8-import-order @@ -30,4 +30,3 @@ dependencies: # For running - hyp3lib>=3,<4 - s1_orbits - - burst2safe diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 0e1fe26e..2074866b 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -117,7 +117,7 @@ def insar_tops_packaged( product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) log.info('Begin ISCE2 TopsApp run') - insar_tops(reference, secondary, download=False) + #insar_tops(reference, secondary, download=False) log.info('ISCE2 TopsApp run completed successfully') product_dir = Path(product_name) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index d416dac0..12c89207 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -205,6 +205,7 @@ def insar_tops_single_burst( def insar_tops_multi_burst( reference: Iterable[str], secondary: Iterable[str], + swaths: list = [1, 2, 3], looks: str = '20x4', apply_water_mask=False, bucket: Optional[str] = None, @@ -229,16 +230,25 @@ def insar_tops_multi_burst( if not reference[0].split('_')[4] == secondary[0].split('_')[4]: raise Exception('The secondary and reference granules do not have the same polarization') - reference_safe_path = burst2safe(reference) + #reference_safe_path = burst2safe(reference) + reference_safe_path = Path('S1A_IW_SLC__1SSV_20230212T025529_20230212T025534_047197_05A9B2_35DE.SAFE') reference_safe = reference_safe_path.name.split('.')[0] - secondary_safe_path = burst2safe(secondary) + #secondary_safe_path = burst2safe(secondary) + secondary_safe_path = Path('S1A_IW_SLC__1SSV_20230916T025538_20230916T025542_050347_060FBD_DF45.SAFE') secondary_safe = secondary_safe_path.name.split('.')[0] + + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) + polarization = reference[0].split('_')[4] log.info('Begin ISCE2 TopsApp run') insar_tops_packaged( reference=reference_safe, secondary=secondary_safe, - looks=looks, + swaths=swaths, + polarization=polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, apply_water_mask=apply_water_mask, bucket=bucket, bucket_prefix=bucket_prefix diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 394c49e3..5d7cff21 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -57,7 +57,6 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo Returns: The name of the interferogram product. """ - breakpoint() reference_split = reference.split('_') secondary_split = secondary.split('_') @@ -322,6 +321,8 @@ def make_readme( wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' info = gdal.Info(str(wrapped_phase_path), format='json') secondary_granule_datetime_str = secondary_scene.split('_')[3] + if not 'T' in secondary_granule_datetime_str: + secondary_granule_datetime_str = secondary_scene.split('_')[5] payload = { 'processing_date': datetime.now(timezone.utc), @@ -483,9 +484,12 @@ def make_parameter_file( EARTH_RADIUS = 6337286.638938101 parser = etree.XMLParser(encoding='utf-8', recover=True) - - ref_tag = reference_scene[-10:-6] - sec_tag = secondary_scene[-10:-6] + if 'BURST' in reference_scene: + ref_tag = reference_scene[-10:-6] + sec_tag = secondary_scene[-10:-6] + else: + ref_tag = reference_scene[-4::] + sec_tag = secondary_scene[-4::] reference_safe = [file for file in os.listdir('.') if file.endswith(f'{ref_tag}.SAFE')][0] secondary_safe = [file for file in os.listdir('.') if file.endswith(f'{sec_tag}.SAFE')][0] diff --git a/src/hyp3_isce2/slc.py b/src/hyp3_isce2/slc.py index b26893f4..7baad51c 100644 --- a/src/hyp3_isce2/slc.py +++ b/src/hyp3_isce2/slc.py @@ -56,6 +56,18 @@ def get_geometry_from_manifest(manifest_path: Path): return footprint +def get_relative_orbit(granule: Path): + manifest_path = granule / 'manifest.safe' + manifest = ET.parse(manifest_path).getroot() + + frame_element = [x for x in manifest.findall('.//metadataObject') if x.get('ID') == 'measurementOrbitReference'][0] + frame_element = [x for x in frame_element.findall('.//metadataWrap/xmlData')][0] + + print(frame_element.get('type')) + relative_orbit = frame_element.findall('.//relativeOrbitNumber')[0].text + return relative_orbit + + def get_dem_bounds(reference_granule: Path, secondary_granule: Path) -> tuple: """Get the bounds of the DEM to use in processing from SAFE KML files From 292ff9643fd6b194670a4532a9e698a3d6a8a018 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Tue, 13 Aug 2024 17:59:42 -0800 Subject: [PATCH 10/81] Fixing small issues --- src/hyp3_isce2/slc.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/hyp3_isce2/slc.py b/src/hyp3_isce2/slc.py index 7baad51c..b26893f4 100644 --- a/src/hyp3_isce2/slc.py +++ b/src/hyp3_isce2/slc.py @@ -56,18 +56,6 @@ def get_geometry_from_manifest(manifest_path: Path): return footprint -def get_relative_orbit(granule: Path): - manifest_path = granule / 'manifest.safe' - manifest = ET.parse(manifest_path).getroot() - - frame_element = [x for x in manifest.findall('.//metadataObject') if x.get('ID') == 'measurementOrbitReference'][0] - frame_element = [x for x in frame_element.findall('.//metadataWrap/xmlData')][0] - - print(frame_element.get('type')) - relative_orbit = frame_element.findall('.//relativeOrbitNumber')[0].text - return relative_orbit - - def get_dem_bounds(reference_granule: Path, secondary_granule: Path) -> tuple: """Get the bounds of the DEM to use in processing from SAFE KML files From 91dbcfb27dfd7bfbd3d99527541634d1d40db2c7 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Tue, 13 Aug 2024 18:04:34 -0800 Subject: [PATCH 11/81] Removing insar_tops_multi_bursts workflow --- environment.yml | 2 +- pyproject.toml | 2 -- src/hyp3_isce2/__main__.py | 2 +- src/hyp3_isce2/insar_tops.py | 2 +- src/hyp3_isce2/insar_tops_burst.py | 6 ++---- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/environment.yml b/environment.yml index dec70987..ed4fda34 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: hyp3-isce2-forr +name: hyp3-isce2 channels: - conda-forge - nodefaults diff --git a/pyproject.toml b/pyproject.toml index 7be9bf0c..8d06e78a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,14 +56,12 @@ insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" -insar_tops_multi_bursts = "hyp3_isce2.insar_tops_multi_bursts:main" [project.entry-points.hyp3] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" -insar_tops_multi_bursts = "hyp3_isce2.insar_tops_multi_bursts:main" [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/hyp3_isce2/__main__.py b/src/hyp3_isce2/__main__.py index 2fa21782..9d50385e 100644 --- a/src/hyp3_isce2/__main__.py +++ b/src/hyp3_isce2/__main__.py @@ -19,7 +19,7 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '++process', - choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts', 'insar_tops_multi_bursts'], + choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts'], default='insar_tops_burst', help='Select the HyP3 entrypoint to use', # HyP3 entrypoints are specified in `pyproject.toml` ) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 2074866b..0e1fe26e 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -117,7 +117,7 @@ def insar_tops_packaged( product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) log.info('Begin ISCE2 TopsApp run') - #insar_tops(reference, secondary, download=False) + insar_tops(reference, secondary, download=False) log.info('ISCE2 TopsApp run completed successfully') product_dir = Path(product_name) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 12c89207..65603f16 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -230,11 +230,9 @@ def insar_tops_multi_burst( if not reference[0].split('_')[4] == secondary[0].split('_')[4]: raise Exception('The secondary and reference granules do not have the same polarization') - #reference_safe_path = burst2safe(reference) - reference_safe_path = Path('S1A_IW_SLC__1SSV_20230212T025529_20230212T025534_047197_05A9B2_35DE.SAFE') + reference_safe_path = burst2safe(reference) reference_safe = reference_safe_path.name.split('.')[0] - #secondary_safe_path = burst2safe(secondary) - secondary_safe_path = Path('S1A_IW_SLC__1SSV_20230916T025538_20230916T025542_050347_060FBD_DF45.SAFE') + secondary_safe_path = burst2safe(secondary) secondary_safe = secondary_safe_path.name.split('.')[0] range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] From adc08201497b7cfe80289f5fb34a402e19d7885e Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Tue, 13 Aug 2024 18:07:31 -0800 Subject: [PATCH 12/81] Correcting typos --- src/hyp3_isce2/insar_tops_burst.py | 2 +- src/hyp3_isce2/packaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 65603f16..21ee2e1e 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -234,7 +234,7 @@ def insar_tops_multi_burst( reference_safe = reference_safe_path.name.split('.')[0] secondary_safe_path = burst2safe(secondary) secondary_safe = secondary_safe_path.name.split('.')[0] - + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) polarization = reference[0].split('_')[4] diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 5d7cff21..e263db59 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -321,7 +321,7 @@ def make_readme( wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' info = gdal.Info(str(wrapped_phase_path), format='json') secondary_granule_datetime_str = secondary_scene.split('_')[3] - if not 'T' in secondary_granule_datetime_str: + if 'T' not in secondary_granule_datetime_str: secondary_granule_datetime_str = secondary_scene.split('_')[5] payload = { From 98afd685445e9e174024e8de9d01b3c7b0a22123 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Wed, 14 Aug 2024 22:30:16 -0800 Subject: [PATCH 13/81] Adding relative orbit and lons/lats to product name --- pyproject.toml | 2 +- src/hyp3_isce2/packaging.py | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8d06e78a..1a573a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "gdal", "hyp3lib>=3,<4", "s1_orbits", - "burst2safe>=1.0.0" + "burst2safe" # Conda-forge only dependencies are listed below # "opencv", # "isce2>=2.6.3", diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index e263db59..1ab4b1e2 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -1,7 +1,9 @@ +import glob import os import subprocess from dataclasses import dataclass from datetime import datetime, timezone +import numpy as np from pathlib import Path from secrets import token_hex from typing import Iterable, Optional @@ -16,6 +18,7 @@ import hyp3_isce2 import hyp3_isce2.metadata.util from hyp3_isce2.burst import BurstPosition +from hyp3_isce2.slc import get_geometry_from_manifest from hyp3_isce2.utils import get_projection, utm_from_lon_lat @@ -61,13 +64,23 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo secondary_split = secondary.split('_') if slc: + parser = etree.XMLParser(encoding='utf-8', recover=True) + safe = '{http://www.esa.int/safe/sentinel-1.0}' platform = reference_split[0] reference_date = reference_split[5][0:8] secondary_date = secondary_split[5][0:8] - polarization = reference_split[4] - # TODO: Remove hard code - polarization = 'VV' - name_parts = [platform] + polarization = os.path.basename(glob.glob(f'{reference}.SAFE/annotation/s1*')[0]).split('-')[3].upper() + ref_manifest_xml = etree.parse(f'{reference}.SAFE/manifest.safe', parser) + metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' + relative_orbit_number_query = metadata_path + safe + 'relativeOrbitNumber' + orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) + footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) + lons, lats = footprint.exterior.coords.xy + lon_lims = [np.min(lons),np.max(lons)] + lat_lims = [np.min(lats),np.max(lats)] + lons = ['E'+str(abs(int(lon))) if lon>=0 else 'W'+str(abs(int(lon))) for lon in lons] + lats = ['N'+str(abs(int(lat))) if lat>=0 else 'S'+str(abs(int(lat))) for lat in lats] + name_parts = [platform, orbit_number, lons[0], lons[1], lats[0], lats[1]] else: platform = reference_split[0] burst_id = reference_split[1] @@ -76,7 +89,6 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo secondary_date = secondary_split[3][0:8] polarization = reference_split[4] name_parts = [platform, burst_id, image_plus_swath] - product_type = 'INT' pixel_spacing = str(int(pixel_spacing)) product_id = token_hex(2).upper() From fc6ca19984883dc9df973fabe7ad6ff04cd5c557 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Wed, 14 Aug 2024 22:38:34 -0800 Subject: [PATCH 14/81] Cleaning the code --- src/hyp3_isce2/burst.py | 2 +- src/hyp3_isce2/packaging.py | 10 +++++----- src/hyp3_isce2/utils.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 2c5e37a5..03c05020 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -532,7 +532,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value + mask[position.first_valid_line: last_line, position.first_valid_sample: last_sample] = identity_value else: mask[:, :] = identity_value diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 1ab4b1e2..09d77bc0 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -3,12 +3,12 @@ import subprocess from dataclasses import dataclass from datetime import datetime, timezone -import numpy as np from pathlib import Path from secrets import token_hex from typing import Iterable, Optional import isce +import numpy as np from hyp3lib.aws import upload_file_to_s3 from hyp3lib.image import create_thumbnail from lxml import etree @@ -76,10 +76,10 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) lons, lats = footprint.exterior.coords.xy - lon_lims = [np.min(lons),np.max(lons)] - lat_lims = [np.min(lats),np.max(lats)] - lons = ['E'+str(abs(int(lon))) if lon>=0 else 'W'+str(abs(int(lon))) for lon in lons] - lats = ['N'+str(abs(int(lat))) if lat>=0 else 'S'+str(abs(int(lat))) for lat in lats] + lon_lims = [np.min(lons), np.max(lons)] + lat_lims = [np.min(lats), np.max(lats)] + lons = ['E'+str(abs(int(lon))) if lon >= 0 else 'W'+str(abs(int(lon))) for lon in lon_lims] + lats = ['N'+str(abs(int(lat))) if lat >= 0 else 'S'+str(abs(int(lat))) for lat in lat_lims] name_parts = [platform, orbit_number, lons[0], lons[1], lats[0], lats[1]] else: platform = reference_split[0] diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index 798305da..cce0af7c 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -73,7 +73,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i :: image_obj.bands] + new_array[i, :, :] = array[i:: image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -238,7 +238,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i :: image_obj.bands] = array[i, :, :] + new_array[i:: image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') From ee88f995160ad4e7e6fbb967a45777084ad46cd1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 15 Aug 2024 12:49:59 -0500 Subject: [PATCH 15/81] format lat/lon strings --- src/hyp3_isce2/packaging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 09d77bc0..dcfb5eb2 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -76,11 +76,11 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) lons, lats = footprint.exterior.coords.xy - lon_lims = [np.min(lons), np.max(lons)] - lat_lims = [np.min(lats), np.max(lats)] - lons = ['E'+str(abs(int(lon))) if lon >= 0 else 'W'+str(abs(int(lon))) for lon in lon_lims] - lats = ['N'+str(abs(int(lat))) if lat >= 0 else 'S'+str(abs(int(lat))) for lat in lat_lims] - name_parts = [platform, orbit_number, lons[0], lons[1], lats[0], lats[1]] + lat_string = lambda lat: ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') + lon_string = lambda lon: ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') + lat_lims = [lat_string(lat) for lat in [np.min(lats), np.max(lats)]] + lon_lims = [lon_string(lon) for lon in [np.min(lons), np.max(lons)]] + name_parts = [platform, orbit_number, lon_lims[0], lat_lims[0], lon_lims[1], lat_lims[1]] else: platform = reference_split[0] burst_id = reference_split[1] From 51fee239d85cb9ec57351f8da63760b7615b8b53 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 15 Aug 2024 12:58:59 -0500 Subject: [PATCH 16/81] flake8 --- src/hyp3_isce2/packaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index dcfb5eb2..8ab91f5b 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -76,8 +76,8 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) lons, lats = footprint.exterior.coords.xy - lat_string = lambda lat: ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') - lon_string = lambda lon: ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') + def lat_string(lat): ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') + def lon_string(lon): ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') lat_lims = [lat_string(lat) for lat in [np.min(lats), np.max(lats)]] lon_lims = [lon_string(lon) for lon in [np.min(lons), np.max(lons)]] name_parts = [platform, orbit_number, lon_lims[0], lat_lims[0], lon_lims[1], lat_lims[1]] From 4e21974f8740f8060ba729091b2a5dbe6175977c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 15 Aug 2024 13:01:15 -0500 Subject: [PATCH 17/81] add return --- src/hyp3_isce2/packaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 8ab91f5b..dc0fcf73 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -76,8 +76,8 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) lons, lats = footprint.exterior.coords.xy - def lat_string(lat): ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') - def lon_string(lon): ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') + def lat_string(lat): return ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') + def lon_string(lon): return ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') lat_lims = [lat_string(lat) for lat in [np.min(lats), np.max(lats)]] lon_lims = [lon_string(lon) for lon in [np.min(lons), np.max(lons)]] name_parts = [platform, orbit_number, lon_lims[0], lat_lims[0], lon_lims[1], lat_lims[1]] From b90c416935f341df1ac48492e809d465767ef72a Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Thu, 15 Aug 2024 21:51:06 -0800 Subject: [PATCH 18/81] Fixing issues in insar_tops and insar_stripmap --- src/hyp3_isce2/insar_stripmap.py | 36 ++++---- src/hyp3_isce2/insar_tops.py | 18 ++-- src/hyp3_isce2/insar_tops_burst.py | 3 +- src/hyp3_isce2/merge_tops_bursts.py | 3 +- src/hyp3_isce2/packaging.py | 129 +-------------------------- src/hyp3_isce2/slc.py | 2 +- src/hyp3_isce2/utils.py | 130 ++++++++++++++++++++++++++++ 7 files changed, 168 insertions(+), 153 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index b0cc8412..86015c3b 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -23,7 +23,7 @@ log = logging.getLogger(__name__) -def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path: +def insar_stripmap(reference_scene: str, secondary_scene: str) -> Path: """Create a Stripmap interferogram Args: @@ -35,12 +35,22 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce Returns: Path to the output files """ - session = asf_search.ASFSession().auth_with_creds(user, password) - - reference_product, secondary_product = asf_search.search( + scenes = sorted([reference_scene, secondary_scene]) + print(scenes) + reference_scene = scenes[0] + secondary_scene = scenes[1] + products = asf_search.search( granule_list=[reference_scene, secondary_scene], - processingLevel=asf_search.L1_0, + processingLevel="L1.0", ) + + if products[0].properties['sceneName']==reference_scene: + reference_product = products[0] + secondary_product = products[1] + else: + reference_product = products[1] + secondary_product = products[0] + assert reference_product.properties['sceneName'] == reference_scene assert secondary_product.properties['sceneName'] == secondary_scene products = (reference_product, secondary_product) @@ -51,7 +61,7 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce dem_path = download_dem_for_isce2(insar_roi, dem_name='glo_30', dem_dir=Path('dem'), buffer=0) urls = [product.properties['url'] for product in products] - asf_search.download_urls(urls=urls, path=os.getcwd(), session=session, processes=2) + asf_search.download_urls(urls=urls, path=os.getcwd(), processes=2) zip_paths = [product.properties['fileName'] for product in products] for zip_path in zip_paths: @@ -93,7 +103,7 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce def get_product_file(product: asf_search.ASFProduct, file_prefix: str) -> str: paths = glob.glob(str(Path(product.properties['fileID']) / f'{file_prefix}*')) - assert len(paths) == 1 + assert len(paths) > 0 return paths[0] @@ -104,10 +114,8 @@ def main(): parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - parser.add_argument('--username', type=str, required=True) - parser.add_argument('--password', type=str, required=True) - parser.add_argument('--reference-scene', type=str, required=True) - parser.add_argument('--secondary-scene', type=str, required=True) + parser.add_argument('--reference', type=str, required=True) + parser.add_argument('--secondary', type=str, required=True) args = parser.parse_args() @@ -117,10 +125,8 @@ def main(): log.info('Begin InSAR Stripmap run') product_dir = insar_stripmap( - user=args.username, - password=args.password, - reference_scene=args.reference_scene, - secondary_scene=args.secondary_scene, + reference_scene=args.reference, + secondary_scene=args.secondary, ) log.info('InSAR Stripmap run completed successfully') diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 0e1fe26e..dd836e4b 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -2,6 +2,7 @@ import argparse import logging +import os import sys from pathlib import Path from shutil import copyfile, make_archive @@ -13,6 +14,7 @@ from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal +from hyp3_isce2.utils import make_browse_image log = logging.getLogger(__name__) @@ -114,12 +116,16 @@ def insar_tops_packaged( Path to the output files """ pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') - product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) log.info('Begin ISCE2 TopsApp run') - insar_tops(reference, secondary, download=False) + if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{reference}.SAFE'): + insar_tops(reference, secondary, download=False) + else: + insar_tops(reference, secondary) log.info('ISCE2 TopsApp run completed successfully') + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + product_dir = Path(product_name) product_dir.mkdir(parents=True, exist_ok=True) @@ -129,7 +135,7 @@ def insar_tops_packaged( if apply_water_mask: packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') - packaging.make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') packaging.make_readme( product_dir=product_dir, product_name=product_name, @@ -157,7 +163,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--reference', type=str, help='Reference granule') parser.add_argument('--secondary', type=str, help='Secondary granule') - parser.add_argument('--polarization', type=str, defualt='VV', help='Polarization to use') + parser.add_argument('--polarization', type=str, default='VV', help='Polarization to use') parser.add_argument( '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' ) @@ -179,8 +185,8 @@ def main(): raise ValueError('Polarization must be one of VV, VH, HV, or HH') insar_tops_packaged( - reference_scene=args.reference, - secondary_scene=args.secondary, + reference=args.reference, + secondary=args.secondary, polarization=args.polarization, azimuth_looks=azimuth_looks, range_looks=range_looks, diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 21ee2e1e..544bf645 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -30,6 +30,7 @@ from hyp3_isce2.utils import ( image_math, isce2_copy, + make_browse_image, oldest_granule_first, resample_to_radar_io, ) @@ -176,7 +177,7 @@ def insar_tops_single_burst( if apply_water_mask: packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') - packaging.make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') packaging.make_readme( product_dir=product_dir, diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index b3c1f667..4ccc327a 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -39,6 +39,7 @@ import hyp3_isce2 import hyp3_isce2.burst as burst_utils from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.packaging import get_pixel_size, translate_outputs from hyp3_isce2.utils import ( ParameterFile, create_image, @@ -56,8 +57,6 @@ logging.basicConfig(stream=sys.stdout, format=log_format, level=logging.INFO, force=True) log = logging.getLogger(__name__) -from hyp3_isce2.insar_tops_burst import get_pixel_size, translate_outputs # noqa - BURST_IFG_DIR = 'fine_interferogram' BURST_GEOM_DIR = 'geom_reference' diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index dc0fcf73..2b79f324 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -19,7 +19,7 @@ import hyp3_isce2.metadata.util from hyp3_isce2.burst import BurstPosition from hyp3_isce2.slc import get_geometry_from_manifest -from hyp3_isce2.utils import get_projection, utm_from_lon_lat +from hyp3_isce2.utils import get_projection, ParameterFile, utm_from_lon_lat @dataclass @@ -284,43 +284,6 @@ def water_mask(unwrapped_phase: str, water_mask: str) -> None: subprocess.run(cmd.split(' '), check=True) -class GDALConfigManager: - """Context manager for setting GDAL config options temporarily""" - - def __init__(self, **options): - """ - Args: - **options: GDAL Config `option=value` keyword arguments. - """ - self.options = options.copy() - self._previous_options = {} - - def __enter__(self): - for key in self.options: - self._previous_options[key] = gdal.GetConfigOption(key) - - for key, value in self.options.items(): - gdal.SetConfigOption(key, value) - - def __exit__(self, exc_type, exc_val, exc_tb): - for key, value in self._previous_options.items(): - gdal.SetConfigOption(key, value) - - -def make_browse_image(input_tif: str, output_png: str) -> None: - with GDALConfigManager(GDAL_PAM_ENABLED='NO'): - stats = gdal.Info(input_tif, format='json', stats=True)['stac']['raster:bands'][0]['stats'] - gdal.Translate( - destName=output_png, - srcDS=input_tif, - format='png', - outputType=gdal.GDT_Byte, - width=2048, - strict=True, - scaleParams=[[stats['minimum'], stats['maximum']]], - ) - - def make_readme( product_dir: Path, product_name: str, @@ -361,96 +324,6 @@ def make_readme( f.write(content) -@dataclass -class ParameterFile: - reference_granule: str - secondary_granule: str - reference_orbit_direction: str - reference_orbit_number: str - secondary_orbit_direction: str - secondary_orbit_number: str - baseline: float - utc_time: float - heading: float - spacecraft_height: float - earth_radius_at_nadir: float - slant_range_near: float - slant_range_center: float - slant_range_far: float - range_looks: int - azimuth_looks: int - insar_phase_filter: bool - phase_filter_parameter: float - range_bandpass_filter: bool - azimuth_bandpass_filter: bool - dem_source: str - dem_resolution: int - unwrapping_type: str - speckle_filter: bool - water_mask: bool - radar_n_lines: Optional[int] = None - radar_n_samples: Optional[int] = None - radar_first_valid_line: Optional[int] = None - radar_n_valid_lines: Optional[int] = None - radar_first_valid_sample: Optional[int] = None - radar_n_valid_samples: Optional[int] = None - multilook_azimuth_time_interval: Optional[float] = None - multilook_range_pixel_size: Optional[float] = None - radar_sensing_stop: Optional[datetime] = None - - def __str__(self): - output_strings = [ - f'Reference Granule: {self.reference_granule}\n', - f'Secondary Granule: {self.secondary_granule}\n', - f'Reference Pass Direction: {self.reference_orbit_direction}\n', - f'Reference Orbit Number: {self.reference_orbit_number}\n', - f'Secondary Pass Direction: {self.secondary_orbit_direction}\n', - f'Secondary Orbit Number: {self.secondary_orbit_number}\n', - f'Baseline: {self.baseline}\n', - f'UTC time: {self.utc_time}\n', - f'Heading: {self.heading}\n', - f'Spacecraft height: {self.spacecraft_height}\n', - f'Earth radius at nadir: {self.earth_radius_at_nadir}\n', - f'Slant range near: {self.slant_range_near}\n', - f'Slant range center: {self.slant_range_center}\n', - f'Slant range far: {self.slant_range_far}\n', - f'Range looks: {self.range_looks}\n', - f'Azimuth looks: {self.azimuth_looks}\n', - f'INSAR phase filter: {"yes" if self.insar_phase_filter else "no"}\n', - f'Phase filter parameter: {self.phase_filter_parameter}\n', - f'Range bandpass filter: {"yes" if self.range_bandpass_filter else "no"}\n', - f'Azimuth bandpass filter: {"yes" if self.azimuth_bandpass_filter else "no"}\n', - f'DEM source: {self.dem_source}\n', - f'DEM resolution (m): {self.dem_resolution}\n', - f'Unwrapping type: {self.unwrapping_type}\n', - f'Speckle filter: {"yes" if self.speckle_filter else "no"}\n', - f'Water mask: {"yes" if self.water_mask else "no"}\n', - ] - - # TODO could use a more robust way to check if radar data is present - if self.radar_n_lines: - radar_data = [ - f'Radar n lines: {self.radar_n_lines}\n', - f'Radar n samples: {self.radar_n_samples}\n', - f'Radar first valid line: {self.radar_first_valid_line}\n', - f'Radar n valid lines: {self.radar_n_valid_lines}\n', - f'Radar first valid sample: {self.radar_first_valid_sample}\n', - f'Radar n valid samples: {self.radar_n_valid_samples}\n', - f'Multilook azimuth time interval: {self.multilook_azimuth_time_interval}\n', - f'Multilook range pixel size: {self.multilook_range_pixel_size}\n', - f'Radar sensing stop: {datetime.strftime(self.radar_sensing_stop, "%Y-%m-%dT%H:%M:%S.%f")}\n', - ] - output_strings += radar_data - - return ''.join(output_strings) - - def __repr__(self): - return self.__str__() - - def write(self, out_path: Path): - out_path.write_text(self.__str__()) - - def find_available_swaths(base_dir: Path | str) -> list[str]: """Find the available swaths in the given directory diff --git a/src/hyp3_isce2/slc.py b/src/hyp3_isce2/slc.py index b26893f4..02be9468 100644 --- a/src/hyp3_isce2/slc.py +++ b/src/hyp3_isce2/slc.py @@ -33,7 +33,7 @@ def get_granule(granule: str) -> Path: def unzip_granule(zip_file: Path, remove: bool = False) -> Path: with ZipFile(zip_file) as z: z.extractall() - safe_dir = next(item.filename for item in z.infolist() if item.is_dir() and item.filename.endswith('.SAFE/')) + safe_dir = zip_file.split('.')[0]+'.SAFE/' if remove: os.remove(zip_file) return safe_dir.strip('/') diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index cce0af7c..b1d7d191 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -1,5 +1,8 @@ import shutil import subprocess +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path from typing import Optional import isceobj @@ -12,6 +15,119 @@ gdal.UseExceptions() +class GDALConfigManager: + """Context manager for setting GDAL config options temporarily""" + + def __init__(self, **options): + """ + Args: + **options: GDAL Config `option=value` keyword arguments. + """ + self.options = options.copy() + self._previous_options = {} + + def __enter__(self): + for key in self.options: + self._previous_options[key] = gdal.GetConfigOption(key) + + for key, value in self.options.items(): + gdal.SetConfigOption(key, value) + + def __exit__(self, exc_type, exc_val, exc_tb): + for key, value in self._previous_options.items(): + gdal.SetConfigOption(key, value) + + +@dataclass +class ParameterFile: + reference_granule: str + secondary_granule: str + reference_orbit_direction: str + reference_orbit_number: str + secondary_orbit_direction: str + secondary_orbit_number: str + baseline: float + utc_time: float + heading: float + spacecraft_height: float + earth_radius_at_nadir: float + slant_range_near: float + slant_range_center: float + slant_range_far: float + range_looks: int + azimuth_looks: int + insar_phase_filter: bool + phase_filter_parameter: float + range_bandpass_filter: bool + azimuth_bandpass_filter: bool + dem_source: str + dem_resolution: int + unwrapping_type: str + speckle_filter: bool + water_mask: bool + radar_n_lines: Optional[int] = None + radar_n_samples: Optional[int] = None + radar_first_valid_line: Optional[int] = None + radar_n_valid_lines: Optional[int] = None + radar_first_valid_sample: Optional[int] = None + radar_n_valid_samples: Optional[int] = None + multilook_azimuth_time_interval: Optional[float] = None + multilook_range_pixel_size: Optional[float] = None + radar_sensing_stop: Optional[datetime] = None + + def __str__(self): + output_strings = [ + f'Reference Granule: {self.reference_granule}\n', + f'Secondary Granule: {self.secondary_granule}\n', + f'Reference Pass Direction: {self.reference_orbit_direction}\n', + f'Reference Orbit Number: {self.reference_orbit_number}\n', + f'Secondary Pass Direction: {self.secondary_orbit_direction}\n', + f'Secondary Orbit Number: {self.secondary_orbit_number}\n', + f'Baseline: {self.baseline}\n', + f'UTC time: {self.utc_time}\n', + f'Heading: {self.heading}\n', + f'Spacecraft height: {self.spacecraft_height}\n', + f'Earth radius at nadir: {self.earth_radius_at_nadir}\n', + f'Slant range near: {self.slant_range_near}\n', + f'Slant range center: {self.slant_range_center}\n', + f'Slant range far: {self.slant_range_far}\n', + f'Range looks: {self.range_looks}\n', + f'Azimuth looks: {self.azimuth_looks}\n', + f'INSAR phase filter: {"yes" if self.insar_phase_filter else "no"}\n', + f'Phase filter parameter: {self.phase_filter_parameter}\n', + f'Range bandpass filter: {"yes" if self.range_bandpass_filter else "no"}\n', + f'Azimuth bandpass filter: {"yes" if self.azimuth_bandpass_filter else "no"}\n', + f'DEM source: {self.dem_source}\n', + f'DEM resolution (m): {self.dem_resolution}\n', + f'Unwrapping type: {self.unwrapping_type}\n', + f'Speckle filter: {"yes" if self.speckle_filter else "no"}\n', + f'Water mask: {"yes" if self.water_mask else "no"}\n', + ] + + # TODO could use a more robust way to check if radar data is present + if self.radar_n_lines: + radar_data = [ + f'Radar n lines: {self.radar_n_lines}\n', + f'Radar n samples: {self.radar_n_samples}\n', + f'Radar first valid line: {self.radar_first_valid_line}\n', + f'Radar n valid lines: {self.radar_n_valid_lines}\n', + f'Radar first valid sample: {self.radar_first_valid_sample}\n', + f'Radar n valid samples: {self.radar_n_valid_samples}\n', + f'Multilook azimuth time interval: {self.multilook_azimuth_time_interval}\n', + f'Multilook range pixel size: {self.multilook_range_pixel_size}\n', + f'Radar sensing stop: {datetime.strftime(self.radar_sensing_stop, "%Y-%m-%dT%H:%M:%S.%f")}\n', + ] + output_strings += radar_data + + return ''.join(output_strings) + + def __repr__(self): + return self.__str__() + + def write(self, out_path: Path): + out_path.write_text(self.__str__()) + + def utm_from_lon_lat(lon: float, lat: float) -> int: """Get the UTM zone EPSG code from a longitude and latitude. See https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system @@ -49,6 +165,20 @@ def extent_from_geotransform(geotransform: tuple, x_size: int, y_size: int) -> t return extent +def make_browse_image(input_tif: str, output_png: str) -> None: + with GDALConfigManager(GDAL_PAM_ENABLED='NO'): + stats = gdal.Info(input_tif, format='json', stats=True)['stac']['raster:bands'][0]['stats'] + gdal.Translate( + destName=output_png, + srcDS=input_tif, + format='png', + outputType=gdal.GDT_Byte, + width=2048, + strict=True, + scaleParams=[[stats['minimum'], stats['maximum']]], + ) + + def oldest_granule_first(g1, g2): if g1[14:29] <= g2[14:29]: return g1, g2 From 0eddc0053084a24f00454809eebcfcbae5828812 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Thu, 15 Aug 2024 21:53:37 -0800 Subject: [PATCH 19/81] flake8 --- src/hyp3_isce2/insar_stripmap.py | 2 +- src/hyp3_isce2/packaging.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 86015c3b..b0d181a6 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -44,7 +44,7 @@ def insar_stripmap(reference_scene: str, secondary_scene: str) -> Path: processingLevel="L1.0", ) - if products[0].properties['sceneName']==reference_scene: + if products[0].properties['sceneName'] == reference_scene: reference_product = products[0] secondary_product = products[1] else: diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 2b79f324..b5e49565 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -19,7 +19,7 @@ import hyp3_isce2.metadata.util from hyp3_isce2.burst import BurstPosition from hyp3_isce2.slc import get_geometry_from_manifest -from hyp3_isce2.utils import get_projection, ParameterFile, utm_from_lon_lat +from hyp3_isce2.utils import ParameterFile, get_projection, utm_from_lon_lat @dataclass From 88af29d9553ddab14eedbea63c7c6957e0ee9711 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 15 Aug 2024 22:05:11 -0800 Subject: [PATCH 20/81] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 276c86df..7c5c370b 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,17 @@ The HyP3-ISCE2 plugin provides a set of workflows to process SAR satellite data ## Usage The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python or via a CLI) that can be used to process SAR data using ISCE2. The workflows currently included in this plugin are: +- `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow - `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow -- `insar_tops_burst`: A workflow for creating single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow +- `insar_tops_burst`: A workflow for creating multi-bursts Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow --- To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow: ``` python -m hyp3_isce2 ++process insar_tops_burst \ - S1_136231_IW2_20200604T022312_VV_7C85-BURST \ - S1_136231_IW2_20200616T022313_VV_5D11-BURST \ + --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136231_IW3_20200604T022313_VV_7C85-BURST\ + --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136231_IW3_20200616T022314_VV_5D11-BURST\ --looks 20x4 \ --apply-water-mask True ``` From 07da02f95da472afecb2a7122520c070f03f4e1e Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 15 Aug 2024 22:15:18 -0800 Subject: [PATCH 21/81] Update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 818caee5..34190aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.0.0] +### Added +- New funcionality to process multi bursts with `insar_tops_burst`. +- The `--reference` and `--secondary` command-line options to indicate the reference and secondary bursts. +- `burst2safe` is now called to get SAFE files from bursts. +### Changed +- Product name has changed to indicate relative orbit, lon/lat extent and reference and secondary dates. + ## [2.0.0] ### Changed - Orbit files are now retrieved using the [s1-orbits](https://github.com/ASFHyP3/sentinel1-orbits-py) library. From 182f506ece1366a95f01ab3bb04af7cd3ccd350f Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 06:21:28 +0000 Subject: [PATCH 22/81] update coverage image --- images/coverage.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index dfb99a19..012a8497 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 67% - 67% + 70% + 70% From 227237d2cd9fed33b66476350b2f795e10e8df49 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Thu, 15 Aug 2024 23:40:42 -0800 Subject: [PATCH 23/81] Fixing water mask for insar_tops --- insar_tops.py | 211 ++++++++++++++++++++++++++++++ insar_tops_burst.py | 306 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 517 insertions(+) create mode 100644 insar_tops.py create mode 100644 insar_tops_burst.py diff --git a/insar_tops.py b/insar_tops.py new file mode 100644 index 00000000..baf81724 --- /dev/null +++ b/insar_tops.py @@ -0,0 +1,211 @@ +"""Create a full SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" + +import argparse +import logging +import os +import sys +from pathlib import Path +from shutil import copyfile, make_archive + +from hyp3lib.util import string_is_true +from s1_orbits import fetch_for_scene + +from hyp3_isce2 import packaging, slc, topsapp +from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.logger import configure_root_logger +from hyp3_isce2.s1_auxcal import download_aux_cal +from hyp3_isce2.utils import make_browse_image + + +log = logging.getLogger(__name__) + + +def insar_tops( + reference_scene: str, + secondary_scene: str, + swaths: list = [1, 2, 3], + polarization: str = 'VV', + azimuth_looks: int = 4, + range_looks: int = 20, + apply_water_mask: bool = False, + download: bool = True, +) -> Path: + """Create a full-SLC interferogram + + Args: + reference_scene: Reference SLC name + secondary_scene: Secondary SLC name + swaths: Swaths to process + polarization: Polarization to use + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + + Returns: + Path to the output files + """ + orbit_dir = Path('orbits') + aux_cal_dir = Path('aux_cal') + dem_dir = Path('dem') + + if download: + ref_dir = slc.get_granule(reference_scene) + sec_dir = slc.get_granule(secondary_scene) + else: + ref_dir = Path(reference_scene + '.SAFE') + sec_dir = Path(secondary_scene + '.SAFE') + roi = slc.get_dem_bounds(ref_dir, sec_dir) + log.info(f'DEM ROI: {roi}') + + dem_path = download_dem_for_isce2(roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0) + download_aux_cal(aux_cal_dir) + + orbit_dir.mkdir(exist_ok=True, parents=True) + for granule in (reference_scene, secondary_scene): + log.info(f'Downloading orbit file for {granule}') + orbit_file = fetch_for_scene(granule, dir=orbit_dir) + log.info(f'Got orbit file {orbit_file} from s1_orbits') + + config = topsapp.TopsappBurstConfig( + reference_safe=f'{reference_scene}.SAFE', + secondary_safe=f'{secondary_scene}.SAFE', + polarization=polarization, + orbit_directory=str(orbit_dir), + aux_cal_directory=str(aux_cal_dir), + dem_filename=str(dem_path), + geocode_dem_filename=str(dem_path), + roi=roi, + swaths=swaths, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + ) + config_path = config.write_template('topsApp.xml') + + if apply_water_mask: + topsapp.run_topsapp_burst(start='startup', end='filter', config_xml=config_path) + water_mask_path = 'water_mask.wgs84' + create_water_mask(str(dem_path), water_mask_path) + multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) + multilook('merged/lat.rdr.full', outname='merged/lat.rdr', alks=azimuth_looks, rlks=range_looks) + resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') + isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') + image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') + topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) + isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') + else: + topsapp.run_topsapp_burst(start='startup', end='unwrap2stage', config_xml=config_path) + copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') + topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) + + return Path('merged') + + +def insar_tops_packaged( + reference: str, + secondary: str, + swaths: list = [1, 2, 3], + polarization: str = 'VV', + azimuth_looks: int = 4, + range_looks: int = 20, + apply_water_mask: bool = True, + download: bool = True, + bucket: str = None, + bucket_prefix: str = '', +) -> Path: + """Create a full-SLC interferogram + + Args: + reference: Reference SLC name + secondary: Secondary SLC name + swaths: Swaths to process + polarization: Polarization to use + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + apply_water_mask: Apply water mask to unwrapped phase + download: Download the SLCs + bucket: AWS S3 bucket to upload the final product to + bucket_prefix: Bucket prefix to prefix to use when uploading the final product + + Returns: + Path to the output files + """ + pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') + + log.info('Begin ISCE2 TopsApp run') + if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE'): + insar_tops(reference, secondary, apply_water_mask=apply_water_mask, download=False) + else: + insar_tops(reference, secondary, apply_water_mask=apply_water_mask) + log.info('ISCE2 TopsApp run completed successfully') + + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + + product_dir = Path(product_name) + product_dir.mkdir(parents=True, exist_ok=True) + + packaging.translate_outputs(product_name, pixel_size=pixel_size) + + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' + if apply_water_mask: + packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') + + make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + packaging.make_readme( + product_dir=product_dir, + product_name=product_name, + reference_scene=reference, + secondary_scene=secondary, + range_looks=range_looks, + azimuth_looks=azimuth_looks, + apply_water_mask=apply_water_mask, + ) + packaging.make_parameter_file( + Path(f'{product_name}/{product_name}.txt'), + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + ) + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + if bucket: + packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) + + +def main(): + """HyP3 entrypoint for the SLC TOPS workflow""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--reference', type=str, help='Reference granule') + parser.add_argument('--secondary', type=str, help='Secondary granule') + parser.add_argument('--polarization', type=str, default='VV', help='Polarization to use') + parser.add_argument( + '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' + ) + parser.add_argument( + '--apply-water-mask', + type=string_is_true, + default=False, + help='Apply a water body mask before unwrapping.', + ) + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') + + args = parser.parse_args() + configure_root_logger() + log.debug(' '.join(sys.argv)) + + range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] + if args.polarization not in ['VV', 'VH', 'HV', 'HH']: + raise ValueError('Polarization must be one of VV, VH, HV, or HH') + + insar_tops_packaged( + reference=args.reference, + secondary=args.secondary, + polarization=args.polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) + + log.info('ISCE2 TopsApp run completed successfully') diff --git a/insar_tops_burst.py b/insar_tops_burst.py new file mode 100644 index 00000000..544bf645 --- /dev/null +++ b/insar_tops_burst.py @@ -0,0 +1,306 @@ +"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" + +import argparse +import logging +import sys +from pathlib import Path +from shutil import copyfile, make_archive +from typing import Iterable, Optional + +import isce # noqa +from burst2safe.burst2safe import burst2safe +from hyp3lib.util import string_is_true +from isceobj.TopsProc.runMergeBursts import multilook +from osgeo import gdal +from s1_orbits import fetch_for_scene + +from hyp3_isce2 import packaging, topsapp +from hyp3_isce2.burst import ( + download_bursts, + get_burst_params, + get_isce2_burst_bbox, + get_region_of_interest, + multilook_radar_merge_inputs, + validate_bursts, +) +from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.insar_tops import insar_tops_packaged +from hyp3_isce2.logger import configure_root_logger +from hyp3_isce2.s1_auxcal import download_aux_cal +from hyp3_isce2.utils import ( + image_math, + isce2_copy, + make_browse_image, + oldest_granule_first, + resample_to_radar_io, +) +from hyp3_isce2.water_mask import create_water_mask + + +gdal.UseExceptions() + +log = logging.getLogger(__name__) + + +def insar_tops_burst( + reference_scene: str, + secondary_scene: str, + swath_number: int, + azimuth_looks: int = 4, + range_looks: int = 20, + apply_water_mask: bool = False, +) -> Path: + """Create a burst interferogram + + Args: + reference_scene: Reference burst name + secondary_scene: Secondary burst name + swath_number: Number of swath to grab bursts from (1, 2, or 3) for IW + azimuth_looks: Number of azimuth looks + range_looks: Number of range looks + apply_water_mask: Whether to apply a pre-unwrap water mask + + Returns: + Path to results directory + """ + orbit_dir = Path('orbits') + aux_cal_dir = Path('aux_cal') + dem_dir = Path('dem') + + ref_params = get_burst_params(reference_scene) + sec_params = get_burst_params(secondary_scene) + + ref_metadata, sec_metadata = download_bursts([ref_params, sec_params]) + + is_ascending = ref_metadata.orbit_direction == 'ascending' + ref_footprint = get_isce2_burst_bbox(ref_params) + sec_footprint = get_isce2_burst_bbox(sec_params) + + insar_roi = get_region_of_interest(ref_footprint, sec_footprint, is_ascending=is_ascending) + dem_roi = ref_footprint.intersection(sec_footprint).bounds + + if abs(dem_roi[0] - dem_roi[2]) > 180.0 and dem_roi[0] * dem_roi[2] < 0.0: + raise ValueError('Products that cross the anti-meridian are not currently supported.') + + log.info(f'InSAR ROI: {insar_roi}') + log.info(f'DEM ROI: {dem_roi}') + + dem_path = download_dem_for_isce2(dem_roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0, resample_20m=False) + download_aux_cal(aux_cal_dir) + + if range_looks == 5: + geocode_dem_path = download_dem_for_isce2( + dem_roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0, resample_20m=True + ) + else: + geocode_dem_path = dem_path + + orbit_dir.mkdir(exist_ok=True, parents=True) + for granule in (ref_params.granule, sec_params.granule): + log.info(f'Downloading orbit file for {granule}') + orbit_file = fetch_for_scene(granule, dir=orbit_dir) + log.info(f'Got orbit file {orbit_file} from s1_orbits') + + config = topsapp.TopsappBurstConfig( + reference_safe=f'{ref_params.granule}.SAFE', + secondary_safe=f'{sec_params.granule}.SAFE', + polarization=ref_params.polarization, + orbit_directory=str(orbit_dir), + aux_cal_directory=str(aux_cal_dir), + roi=insar_roi, + dem_filename=str(dem_path), + geocode_dem_filename=str(geocode_dem_path), + swaths=swath_number, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + ) + config_path = config.write_template('topsApp.xml') + + topsapp.run_topsapp_burst(start='startup', end='preprocess', config_xml=config_path) + topsapp.swap_burst_vrts() + if apply_water_mask: + topsapp.run_topsapp_burst(start='computeBaselines', end='filter', config_xml=config_path) + water_mask_path = 'water_mask.wgs84' + create_water_mask(str(dem_path), water_mask_path) + multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) + multilook('merged/lat.rdr.full', outname='merged/lat.rdr', alks=azimuth_looks, rlks=range_looks) + resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') + isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') + image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') + topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) + isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') + else: + topsapp.run_topsapp_burst(start='computeBaselines', end='unwrap2stage', config_xml=config_path) + copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') + topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) + + return Path('merged') + + +def insar_tops_single_burst( + reference: str, + secondary: str, + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + reference, secondary = oldest_granule_first(reference, secondary) + validate_bursts(reference, secondary) + swath_number = int(reference[12]) + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + + log.info('Begin ISCE2 TopsApp run') + + insar_tops_burst( + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + swath_number=swath_number, + apply_water_mask=apply_water_mask, + ) + + log.info('ISCE2 TopsApp run completed successfully') + + multilook_position = multilook_radar_merge_inputs(swath_number, rg_looks=range_looks, az_looks=azimuth_looks) + + pixel_size = packaging.get_pixel_size(looks) + product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size), slc=False) + + product_dir = Path(product_name) + product_dir.mkdir(parents=True, exist_ok=True) + + packaging.translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) + + unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' + if apply_water_mask: + packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') + + make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') + + packaging.make_readme( + product_dir=product_dir, + product_name=product_name, + reference_scene=reference, + secondary_scene=secondary, + range_looks=range_looks, + azimuth_looks=azimuth_looks, + apply_water_mask=apply_water_mask, + ) + packaging.make_parameter_file( + Path(f'{product_name}/{product_name}.txt'), + reference_scene=reference, + secondary_scene=secondary, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + multilook_position=multilook_position, + apply_water_mask=apply_water_mask, + ) + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) + + if bucket: + packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) + + +def insar_tops_multi_burst( + reference: Iterable[str], + secondary: Iterable[str], + swaths: list = [1, 2, 3], + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + + if len(list(set(ref_ids) - set(sec_ids))) > 0: + raise Exception( + 'The reference bursts ' + + ', '.join(list(set(ref_ids) - set(sec_ids))) + + ' do not have the correspondant bursts in the secondary granules' + ) + elif len(list(set(sec_ids) - set(ref_ids))) > 0: + raise Exception( + 'The secondary bursts ' + + ', '.join(list(set(sec_ids) - set(ref_ids))) + + ' do not have the correspondant bursts in the reference granules' + ) + + if not reference[0].split('_')[4] == secondary[0].split('_')[4]: + raise Exception('The secondary and reference granules do not have the same polarization') + + reference_safe_path = burst2safe(reference) + reference_safe = reference_safe_path.name.split('.')[0] + secondary_safe_path = burst2safe(secondary) + secondary_safe = secondary_safe_path.name.split('.')[0] + + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) + polarization = reference[0].split('_')[4] + + log.info('Begin ISCE2 TopsApp run') + insar_tops_packaged( + reference=reference_safe, + secondary=secondary_safe, + swaths=swaths, + polarization=polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + bucket=bucket, + bucket_prefix=bucket_prefix + ) + log.info('ISCE2 TopsApp run completed successfully') + + +def main(): + """HyP3 entrypoint for the burst TOPS workflow""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') + parser.add_argument( + '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' + ) + parser.add_argument( + '--apply-water-mask', + type=string_is_true, + default=False, + help='Apply a water body mask before unwrapping.', + ) + # Allows granules to be given as a space-delimited list of strings (e.g. foo bar) or as a single + # quoted string that contains spaces (e.g. "foo bar"). AWS Batch uses the latter format when + # invoking the container command. + parser.add_argument('--reference', type=str.split, nargs='+', help='List of granules for the reference bursts') + parser.add_argument('--secondary', type=str.split, nargs='+', help='List of granules for the secondary bursts') + + args = parser.parse_args() + + args.reference = [item for sublist in args.reference for item in sublist] + args.secondary = [item for sublist in args.secondary for item in sublist] + if len(args.reference) != len(args.secondary): + parser.error('Number of reference and secondary granules must be the same') + + configure_root_logger() + log.debug(' '.join(sys.argv)) + + if len(args.reference) == 1: + insar_tops_single_burst( + reference=args.reference[0], + secondary=args.secondary[0], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) + else: + insar_tops_multi_burst( + reference=args.reference, + secondary=args.secondary, + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) From e07df9b973a7d8b93c21405252183c80d10b3655 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Thu, 15 Aug 2024 23:42:22 -0800 Subject: [PATCH 24/81] Fixing water mask for insar_tops --- insar_tops.py | 211 ------------------------ insar_tops_burst.py | 306 ----------------------------------- src/hyp3_isce2/insar_tops.py | 21 ++- 3 files changed, 17 insertions(+), 521 deletions(-) delete mode 100644 insar_tops.py delete mode 100644 insar_tops_burst.py diff --git a/insar_tops.py b/insar_tops.py deleted file mode 100644 index baf81724..00000000 --- a/insar_tops.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Create a full SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" - -import argparse -import logging -import os -import sys -from pathlib import Path -from shutil import copyfile, make_archive - -from hyp3lib.util import string_is_true -from s1_orbits import fetch_for_scene - -from hyp3_isce2 import packaging, slc, topsapp -from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logger import configure_root_logger -from hyp3_isce2.s1_auxcal import download_aux_cal -from hyp3_isce2.utils import make_browse_image - - -log = logging.getLogger(__name__) - - -def insar_tops( - reference_scene: str, - secondary_scene: str, - swaths: list = [1, 2, 3], - polarization: str = 'VV', - azimuth_looks: int = 4, - range_looks: int = 20, - apply_water_mask: bool = False, - download: bool = True, -) -> Path: - """Create a full-SLC interferogram - - Args: - reference_scene: Reference SLC name - secondary_scene: Secondary SLC name - swaths: Swaths to process - polarization: Polarization to use - azimuth_looks: Number of azimuth looks - range_looks: Number of range looks - - Returns: - Path to the output files - """ - orbit_dir = Path('orbits') - aux_cal_dir = Path('aux_cal') - dem_dir = Path('dem') - - if download: - ref_dir = slc.get_granule(reference_scene) - sec_dir = slc.get_granule(secondary_scene) - else: - ref_dir = Path(reference_scene + '.SAFE') - sec_dir = Path(secondary_scene + '.SAFE') - roi = slc.get_dem_bounds(ref_dir, sec_dir) - log.info(f'DEM ROI: {roi}') - - dem_path = download_dem_for_isce2(roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0) - download_aux_cal(aux_cal_dir) - - orbit_dir.mkdir(exist_ok=True, parents=True) - for granule in (reference_scene, secondary_scene): - log.info(f'Downloading orbit file for {granule}') - orbit_file = fetch_for_scene(granule, dir=orbit_dir) - log.info(f'Got orbit file {orbit_file} from s1_orbits') - - config = topsapp.TopsappBurstConfig( - reference_safe=f'{reference_scene}.SAFE', - secondary_safe=f'{secondary_scene}.SAFE', - polarization=polarization, - orbit_directory=str(orbit_dir), - aux_cal_directory=str(aux_cal_dir), - dem_filename=str(dem_path), - geocode_dem_filename=str(dem_path), - roi=roi, - swaths=swaths, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - ) - config_path = config.write_template('topsApp.xml') - - if apply_water_mask: - topsapp.run_topsapp_burst(start='startup', end='filter', config_xml=config_path) - water_mask_path = 'water_mask.wgs84' - create_water_mask(str(dem_path), water_mask_path) - multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) - multilook('merged/lat.rdr.full', outname='merged/lat.rdr', alks=azimuth_looks, rlks=range_looks) - resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') - isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') - image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') - topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) - isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') - else: - topsapp.run_topsapp_burst(start='startup', end='unwrap2stage', config_xml=config_path) - copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') - topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) - - return Path('merged') - - -def insar_tops_packaged( - reference: str, - secondary: str, - swaths: list = [1, 2, 3], - polarization: str = 'VV', - azimuth_looks: int = 4, - range_looks: int = 20, - apply_water_mask: bool = True, - download: bool = True, - bucket: str = None, - bucket_prefix: str = '', -) -> Path: - """Create a full-SLC interferogram - - Args: - reference: Reference SLC name - secondary: Secondary SLC name - swaths: Swaths to process - polarization: Polarization to use - azimuth_looks: Number of azimuth looks - range_looks: Number of range looks - apply_water_mask: Apply water mask to unwrapped phase - download: Download the SLCs - bucket: AWS S3 bucket to upload the final product to - bucket_prefix: Bucket prefix to prefix to use when uploading the final product - - Returns: - Path to the output files - """ - pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') - - log.info('Begin ISCE2 TopsApp run') - if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE'): - insar_tops(reference, secondary, apply_water_mask=apply_water_mask, download=False) - else: - insar_tops(reference, secondary, apply_water_mask=apply_water_mask) - log.info('ISCE2 TopsApp run completed successfully') - - product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) - - product_dir = Path(product_name) - product_dir.mkdir(parents=True, exist_ok=True) - - packaging.translate_outputs(product_name, pixel_size=pixel_size) - - unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' - if apply_water_mask: - packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') - - make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') - packaging.make_readme( - product_dir=product_dir, - product_name=product_name, - reference_scene=reference, - secondary_scene=secondary, - range_looks=range_looks, - azimuth_looks=azimuth_looks, - apply_water_mask=apply_water_mask, - ) - packaging.make_parameter_file( - Path(f'{product_name}/{product_name}.txt'), - reference_scene=reference, - secondary_scene=secondary, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=apply_water_mask, - ) - output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - if bucket: - packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) - - -def main(): - """HyP3 entrypoint for the SLC TOPS workflow""" - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--reference', type=str, help='Reference granule') - parser.add_argument('--secondary', type=str, help='Secondary granule') - parser.add_argument('--polarization', type=str, default='VV', help='Polarization to use') - parser.add_argument( - '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' - ) - parser.add_argument( - '--apply-water-mask', - type=string_is_true, - default=False, - help='Apply a water body mask before unwrapping.', - ) - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - - args = parser.parse_args() - configure_root_logger() - log.debug(' '.join(sys.argv)) - - range_looks, azimuth_looks = [int(looks) for looks in args.looks.split('x')] - if args.polarization not in ['VV', 'VH', 'HV', 'HH']: - raise ValueError('Polarization must be one of VV, VH, HV, or HH') - - insar_tops_packaged( - reference=args.reference, - secondary=args.secondary, - polarization=args.polarization, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) - - log.info('ISCE2 TopsApp run completed successfully') diff --git a/insar_tops_burst.py b/insar_tops_burst.py deleted file mode 100644 index 544bf645..00000000 --- a/insar_tops_burst.py +++ /dev/null @@ -1,306 +0,0 @@ -"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" - -import argparse -import logging -import sys -from pathlib import Path -from shutil import copyfile, make_archive -from typing import Iterable, Optional - -import isce # noqa -from burst2safe.burst2safe import burst2safe -from hyp3lib.util import string_is_true -from isceobj.TopsProc.runMergeBursts import multilook -from osgeo import gdal -from s1_orbits import fetch_for_scene - -from hyp3_isce2 import packaging, topsapp -from hyp3_isce2.burst import ( - download_bursts, - get_burst_params, - get_isce2_burst_bbox, - get_region_of_interest, - multilook_radar_merge_inputs, - validate_bursts, -) -from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.insar_tops import insar_tops_packaged -from hyp3_isce2.logger import configure_root_logger -from hyp3_isce2.s1_auxcal import download_aux_cal -from hyp3_isce2.utils import ( - image_math, - isce2_copy, - make_browse_image, - oldest_granule_first, - resample_to_radar_io, -) -from hyp3_isce2.water_mask import create_water_mask - - -gdal.UseExceptions() - -log = logging.getLogger(__name__) - - -def insar_tops_burst( - reference_scene: str, - secondary_scene: str, - swath_number: int, - azimuth_looks: int = 4, - range_looks: int = 20, - apply_water_mask: bool = False, -) -> Path: - """Create a burst interferogram - - Args: - reference_scene: Reference burst name - secondary_scene: Secondary burst name - swath_number: Number of swath to grab bursts from (1, 2, or 3) for IW - azimuth_looks: Number of azimuth looks - range_looks: Number of range looks - apply_water_mask: Whether to apply a pre-unwrap water mask - - Returns: - Path to results directory - """ - orbit_dir = Path('orbits') - aux_cal_dir = Path('aux_cal') - dem_dir = Path('dem') - - ref_params = get_burst_params(reference_scene) - sec_params = get_burst_params(secondary_scene) - - ref_metadata, sec_metadata = download_bursts([ref_params, sec_params]) - - is_ascending = ref_metadata.orbit_direction == 'ascending' - ref_footprint = get_isce2_burst_bbox(ref_params) - sec_footprint = get_isce2_burst_bbox(sec_params) - - insar_roi = get_region_of_interest(ref_footprint, sec_footprint, is_ascending=is_ascending) - dem_roi = ref_footprint.intersection(sec_footprint).bounds - - if abs(dem_roi[0] - dem_roi[2]) > 180.0 and dem_roi[0] * dem_roi[2] < 0.0: - raise ValueError('Products that cross the anti-meridian are not currently supported.') - - log.info(f'InSAR ROI: {insar_roi}') - log.info(f'DEM ROI: {dem_roi}') - - dem_path = download_dem_for_isce2(dem_roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0, resample_20m=False) - download_aux_cal(aux_cal_dir) - - if range_looks == 5: - geocode_dem_path = download_dem_for_isce2( - dem_roi, dem_name='glo_30', dem_dir=dem_dir, buffer=0, resample_20m=True - ) - else: - geocode_dem_path = dem_path - - orbit_dir.mkdir(exist_ok=True, parents=True) - for granule in (ref_params.granule, sec_params.granule): - log.info(f'Downloading orbit file for {granule}') - orbit_file = fetch_for_scene(granule, dir=orbit_dir) - log.info(f'Got orbit file {orbit_file} from s1_orbits') - - config = topsapp.TopsappBurstConfig( - reference_safe=f'{ref_params.granule}.SAFE', - secondary_safe=f'{sec_params.granule}.SAFE', - polarization=ref_params.polarization, - orbit_directory=str(orbit_dir), - aux_cal_directory=str(aux_cal_dir), - roi=insar_roi, - dem_filename=str(dem_path), - geocode_dem_filename=str(geocode_dem_path), - swaths=swath_number, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - ) - config_path = config.write_template('topsApp.xml') - - topsapp.run_topsapp_burst(start='startup', end='preprocess', config_xml=config_path) - topsapp.swap_burst_vrts() - if apply_water_mask: - topsapp.run_topsapp_burst(start='computeBaselines', end='filter', config_xml=config_path) - water_mask_path = 'water_mask.wgs84' - create_water_mask(str(dem_path), water_mask_path) - multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) - multilook('merged/lat.rdr.full', outname='merged/lat.rdr', alks=azimuth_looks, rlks=range_looks) - resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') - isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') - image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') - topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) - isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') - else: - topsapp.run_topsapp_burst(start='computeBaselines', end='unwrap2stage', config_xml=config_path) - copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') - topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) - - return Path('merged') - - -def insar_tops_single_burst( - reference: str, - secondary: str, - looks: str = '20x4', - apply_water_mask=False, - bucket: Optional[str] = None, - bucket_prefix: str = '', -): - reference, secondary = oldest_granule_first(reference, secondary) - validate_bursts(reference, secondary) - swath_number = int(reference[12]) - range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] - - log.info('Begin ISCE2 TopsApp run') - - insar_tops_burst( - reference_scene=reference, - secondary_scene=secondary, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - swath_number=swath_number, - apply_water_mask=apply_water_mask, - ) - - log.info('ISCE2 TopsApp run completed successfully') - - multilook_position = multilook_radar_merge_inputs(swath_number, rg_looks=range_looks, az_looks=azimuth_looks) - - pixel_size = packaging.get_pixel_size(looks) - product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size), slc=False) - - product_dir = Path(product_name) - product_dir.mkdir(parents=True, exist_ok=True) - - packaging.translate_outputs(product_name, pixel_size=pixel_size, include_radar=True, use_multilooked=True) - - unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' - if apply_water_mask: - packaging.water_mask(unwrapped_phase, f'{product_name}/{product_name}_water_mask.tif') - - make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') - - packaging.make_readme( - product_dir=product_dir, - product_name=product_name, - reference_scene=reference, - secondary_scene=secondary, - range_looks=range_looks, - azimuth_looks=azimuth_looks, - apply_water_mask=apply_water_mask, - ) - packaging.make_parameter_file( - Path(f'{product_name}/{product_name}.txt'), - reference_scene=reference, - secondary_scene=secondary, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - multilook_position=multilook_position, - apply_water_mask=apply_water_mask, - ) - output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_name) - - if bucket: - packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) - - -def insar_tops_multi_burst( - reference: Iterable[str], - secondary: Iterable[str], - swaths: list = [1, 2, 3], - looks: str = '20x4', - apply_water_mask=False, - bucket: Optional[str] = None, - bucket_prefix: str = '', -): - ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - - if len(list(set(ref_ids) - set(sec_ids))) > 0: - raise Exception( - 'The reference bursts ' - + ', '.join(list(set(ref_ids) - set(sec_ids))) - + ' do not have the correspondant bursts in the secondary granules' - ) - elif len(list(set(sec_ids) - set(ref_ids))) > 0: - raise Exception( - 'The secondary bursts ' - + ', '.join(list(set(sec_ids) - set(ref_ids))) - + ' do not have the correspondant bursts in the reference granules' - ) - - if not reference[0].split('_')[4] == secondary[0].split('_')[4]: - raise Exception('The secondary and reference granules do not have the same polarization') - - reference_safe_path = burst2safe(reference) - reference_safe = reference_safe_path.name.split('.')[0] - secondary_safe_path = burst2safe(secondary) - secondary_safe = secondary_safe_path.name.split('.')[0] - - range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] - swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) - polarization = reference[0].split('_')[4] - - log.info('Begin ISCE2 TopsApp run') - insar_tops_packaged( - reference=reference_safe, - secondary=secondary_safe, - swaths=swaths, - polarization=polarization, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=apply_water_mask, - bucket=bucket, - bucket_prefix=bucket_prefix - ) - log.info('ISCE2 TopsApp run completed successfully') - - -def main(): - """HyP3 entrypoint for the burst TOPS workflow""" - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - parser.add_argument( - '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' - ) - parser.add_argument( - '--apply-water-mask', - type=string_is_true, - default=False, - help='Apply a water body mask before unwrapping.', - ) - # Allows granules to be given as a space-delimited list of strings (e.g. foo bar) or as a single - # quoted string that contains spaces (e.g. "foo bar"). AWS Batch uses the latter format when - # invoking the container command. - parser.add_argument('--reference', type=str.split, nargs='+', help='List of granules for the reference bursts') - parser.add_argument('--secondary', type=str.split, nargs='+', help='List of granules for the secondary bursts') - - args = parser.parse_args() - - args.reference = [item for sublist in args.reference for item in sublist] - args.secondary = [item for sublist in args.secondary for item in sublist] - if len(args.reference) != len(args.secondary): - parser.error('Number of reference and secondary granules must be the same') - - configure_root_logger() - log.debug(' '.join(sys.argv)) - - if len(args.reference) == 1: - insar_tops_single_burst( - reference=args.reference[0], - secondary=args.secondary[0], - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) - else: - insar_tops_multi_burst( - reference=args.reference, - secondary=args.secondary, - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index dd836e4b..baf81724 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -27,6 +27,7 @@ def insar_tops( polarization: str = 'VV', azimuth_looks: int = 4, range_looks: int = 20, + apply_water_mask: bool = False, download: bool = True, ) -> Path: """Create a full-SLC interferogram @@ -79,7 +80,19 @@ def insar_tops( ) config_path = config.write_template('topsApp.xml') - topsapp.run_topsapp_burst(start='startup', end='unwrap2stage', config_xml=config_path) + if apply_water_mask: + topsapp.run_topsapp_burst(start='startup', end='filter', config_xml=config_path) + water_mask_path = 'water_mask.wgs84' + create_water_mask(str(dem_path), water_mask_path) + multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) + multilook('merged/lat.rdr.full', outname='merged/lat.rdr', alks=azimuth_looks, rlks=range_looks) + resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') + isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') + image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') + topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) + isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') + else: + topsapp.run_topsapp_burst(start='startup', end='unwrap2stage', config_xml=config_path) copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) @@ -118,10 +131,10 @@ def insar_tops_packaged( pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') log.info('Begin ISCE2 TopsApp run') - if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{reference}.SAFE'): - insar_tops(reference, secondary, download=False) + if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE'): + insar_tops(reference, secondary, apply_water_mask=apply_water_mask, download=False) else: - insar_tops(reference, secondary) + insar_tops(reference, secondary, apply_water_mask=apply_water_mask) log.info('ISCE2 TopsApp run completed successfully') product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) From 224fea068d6fff06820e31e48968e617d23eff56 Mon Sep 17 00:00:00 2001 From: Mario Angarita Date: Thu, 15 Aug 2024 23:46:58 -0800 Subject: [PATCH 25/81] flake8 --- src/hyp3_isce2/insar_tops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index baf81724..39e17847 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -8,13 +8,15 @@ from shutil import copyfile, make_archive from hyp3lib.util import string_is_true +from isceobj.TopsProc.runMergeBursts import multilook from s1_orbits import fetch_for_scene from hyp3_isce2 import packaging, slc, topsapp from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal -from hyp3_isce2.utils import make_browse_image +from hyp3_isce2.utils import image_math, isce2_copy, make_browse_image, resample_to_radar_io +from hyp3_isce2.water_mask import create_water_mask log = logging.getLogger(__name__) From 9946ba3218a8c37a519dc3eeae41ad2be19873d8 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 10:45:53 -0500 Subject: [PATCH 26/81] created test_packaging --- tests/test_burst.py | 15 --------------- tests/test_insar_tops_burst.py | 7 ------- tests/test_packaging.py | 23 +++++++++++++++++++++++ 3 files changed, 23 insertions(+), 22 deletions(-) delete mode 100644 tests/test_insar_tops_burst.py create mode 100644 tests/test_packaging.py diff --git a/tests/test_burst.py b/tests/test_burst.py index 645a6759..318d6c31 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -2,7 +2,6 @@ from collections import namedtuple from datetime import datetime from pathlib import Path -from re import match from unittest.mock import patch import asf_search @@ -107,20 +106,6 @@ def test_get_region_of_interest(tmp_path, orbit): assert not roi.intersects(ref_bbox_post) -def test_get_product_name(): - reference_name = 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' - secondary_name = 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' - - name_20m = burst.get_product_name(reference_name, secondary_name, pixel_spacing=20.0) - name_80m = burst.get_product_name(reference_name, secondary_name, pixel_spacing=80) - - assert match('[A-F0-9]{4}', name_20m[-4:]) is not None - assert match('[A-F0-9]{4}', name_80m[-4:]) is not None - - assert name_20m.startswith('S1_136231_IW2_20200604_20200616_VV_INT20') - assert name_80m.startswith('S1_136231_IW2_20200604_20200616_VV_INT80') - - def mock_asf_search_results( slc_name: str, subswath: str, polarization: str, burst_index: int ) -> asf_search.ASFSearchResults: diff --git a/tests/test_insar_tops_burst.py b/tests/test_insar_tops_burst.py deleted file mode 100644 index 28690f6a..00000000 --- a/tests/test_insar_tops_burst.py +++ /dev/null @@ -1,7 +0,0 @@ -from hyp3_isce2 import insar_tops_burst - - -def test_get_pixel_size(): - assert insar_tops_burst.get_pixel_size('20x4') == 80.0 - assert insar_tops_burst.get_pixel_size('10x2') == 40.0 - assert insar_tops_burst.get_pixel_size('5x1') == 20.0 diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 00000000..ac9fadee --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,23 @@ +from re import match + +from hyp3_isce2 import packaging + + +def test_get_product_name(): + reference_name = 'S1_136231_IW2_20200604T022312_VV_7C85-BURST' + secondary_name = 'S1_136231_IW2_20200616T022313_VV_5D11-BURST' + + name_20m = packaging.get_product_name(reference_name, secondary_name, pixel_spacing=20.0, slc=False) + name_80m = packaging.get_product_name(reference_name, secondary_name, pixel_spacing=80, slc=False) + + assert match('[A-F0-9]{4}', name_20m[-4:]) is not None + assert match('[A-F0-9]{4}', name_80m[-4:]) is not None + + assert name_20m.startswith('S1_136231_IW2_20200604_20200616_VV_INT20') + assert name_80m.startswith('S1_136231_IW2_20200604_20200616_VV_INT80') + + +def test_get_pixel_size(): + assert packaging.get_pixel_size('20x4') == 80.0 + assert packaging.get_pixel_size('10x2') == 40.0 + assert packaging.get_pixel_size('5x1') == 20.0 From 74c565fe54df3c7b01a9f14bfdd23a3cc56ed78b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 15:48:28 +0000 Subject: [PATCH 27/81] update coverage image --- images/coverage.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 012a8497..ffd257bd 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 70% - 70% + 71% + 71% From 4409aecc9461a6a62d8c3d83c03482c796b56161 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 10:52:58 -0500 Subject: [PATCH 28/81] fixed slc test --- tests/test_slc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_slc.py b/tests/test_slc.py index 028de11a..6db1f343 100644 --- a/tests/test_slc.py +++ b/tests/test_slc.py @@ -51,7 +51,7 @@ def test_get_geometry_from_kml(test_data_dir): def test_dem_bounds(mocker): - mocker.patch('hyp3_isce2.slc.get_geometry_from_kml') - slc.get_geometry_from_kml.side_effect = [box(-1, -1, 1, 1), box(0, 0, 2, 2)] + mocker.patch('hyp3_isce2.slc.get_geometry_from_manifest') + slc.get_geometry_from_manifest.side_effect = [box(-1, -1, 1, 1), box(0, 0, 2, 2)] bounds = slc.get_dem_bounds(Path('ref'), Path('sec')) assert bounds == (0, 0, 1, 1) From 745454cb6b344cc013cd52edcf2c6e19dcf464f1 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 14:42:06 -0500 Subject: [PATCH 29/81] change interface back to granules --- src/hyp3_isce2/insar_tops_burst.py | 33 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 544bf645..f5bfda9f 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -270,26 +270,31 @@ def main(): default=False, help='Apply a water body mask before unwrapping.', ) - # Allows granules to be given as a space-delimited list of strings (e.g. foo bar) or as a single - # quoted string that contains spaces (e.g. "foo bar"). AWS Batch uses the latter format when - # invoking the container command. - parser.add_argument('--reference', type=str.split, nargs='+', help='List of granules for the reference bursts') - parser.add_argument('--secondary', type=str.split, nargs='+', help='List of granules for the secondary bursts') + parser.add_argument( + 'granules', + type=str.split, + nargs='+', + help='List of references in quotes and list of secondaries in quotes i.e. "ref" "sec" or "ref1 ref2" "sec1 sec2"' + ) args = parser.parse_args() - args.reference = [item for sublist in args.reference for item in sublist] - args.secondary = [item for sublist in args.secondary for item in sublist] - if len(args.reference) != len(args.secondary): - parser.error('Number of reference and secondary granules must be the same') + granules = args.granules + if len(granules) != 2: + parser.error('No more than two lists of granules may be provided.') + if len(granules[0]) != len(granules[1]): + parser.error('Number of references must match the number of secondaries.') + + references = granules[0] + secondaries = granules[1] configure_root_logger() log.debug(' '.join(sys.argv)) - if len(args.reference) == 1: + if len(references) == 1: insar_tops_single_burst( - reference=args.reference[0], - secondary=args.secondary[0], + reference=references[0], + secondary=secondaries[0], looks=args.looks, apply_water_mask=args.apply_water_mask, bucket=args.bucket, @@ -297,8 +302,8 @@ def main(): ) else: insar_tops_multi_burst( - reference=args.reference, - secondary=args.secondary, + reference=references, + secondary=secondaries, looks=args.looks, apply_water_mask=args.apply_water_mask, bucket=args.bucket, From e44aea4a140e40f4ac01ba9ab60074881e1a694c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 14:42:23 -0500 Subject: [PATCH 30/81] add back parameters --- src/hyp3_isce2/insar_tops.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 39e17847..717e912c 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -133,10 +133,20 @@ def insar_tops_packaged( pixel_size = packaging.get_pixel_size(f'{range_looks}x{azimuth_looks}') log.info('Begin ISCE2 TopsApp run') - if os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE'): - insar_tops(reference, secondary, apply_water_mask=apply_water_mask, download=False) - else: - insar_tops(reference, secondary, apply_water_mask=apply_water_mask) + + do_download = os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE') + + insar_tops( + reference=reference, + secondary=secondary, + swaths=swaths, + polarization=polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + download=do_download + ) + log.info('ISCE2 TopsApp run completed successfully') product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) From 260aceae29dee2e29e1408f07616d8a8873e153b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 14:45:02 -0500 Subject: [PATCH 31/81] shortened help --- src/hyp3_isce2/insar_tops_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index f5bfda9f..c09213c4 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -274,7 +274,7 @@ def main(): 'granules', type=str.split, nargs='+', - help='List of references in quotes and list of secondaries in quotes i.e. "ref" "sec" or "ref1 ref2" "sec1 sec2"' + help='List of references and list of secondaries i.e. ref sec or "ref1 ref2" "sec1 sec2"' ) args = parser.parse_args() From fe3013b8b4cf791b1facc5f2a37f45600400b37b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 15:10:33 -0500 Subject: [PATCH 32/81] update readme --- README.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7c5c370b..6328fc78 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,28 @@ The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python - `insar_tops_burst`: A workflow for creating multi-bursts Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow --- -To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow: +To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow with a single burst pair: ``` python -m hyp3_isce2 ++process insar_tops_burst \ - --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136231_IW3_20200604T022313_VV_7C85-BURST\ - --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136231_IW3_20200616T022314_VV_5D11-BURST\ + S1_136231_IW2_20200604T022312_VV_7C85-BURST\ + S1_136231_IW2_20200616T022313_VV_5D11-BURST\ --looks 20x4 \ --apply-water-mask True ``` -This command will create a Sentinel-1 interferogram that contains a deformation signal related to a -2020 Iranian earthquake. +and, for multiple burst pairs: + +``` +python -m hyp3_isce2 ++process insar_tops_burst \ + "S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST"\ + "S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST"\ + --looks 20x4 \ + --apply-water-mask True +``` + +These commands will both create a Sentinel-1 interferogram that contains a deformation signal related to a +2020 Iranian earthquake. ### Product Merging Utility Usage **This feature is under active development and is subject to change!** From 02f08adf8e29c74bac81f73159a392ed69adcfd9 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 16 Aug 2024 15:19:07 -0500 Subject: [PATCH 33/81] fixed param names --- src/hyp3_isce2/insar_tops.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 717e912c..dd988f42 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -109,7 +109,6 @@ def insar_tops_packaged( azimuth_looks: int = 4, range_looks: int = 20, apply_water_mask: bool = True, - download: bool = True, bucket: str = None, bucket_prefix: str = '', ) -> Path: @@ -137,8 +136,8 @@ def insar_tops_packaged( do_download = os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE') insar_tops( - reference=reference, - secondary=secondary, + reference_scene=reference, + secondary_scene=secondary, swaths=swaths, polarization=polarization, azimuth_looks=azimuth_looks, From 012731443ae9dbc0d1d9bddb859c90536a972b98 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:27:37 -0500 Subject: [PATCH 34/81] separated mutli burst workflow --- pyproject.toml | 2 + src/hyp3_isce2/__main__.py | 2 +- src/hyp3_isce2/insar_tops_burst.py | 91 ++-------------- src/hyp3_isce2/insar_tops_multi_burst.py | 133 +++++++++++++++++++++++ 4 files changed, 148 insertions(+), 80 deletions(-) create mode 100644 src/hyp3_isce2/insar_tops_multi_burst.py diff --git a/pyproject.toml b/pyproject.toml index 1a573a03..6d59bc2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,12 +53,14 @@ Documentation = "https://hyp3-docs.asf.alaska.edu" [project.scripts] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" +insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" [project.entry-points.hyp3] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" +insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" diff --git a/src/hyp3_isce2/__main__.py b/src/hyp3_isce2/__main__.py index 9d50385e..a4a05a42 100644 --- a/src/hyp3_isce2/__main__.py +++ b/src/hyp3_isce2/__main__.py @@ -19,7 +19,7 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '++process', - choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts'], + choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts', 'insar_tops_multi_burst'], default='insar_tops_burst', help='Select the HyP3 entrypoint to use', # HyP3 entrypoints are specified in `pyproject.toml` ) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index c09213c4..70831471 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -8,7 +8,6 @@ from typing import Iterable, Optional import isce # noqa -from burst2safe.burst2safe import burst2safe from hyp3lib.util import string_is_true from isceobj.TopsProc.runMergeBursts import multilook from osgeo import gdal @@ -24,7 +23,6 @@ validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.insar_tops import insar_tops_packaged from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( @@ -203,58 +201,6 @@ def insar_tops_single_burst( packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) -def insar_tops_multi_burst( - reference: Iterable[str], - secondary: Iterable[str], - swaths: list = [1, 2, 3], - looks: str = '20x4', - apply_water_mask=False, - bucket: Optional[str] = None, - bucket_prefix: str = '', -): - ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - - if len(list(set(ref_ids) - set(sec_ids))) > 0: - raise Exception( - 'The reference bursts ' - + ', '.join(list(set(ref_ids) - set(sec_ids))) - + ' do not have the correspondant bursts in the secondary granules' - ) - elif len(list(set(sec_ids) - set(ref_ids))) > 0: - raise Exception( - 'The secondary bursts ' - + ', '.join(list(set(sec_ids) - set(ref_ids))) - + ' do not have the correspondant bursts in the reference granules' - ) - - if not reference[0].split('_')[4] == secondary[0].split('_')[4]: - raise Exception('The secondary and reference granules do not have the same polarization') - - reference_safe_path = burst2safe(reference) - reference_safe = reference_safe_path.name.split('.')[0] - secondary_safe_path = burst2safe(secondary) - secondary_safe = secondary_safe_path.name.split('.')[0] - - range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] - swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) - polarization = reference[0].split('_')[4] - - log.info('Begin ISCE2 TopsApp run') - insar_tops_packaged( - reference=reference_safe, - secondary=secondary_safe, - swaths=swaths, - polarization=polarization, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=apply_water_mask, - bucket=bucket, - bucket_prefix=bucket_prefix - ) - log.info('ISCE2 TopsApp run completed successfully') - - def main(): """HyP3 entrypoint for the burst TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -274,38 +220,25 @@ def main(): 'granules', type=str.split, nargs='+', - help='List of references and list of secondaries i.e. ref sec or "ref1 ref2" "sec1 sec2"' + help='Reference and secondary scene names' ) args = parser.parse_args() - granules = args.granules + granules = [item for sublist in args.granules for item in sublist] if len(granules) != 2: parser.error('No more than two lists of granules may be provided.') - if len(granules[0]) != len(granules[1]): - parser.error('Number of references must match the number of secondaries.') - - references = granules[0] - secondaries = granules[1] + if len(granules[0]) != len(granules[1]) != 1: + parser.error('Must include 1 reference and 1 secondary.') configure_root_logger() log.debug(' '.join(sys.argv)) - if len(references) == 1: - insar_tops_single_burst( - reference=references[0], - secondary=secondaries[0], - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) - else: - insar_tops_multi_burst( - reference=references, - secondary=secondaries, - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) + insar_tops_single_burst( + reference=granules[0], + secondary=granules[1], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) \ No newline at end of file diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py new file mode 100644 index 00000000..93f67e3b --- /dev/null +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -0,0 +1,133 @@ +"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" + +import argparse +import logging +import sys +from typing import Iterable, Optional + +import isce # noqa +from burst2safe.burst2safe import burst2safe +from hyp3lib.util import string_is_true +from osgeo import gdal + +from hyp3_isce2.insar_tops import insar_tops_packaged +from hyp3_isce2.insar_tops_burst import insar_tops_single_burst +from hyp3_isce2.logger import configure_root_logger + + +gdal.UseExceptions() + +log = logging.getLogger(__name__) + + +def insar_tops_multi_burst( + reference: Iterable[str], + secondary: Iterable[str], + swaths: list = [1, 2, 3], + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + + if len(list(set(ref_ids) - set(sec_ids))) > 0: + raise Exception( + 'The reference bursts ' + + ', '.join(list(set(ref_ids) - set(sec_ids))) + + ' do not have the correspondant bursts in the secondary granules' + ) + elif len(list(set(sec_ids) - set(ref_ids))) > 0: + raise Exception( + 'The secondary bursts ' + + ', '.join(list(set(sec_ids) - set(ref_ids))) + + ' do not have the correspondant bursts in the reference granules' + ) + + if not reference[0].split('_')[4] == secondary[0].split('_')[4]: + raise Exception('The secondary and reference granules do not have the same polarization') + + reference_safe_path = burst2safe(reference) + reference_safe = reference_safe_path.name.split('.')[0] + secondary_safe_path = burst2safe(secondary) + secondary_safe = secondary_safe_path.name.split('.')[0] + + range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) + polarization = reference[0].split('_')[4] + + log.info('Begin ISCE2 TopsApp run') + insar_tops_packaged( + reference=reference_safe, + secondary=secondary_safe, + swaths=swaths, + polarization=polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + bucket=bucket, + bucket_prefix=bucket_prefix + ) + log.info('ISCE2 TopsApp run completed successfully') + + +def main(): + """HyP3 entrypoint for the burst TOPS workflow""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') + parser.add_argument( + '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' + ) + parser.add_argument( + '--apply-water-mask', + type=string_is_true, + default=False, + help='Apply a water body mask before unwrapping.', + ) + parser.add_argument( + '--reference', + type=str.split, + nargs='+', + help='List of reference scenes"' + ) + parser.add_argument( + '--secondary', + type=str.split, + nargs='+', + help='List of secondary scenes"' + ) + + args = parser.parse_args() + + references = [item for sublist in args.reference for item in sublist] + secondaries = [item for sublist in args.secondary for item in sublist] + + if (len(references) < 1 or len(secondaries) < 1): + parser.error("Must include at least 1 reference and 1 secondary") + if (len(references) != len(secondaries)): + parser.error("Must have the same number of references and secondaries") + + configure_root_logger() + log.debug(' '.join(sys.argv)) + + if len(references) == 1: + insar_tops_single_burst( + reference=references[0], + secondary=secondaries[0], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) + else: + insar_tops_multi_burst( + reference=references, + secondary=secondaries, + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) From df1a03574ffb8dbae0e022191eaaeb275299243a Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:29:25 -0500 Subject: [PATCH 35/81] updated readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6328fc78..098b63f5 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,9 @@ python -m hyp3_isce2 ++process insar_tops_burst \ and, for multiple burst pairs: ``` -python -m hyp3_isce2 ++process insar_tops_burst \ - "S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST"\ - "S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST"\ +python -m hyp3_isce2 ++process insar_tops_multi_burst \ + --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST\ + --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST\ --looks 20x4 \ --apply-water-mask True ``` From 9561f979cf2f834044c1984e10e9b75aa8ebf5b6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:31:20 -0500 Subject: [PATCH 36/81] updated changelog --- CHANGELOG.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34190aed..7b3d7a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,8 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.0.0] ### Added -- New funcionality to process multi bursts with `insar_tops_burst`. -- The `--reference` and `--secondary` command-line options to indicate the reference and secondary bursts. -- `burst2safe` is now called to get SAFE files from bursts. -### Changed -- Product name has changed to indicate relative orbit, lon/lat extent and reference and secondary dates. - +- `insar_tops_multi_burst` workflow which processes multiple bursts as one SLC. + ## [2.0.0] ### Changed - Orbit files are now retrieved using the [s1-orbits](https://github.com/ASFHyP3/sentinel1-orbits-py) library. From e1bb6fd5a7407bf1f4c998ba8d6347cf40c143c8 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:32:08 -0500 Subject: [PATCH 37/81] flake8 --- src/hyp3_isce2/insar_tops_burst.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 70831471..9ea532f8 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -5,7 +5,7 @@ import sys from pathlib import Path from shutil import copyfile, make_archive -from typing import Iterable, Optional +from typing import Optional import isce # noqa from hyp3lib.util import string_is_true @@ -241,4 +241,4 @@ def main(): apply_water_mask=args.apply_water_mask, bucket=args.bucket, bucket_prefix=args.bucket_prefix, - ) \ No newline at end of file + ) From 5304683fd87b269272255dcd67e87dfc26d8345e Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:33:15 -0500 Subject: [PATCH 38/81] changelog wording --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b3d7a1b..95f1688a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.0.0] ### Added -- `insar_tops_multi_burst` workflow which processes multiple bursts as one SLC. +- `insar_tops_multi_burst` workflow for processing multiple bursts as one SLC. ## [2.0.0] ### Changed From 79d659ff60fb299e79baf4585d1928f67ba5a698 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:36:01 +0000 Subject: [PATCH 39/81] update coverage image --- images/coverage.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index ffd257bd..012a8497 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 71% - 71% + 70% + 70% From 13513cf6590aff3c18ec731e404e486c2a0f6ce2 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:48:43 -0500 Subject: [PATCH 40/81] Update CHANGELOG.md Co-authored-by: Jake Herrmann --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f1688a..b4eb568c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.0.0] +## [2.1.0] ### Added - `insar_tops_multi_burst` workflow for processing multiple bursts as one SLC. From 91c4525d46135ab175c02aed4c73f12e695d1826 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:49:19 -0500 Subject: [PATCH 41/81] Update src/hyp3_isce2/insar_tops_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 9ea532f8..e9a0dd23 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -227,7 +227,7 @@ def main(): granules = [item for sublist in args.granules for item in sublist] if len(granules) != 2: - parser.error('No more than two lists of granules may be provided.') + parser.error('No more than two granules may be provided.') if len(granules[0]) != len(granules[1]) != 1: parser.error('Must include 1 reference and 1 secondary.') From 3a6aee84ad149ab3200c655b48f1d0c3fc2a5ded Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 14:53:40 -0500 Subject: [PATCH 42/81] Update src/hyp3_isce2/insar_tops_multi_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_multi_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index 93f67e3b..1474c056 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -1,4 +1,4 @@ -"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" +"""Create a multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" import argparse import logging From ceb381fff01c0b5cf429f45b61772e99d8a2b1cb Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:02:38 -0500 Subject: [PATCH 43/81] check if ref and sec bursts match --- src/hyp3_isce2/insar_tops_multi_burst.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index 93f67e3b..816888af 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -32,21 +32,8 @@ def insar_tops_multi_burst( ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - if len(list(set(ref_ids) - set(sec_ids))) > 0: - raise Exception( - 'The reference bursts ' - + ', '.join(list(set(ref_ids) - set(sec_ids))) - + ' do not have the correspondant bursts in the secondary granules' - ) - elif len(list(set(sec_ids) - set(ref_ids))) > 0: - raise Exception( - 'The secondary bursts ' - + ', '.join(list(set(sec_ids) - set(ref_ids))) - + ' do not have the correspondant bursts in the reference granules' - ) - - if not reference[0].split('_')[4] == secondary[0].split('_')[4]: - raise Exception('The secondary and reference granules do not have the same polarization') + if ref_ids != sec_ids: + raise Exception('The reference bursts and secondary bursts do not match') reference_safe_path = burst2safe(reference) reference_safe = reference_safe_path.name.split('.')[0] From 22e1ddc51a946c07403162415efdc70b1c8206e7 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:06:16 -0500 Subject: [PATCH 44/81] Update src/hyp3_isce2/insar_tops_multi_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_multi_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index f4cfb701..24151dd2 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -40,7 +40,7 @@ def insar_tops_multi_burst( secondary_safe_path = burst2safe(secondary) secondary_safe = secondary_safe_path.name.split('.')[0] - range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + range_looks, azimuth_looks = [int(value) for value in looks.split('x')] swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) polarization = reference[0].split('_')[4] From d629700e9f8fbf0241a369e3c780be0172a2c100 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:16:54 -0500 Subject: [PATCH 45/81] Update src/hyp3_isce2/insar_tops_multi_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_multi_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index 24151dd2..45fb64ba 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -41,7 +41,7 @@ def insar_tops_multi_burst( secondary_safe = secondary_safe_path.name.split('.')[0] range_looks, azimuth_looks = [int(value) for value in looks.split('x')] - swaths = list(set([int(granule.split('_')[2][2]) for granule in reference])) + swaths = list(set(int(granule.split('_')[2][2]) for granule in reference)) polarization = reference[0].split('_')[4] log.info('Begin ISCE2 TopsApp run') From fef1dcb85110fb16aa57906c702fc96680e432f5 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:21:50 -0500 Subject: [PATCH 46/81] Update src/hyp3_isce2/insar_tops_multi_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_multi_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index 45fb64ba..6640aac6 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -92,7 +92,7 @@ def main(): references = [item for sublist in args.reference for item in sublist] secondaries = [item for sublist in args.secondary for item in sublist] - if (len(references) < 1 or len(secondaries) < 1): + if len(references) < 1 or len(secondaries) < 1: parser.error("Must include at least 1 reference and 1 secondary") if (len(references) != len(secondaries)): parser.error("Must have the same number of references and secondaries") From e10a0406132881f25e7f81ce0090220da8b8c40f Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Mon, 19 Aug 2024 15:22:01 -0500 Subject: [PATCH 47/81] Update src/hyp3_isce2/insar_tops_multi_burst.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_multi_burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py index 6640aac6..44d12b3c 100644 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ b/src/hyp3_isce2/insar_tops_multi_burst.py @@ -94,7 +94,7 @@ def main(): if len(references) < 1 or len(secondaries) < 1: parser.error("Must include at least 1 reference and 1 secondary") - if (len(references) != len(secondaries)): + if len(references) != len(secondaries): parser.error("Must have the same number of references and secondaries") configure_root_logger() From 41869e82397715b8cf5bb26de46494073ad0f9af Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 19 Aug 2024 15:03:07 -0800 Subject: [PATCH 48/81] misc readme and style fixes --- README.md | 20 ++++++++++---------- src/hyp3_isce2/insar_tops_burst.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 098b63f5..8cd64f65 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,18 @@ The HyP3-ISCE2 plugin provides a set of workflows to process SAR satellite data ## Usage The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python or via a CLI) that can be used to process SAR data using ISCE2. The workflows currently included in this plugin are: -- `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow -- `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow -- `insar_tops_burst`: A workflow for creating multi-bursts Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow +- `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow +- `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow +- `insar_tops_burst`: A workflow for creating single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow +- `insar_tops_multi_burst`: A workflow for creating multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow --- -To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow with a single burst pair: +To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow: ``` python -m hyp3_isce2 ++process insar_tops_burst \ - S1_136231_IW2_20200604T022312_VV_7C85-BURST\ - S1_136231_IW2_20200616T022313_VV_5D11-BURST\ + S1_136231_IW2_20200604T022312_VV_7C85-BURST \ + S1_136231_IW2_20200616T022313_VV_5D11-BURST \ --looks 20x4 \ --apply-water-mask True ``` @@ -27,14 +28,13 @@ and, for multiple burst pairs: ``` python -m hyp3_isce2 ++process insar_tops_multi_burst \ - --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST\ - --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST\ + --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST \ + --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST \ --looks 20x4 \ --apply-water-mask True ``` -These commands will both create a Sentinel-1 interferogram that contains a deformation signal related to a -2020 Iranian earthquake. +These commands will both create a Sentinel-1 interferogram that contains a deformation signal related to a 2020 Iranian earthquake. ### Product Merging Utility Usage **This feature is under active development and is subject to change!** diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index e9a0dd23..a042fcef 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -146,7 +146,7 @@ def insar_tops_single_burst( reference, secondary = oldest_granule_first(reference, secondary) validate_bursts(reference, secondary) swath_number = int(reference[12]) - range_looks, azimuth_looks = [int(looks) for looks in looks.split('x')] + range_looks, azimuth_looks = [int(value) for value in looks.split('x')] log.info('Begin ISCE2 TopsApp run') From 7f989fd82e2dc6a7245ad3a03833cf63876ccdc6 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 20 Aug 2024 13:58:03 -0500 Subject: [PATCH 49/81] switch back to adding multiburst to insar_tops_burst --- src/hyp3_isce2/insar_tops_burst.py | 95 ++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index a042fcef..56124cfc 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -1,13 +1,14 @@ -"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" +"""Create a Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow from a set of bursts""" import argparse import logging import sys from pathlib import Path from shutil import copyfile, make_archive -from typing import Optional +from typing import Iterable, Optional import isce # noqa +from burst2safe.burst2safe import burst2safe from hyp3lib.util import string_is_true from isceobj.TopsProc.runMergeBursts import multilook from osgeo import gdal @@ -23,6 +24,7 @@ validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.insar_tops import insar_tops_packaged from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( @@ -201,6 +203,45 @@ def insar_tops_single_burst( packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix) +def insar_tops_multi_burst( + reference: Iterable[str], + secondary: Iterable[str], + swaths: list = [1, 2, 3], + looks: str = '20x4', + apply_water_mask=False, + bucket: Optional[str] = None, + bucket_prefix: str = '', +): + ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + + if ref_ids != sec_ids: + raise Exception('The reference bursts and secondary bursts do not match') + + reference_safe_path = burst2safe(reference) + reference_safe = reference_safe_path.name.split('.')[0] + secondary_safe_path = burst2safe(secondary) + secondary_safe = secondary_safe_path.name.split('.')[0] + + range_looks, azimuth_looks = [int(value) for value in looks.split('x')] + swaths = list(set(int(granule.split('_')[2][2]) for granule in reference)) + polarization = reference[0].split('_')[4] + + log.info('Begin ISCE2 TopsApp run') + insar_tops_packaged( + reference=reference_safe, + secondary=secondary_safe, + swaths=swaths, + polarization=polarization, + azimuth_looks=azimuth_looks, + range_looks=range_looks, + apply_water_mask=apply_water_mask, + bucket=bucket, + bucket_prefix=bucket_prefix, + ) + log.info('ISCE2 TopsApp run completed successfully') + + def main(): """HyP3 entrypoint for the burst TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -211,34 +252,38 @@ def main(): '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' ) parser.add_argument( - '--apply-water-mask', - type=string_is_true, - default=False, - help='Apply a water body mask before unwrapping.', - ) - parser.add_argument( - 'granules', - type=str.split, - nargs='+', - help='Reference and secondary scene names' + '--apply-water-mask', type=string_is_true, default=False, help='Apply a water body mask before unwrapping.' ) + parser.add_argument('--reference', type=str.split, nargs='+', help='List of reference scenes"') + parser.add_argument('--secondary', type=str.split, nargs='+', help='List of secondary scenes"') args = parser.parse_args() - granules = [item for sublist in args.granules for item in sublist] - if len(granules) != 2: - parser.error('No more than two granules may be provided.') - if len(granules[0]) != len(granules[1]) != 1: - parser.error('Must include 1 reference and 1 secondary.') + references = [item for sublist in args.reference for item in sublist] + secondaries = [item for sublist in args.secondary for item in sublist] + if len(references) < 1 or len(secondaries) < 1: + parser.error('Must include at least 1 reference and 1 secondary') + if len(references) != len(secondaries): + parser.error('Must have the same number of references and secondaries') configure_root_logger() log.debug(' '.join(sys.argv)) - insar_tops_single_burst( - reference=granules[0], - secondary=granules[1], - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) + if len(references) == 1: + insar_tops_single_burst( + reference=references[0], + secondary=secondaries[0], + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) + else: + insar_tops_multi_burst( + reference=references, + secondary=secondaries, + looks=args.looks, + apply_water_mask=args.apply_water_mask, + bucket=args.bucket, + bucket_prefix=args.bucket_prefix, + ) From 7a09fa01928310419f85d4db6fccda3df2493984 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 20 Aug 2024 13:59:31 -0500 Subject: [PATCH 50/81] get rid of insar_tops_multi_burst --- pyproject.toml | 1 - src/hyp3_isce2/__main__.py | 2 +- src/hyp3_isce2/insar_tops_multi_burst.py | 120 ----------------------- 3 files changed, 1 insertion(+), 122 deletions(-) delete mode 100644 src/hyp3_isce2/insar_tops_multi_burst.py diff --git a/pyproject.toml b/pyproject.toml index 6d59bc2d..7c75a677 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" [project.entry-points.hyp3] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" -insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" diff --git a/src/hyp3_isce2/__main__.py b/src/hyp3_isce2/__main__.py index a4a05a42..9d50385e 100644 --- a/src/hyp3_isce2/__main__.py +++ b/src/hyp3_isce2/__main__.py @@ -19,7 +19,7 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '++process', - choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts', 'insar_tops_multi_burst'], + choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts'], default='insar_tops_burst', help='Select the HyP3 entrypoint to use', # HyP3 entrypoints are specified in `pyproject.toml` ) diff --git a/src/hyp3_isce2/insar_tops_multi_burst.py b/src/hyp3_isce2/insar_tops_multi_burst.py deleted file mode 100644 index 44d12b3c..00000000 --- a/src/hyp3_isce2/insar_tops_multi_burst.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Create a multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow""" - -import argparse -import logging -import sys -from typing import Iterable, Optional - -import isce # noqa -from burst2safe.burst2safe import burst2safe -from hyp3lib.util import string_is_true -from osgeo import gdal - -from hyp3_isce2.insar_tops import insar_tops_packaged -from hyp3_isce2.insar_tops_burst import insar_tops_single_burst -from hyp3_isce2.logger import configure_root_logger - - -gdal.UseExceptions() - -log = logging.getLogger(__name__) - - -def insar_tops_multi_burst( - reference: Iterable[str], - secondary: Iterable[str], - swaths: list = [1, 2, 3], - looks: str = '20x4', - apply_water_mask=False, - bucket: Optional[str] = None, - bucket_prefix: str = '', -): - ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - - if ref_ids != sec_ids: - raise Exception('The reference bursts and secondary bursts do not match') - - reference_safe_path = burst2safe(reference) - reference_safe = reference_safe_path.name.split('.')[0] - secondary_safe_path = burst2safe(secondary) - secondary_safe = secondary_safe_path.name.split('.')[0] - - range_looks, azimuth_looks = [int(value) for value in looks.split('x')] - swaths = list(set(int(granule.split('_')[2][2]) for granule in reference)) - polarization = reference[0].split('_')[4] - - log.info('Begin ISCE2 TopsApp run') - insar_tops_packaged( - reference=reference_safe, - secondary=secondary_safe, - swaths=swaths, - polarization=polarization, - azimuth_looks=azimuth_looks, - range_looks=range_looks, - apply_water_mask=apply_water_mask, - bucket=bucket, - bucket_prefix=bucket_prefix - ) - log.info('ISCE2 TopsApp run completed successfully') - - -def main(): - """HyP3 entrypoint for the burst TOPS workflow""" - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') - parser.add_argument( - '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' - ) - parser.add_argument( - '--apply-water-mask', - type=string_is_true, - default=False, - help='Apply a water body mask before unwrapping.', - ) - parser.add_argument( - '--reference', - type=str.split, - nargs='+', - help='List of reference scenes"' - ) - parser.add_argument( - '--secondary', - type=str.split, - nargs='+', - help='List of secondary scenes"' - ) - - args = parser.parse_args() - - references = [item for sublist in args.reference for item in sublist] - secondaries = [item for sublist in args.secondary for item in sublist] - - if len(references) < 1 or len(secondaries) < 1: - parser.error("Must include at least 1 reference and 1 secondary") - if len(references) != len(secondaries): - parser.error("Must have the same number of references and secondaries") - - configure_root_logger() - log.debug(' '.join(sys.argv)) - - if len(references) == 1: - insar_tops_single_burst( - reference=references[0], - secondary=secondaries[0], - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) - else: - insar_tops_multi_burst( - reference=references, - secondary=secondaries, - looks=args.looks, - apply_water_mask=args.apply_water_mask, - bucket=args.bucket, - bucket_prefix=args.bucket_prefix, - ) From 5925eb83e0006751f17bf7c827aec768e9e37341 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 20 Aug 2024 14:08:25 -0500 Subject: [PATCH 51/81] update documentation --- CHANGELOG.md | 7 +++++-- README.md | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4eb568c..65615c0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [2.1.0] +## [3.0.0] ### Added -- `insar_tops_multi_burst` workflow for processing multiple bursts as one SLC. +- The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. + +### Changed +- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` arguments instead of one `--granules` argument ## [2.0.0] ### Changed diff --git a/README.md b/README.md index 8cd64f65..dab03143 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python - `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow - `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow -- `insar_tops_burst`: A workflow for creating single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow +- `insar_tops_burst`: A workflow for creating burst-based Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow - `insar_tops_multi_burst`: A workflow for creating multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow --- @@ -18,8 +18,8 @@ To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [W ``` python -m hyp3_isce2 ++process insar_tops_burst \ - S1_136231_IW2_20200604T022312_VV_7C85-BURST \ - S1_136231_IW2_20200616T022313_VV_5D11-BURST \ + --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST \ + --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST \ --looks 20x4 \ --apply-water-mask True ``` From cbfbecd4c0c3dc6e77b48b8144d176093fb92116 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 20 Aug 2024 14:35:20 -0500 Subject: [PATCH 52/81] add function to test granule order --- CHANGELOG.md | 1 + src/hyp3_isce2/insar_tops_burst.py | 8 +++++--- src/hyp3_isce2/utils.py | 29 ++++++++++++++++++++++++++--- tests/test_utils.py | 25 +++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65615c0c..9accc28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.0.0] ### Added - The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. +- Function to check if the reference granules are older than the secondary granules. ### Changed - The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` arguments instead of one `--granules` argument diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 56124cfc..f3532095 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -28,10 +28,10 @@ from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( + check_older_granule_is_reference, image_math, isce2_copy, make_browse_image, - oldest_granule_first, resample_to_radar_io, ) from hyp3_isce2.water_mask import create_water_mask @@ -145,7 +145,7 @@ def insar_tops_single_burst( bucket: Optional[str] = None, bucket_prefix: str = '', ): - reference, secondary = oldest_granule_first(reference, secondary) + check_older_granule_is_reference(reference, secondary) validate_bursts(reference, secondary) swath_number = int(reference[12]) range_looks, azimuth_looks = [int(value) for value in looks.split('x')] @@ -216,7 +216,9 @@ def insar_tops_multi_burst( sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] if ref_ids != sec_ids: - raise Exception('The reference bursts and secondary bursts do not match') + raise Exception('The reference burst(s) and secondary burst(s) do not match') + + check_older_granule_is_reference(reference, secondary) reference_safe_path = burst2safe(reference) reference_safe = reference_safe_path.name.split('.')[0] diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index b1d7d191..ffe23f2f 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Optional +from typing import Iterable, Optional, Union import isceobj import numpy as np @@ -128,6 +128,29 @@ def write(self, out_path: Path): out_path.write_text(self.__str__()) +def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: Union[str, Iterable]) -> None: + """Checks that the reference granule(s) are older than the secondary granule(s). + This is a conventention which ensures that positive interferogram values represent motion away from the satellite. + + Args: + reference: Reference granule(s) + secondary: Secondary granule(s) + """ + if isinstance(reference, str): + reference = [reference] + + if isinstance(secondary, str): + secondary = [secondary] + + ref_dates = list(set([g[14:29] for g in reference])) + sec_dates = list(set([g[14:29] for g in secondary])) + if len(ref_dates) > 1 or len(sec_dates) > 1: + raise ValueError('Reference granule(s) must be from one date and secondary granule(s) must be from one date.') + + if not ref_dates[0] < sec_dates[0]: + raise ValueError('Reference granule(s) must be older than secondary granule(s).') + + def utm_from_lon_lat(lon: float, lat: float) -> int: """Get the UTM zone EPSG code from a longitude and latitude. See https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system @@ -203,7 +226,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i:: image_obj.bands] + new_array[i, :, :] = array[i :: image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -368,7 +391,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i:: image_obj.bands] = array[i, :, :] + new_array[i :: image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') diff --git a/tests/test_utils.py b/tests/test_utils.py index 09e6b8de..e542022f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,17 +4,38 @@ from pathlib import Path from unittest.mock import patch +import isceobj # noqa import numpy as np +import pytest from osgeo import gdal import hyp3_isce2.utils as utils -import isceobj # noqa - gdal.UseExceptions() +def test_check_older_granule_is_reference(): + set1 = ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST'] + utils.check_older_granule_is_reference(set1[0], set1[1]) + + set2 = [ + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], + ] + utils.check_older_granule_is_reference(set2[0], set2[1]) + + set3 = [ + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], + ] + with pytest.raises(ValueError, match='Reference granule\(s\) must be from one date*'): + utils.check_older_granule_is_reference(set3[0], set3[1]) + + with pytest.raises(ValueError, match='Reference granule\(s\) must be older*'): + utils.check_older_granule_is_reference(set1[1], set1[0]) + + def test_utm_from_lon_lat(): assert utils.utm_from_lon_lat(0, 0) == 32631 assert utils.utm_from_lon_lat(-179, -1) == 32701 From 2fa88ea3b64a8bae6490abf41984ebf625679e6a Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 19:38:04 +0000 Subject: [PATCH 53/81] update coverage image --- images/coverage.svg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 012a8497..ffd257bd 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -15,7 +15,7 @@ coverage coverage - 70% - 70% + 71% + 71% From 833041d4fadf01b708ac541654046a350831959a Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 20 Aug 2024 14:54:43 -0500 Subject: [PATCH 54/81] fix flake8 --- src/hyp3_isce2/utils.py | 8 ++++---- tests/test_utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index ffe23f2f..a8ebb94c 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -145,10 +145,10 @@ def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: ref_dates = list(set([g[14:29] for g in reference])) sec_dates = list(set([g[14:29] for g in secondary])) if len(ref_dates) > 1 or len(sec_dates) > 1: - raise ValueError('Reference granule(s) must be from one date and secondary granule(s) must be from one date.') + raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') if not ref_dates[0] < sec_dates[0]: - raise ValueError('Reference granule(s) must be older than secondary granule(s).') + raise ValueError('Reference granules must be older than secondary granules.') def utm_from_lon_lat(lon: float, lat: float) -> int: @@ -226,7 +226,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i :: image_obj.bands] + new_array[i, :, :] = array[i::image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -391,7 +391,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i :: image_obj.bands] = array[i, :, :] + new_array[i::image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') diff --git a/tests/test_utils.py b/tests/test_utils.py index e542022f..4b50687e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -29,10 +29,10 @@ def test_check_older_granule_is_reference(): ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], ] - with pytest.raises(ValueError, match='Reference granule\(s\) must be from one date*'): + with pytest.raises(ValueError, match='Reference granules must be from one date*'): utils.check_older_granule_is_reference(set3[0], set3[1]) - with pytest.raises(ValueError, match='Reference granule\(s\) must be older*'): + with pytest.raises(ValueError, match='Reference granules must be older*'): utils.check_older_granule_is_reference(set1[1], set1[0]) From 4e252b2a6cb2256942628c3019f2374b54495b4c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 21 Aug 2024 10:36:56 -0800 Subject: [PATCH 55/81] delete references to separate `insar_tops_multi_burst` workflow --- README.md | 3 +-- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index dab03143..8f629431 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python - `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow - `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow - `insar_tops_burst`: A workflow for creating burst-based Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow -- `insar_tops_multi_burst`: A workflow for creating multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow --- To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow: @@ -27,7 +26,7 @@ python -m hyp3_isce2 ++process insar_tops_burst \ and, for multiple burst pairs: ``` -python -m hyp3_isce2 ++process insar_tops_multi_burst \ +python -m hyp3_isce2 ++process insar_tops_burst \ --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST \ --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST \ --looks 20x4 \ diff --git a/pyproject.toml b/pyproject.toml index 7c75a677..1a573a03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ Documentation = "https://hyp3-docs.asf.alaska.edu" [project.scripts] insar_tops_burst = "hyp3_isce2.insar_tops_burst:main" -insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main" insar_tops = "hyp3_isce2.insar_tops:main" insar_stripmap = "hyp3_isce2.insar_stripmap:main" merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main" From 2490d7ff8f6d55cef5a2f7baf89beb6b9acb705f Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 21 Aug 2024 10:42:28 -0800 Subject: [PATCH 56/81] delete `oldest_granule_first` --- src/hyp3_isce2/utils.py | 6 ------ tests/test_utils.py | 7 ------- 2 files changed, 13 deletions(-) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index a8ebb94c..cfed3eb7 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -202,12 +202,6 @@ def make_browse_image(input_tif: str, output_png: str) -> None: ) -def oldest_granule_first(g1, g2): - if g1[14:29] <= g2[14:29]: - return g1, g2 - return g2, g1 - - def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: """Read an ISCE2 image file and return the image object and array. diff --git a/tests/test_utils.py b/tests/test_utils.py index 4b50687e..96db4e43 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -75,13 +75,6 @@ def test_gdal_config_manager(): assert gdal.GetConfigOption('OPTION4') == 'VALUE4' -def test_oldest_granule_first(): - oldest = 'S1_249434_IW1_20230511T170732_VV_07DE-BURST' - latest = 'S1_249434_IW1_20230523T170733_VV_8850-BURST' - assert utils.oldest_granule_first(oldest, latest) == (oldest, latest) - assert utils.oldest_granule_first(latest, oldest) == (oldest, latest) - - def test_make_browse_image(): input_tif = 'tests/data/test_geotiff.tif' output_png = 'tests/data/test_browse_image2.png' From 758112de83aa52d44ca9723a60d0b3f5f708e6c5 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 21 Aug 2024 11:09:23 -0800 Subject: [PATCH 57/81] typo and style changes --- src/hyp3_isce2/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index cfed3eb7..8b685bd7 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -130,7 +130,7 @@ def write(self, out_path: Path): def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: Union[str, Iterable]) -> None: """Checks that the reference granule(s) are older than the secondary granule(s). - This is a conventention which ensures that positive interferogram values represent motion away from the satellite. + This is a convention which ensures that positive interferogram values represent motion away from the satellite. Args: reference: Reference granule(s) @@ -142,12 +142,12 @@ def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: if isinstance(secondary, str): secondary = [secondary] - ref_dates = list(set([g[14:29] for g in reference])) - sec_dates = list(set([g[14:29] for g in secondary])) + ref_dates = list(set([g.split('_')[3] for g in reference])) + sec_dates = list(set([g.split('_')[3] for g in secondary])) if len(ref_dates) > 1 or len(sec_dates) > 1: raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') - if not ref_dates[0] < sec_dates[0]: + if ref_dates[0] >= sec_dates[0]: raise ValueError('Reference granules must be older than secondary granules.') From 0b916159ee3a14a650ef703c0dddcbd4ca23c07a Mon Sep 17 00:00:00 2001 From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:08:48 -0500 Subject: [PATCH 58/81] Apply suggestions from code review Co-authored-by: Jake Herrmann --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9accc28a..6521ccad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.0.0] ### Added - The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. -- Function to check if the reference granules are older than the secondary granules. +- The `insar_tops_burst` workflow now exits with an error message if the reference granules are not older than the secondary granules, rather than silently re-ordering the granules. ### Changed -- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` arguments instead of one `--granules` argument +- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists instead of a single list of granules. ## [2.0.0] ### Changed From 042ec683ca321dc0f4f75130faa7349a09cd633b Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 21 Aug 2024 15:10:45 -0500 Subject: [PATCH 59/81] fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6521ccad..2d931c02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.0.0] ### Added - The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. -- The `insar_tops_burst` workflow now exits with an error message if the reference granules are not older than the secondary granules, rather than silently re-ordering the granules. ### Changed - The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists instead of a single list of granules. +- The `insar_tops_burst` workflow now exits with an error message if the reference granules are not older than the secondary granules, rather than silently re-ordering the granules. ## [2.0.0] ### Changed From 3d0c635581b204a3f49c955d88ea105eff84ad78 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Wed, 21 Aug 2024 15:23:12 -0500 Subject: [PATCH 60/81] revise based on code review --- src/hyp3_isce2/insar_tops_burst.py | 16 +++++++-------- tests/test_utils.py | 31 +++++++++++++++--------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index f3532095..74d5322d 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -212,11 +212,11 @@ def insar_tops_multi_burst( bucket: Optional[str] = None, bucket_prefix: str = '', ): - ref_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + ref_unique_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_unique_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - if ref_ids != sec_ids: - raise Exception('The reference burst(s) and secondary burst(s) do not match') + if ref_unique_ids != sec_unique_ids: + raise ValueError('The reference burst(s) and secondary burst(s) do not match') check_older_granule_is_reference(reference, secondary) @@ -248,16 +248,16 @@ def main(): """HyP3 entrypoint for the burst TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') + parser.add_argument('--reference', type=str.split, nargs='+', help='List of reference scenes"') + parser.add_argument('--secondary', type=str.split, nargs='+', help='List of secondary scenes"') parser.add_argument( '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth' ) parser.add_argument( '--apply-water-mask', type=string_is_true, default=False, help='Apply a water body mask before unwrapping.' ) - parser.add_argument('--reference', type=str.split, nargs='+', help='List of reference scenes"') - parser.add_argument('--secondary', type=str.split, nargs='+', help='List of secondary scenes"') + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') args = parser.parse_args() diff --git a/tests/test_utils.py b/tests/test_utils.py index 96db4e43..3b7431a8 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -16,24 +16,25 @@ def test_check_older_granule_is_reference(): - set1 = ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST'] - utils.check_older_granule_is_reference(set1[0], set1[1]) - - set2 = [ - ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], - ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], - ] - utils.check_older_granule_is_reference(set2[0], set2[1]) - - set3 = [ - ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], - ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], - ] + utils.check_older_granule_is_reference( + 'S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST' + ) + + utils.check_older_granule_is_reference( + reference=['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], + ) + with pytest.raises(ValueError, match='Reference granules must be from one date*'): - utils.check_older_granule_is_reference(set3[0], set3[1]) + utils.check_older_granule_is_reference( + reference=['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], + ) with pytest.raises(ValueError, match='Reference granules must be older*'): - utils.check_older_granule_is_reference(set1[1], set1[0]) + utils.check_older_granule_is_reference( + 'S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST' + ) def test_utm_from_lon_lat(): From 7d468d485c511e67795c1d2b08be914b86bed289 Mon Sep 17 00:00:00 2001 From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> Date: Thu, 22 Aug 2024 07:10:36 -0500 Subject: [PATCH 61/81] Apply suggestions from code review Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_tops_burst.py | 6 +++--- src/hyp3_isce2/utils.py | 4 ++-- tests/test_utils.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 74d5322d..a5b78b72 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -212,10 +212,10 @@ def insar_tops_multi_burst( bucket: Optional[str] = None, bucket_prefix: str = '', ): - ref_unique_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_unique_ids = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] + ref_number_swath_pol = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] + sec_number_swath_pol = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - if ref_unique_ids != sec_unique_ids: + if ref_number_swath_pol != sec_number_swath_pol: raise ValueError('The reference burst(s) and secondary burst(s) do not match') check_older_granule_is_reference(reference, secondary) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index 8b685bd7..88ece07d 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -142,8 +142,8 @@ def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: if isinstance(secondary, str): secondary = [secondary] - ref_dates = list(set([g.split('_')[3] for g in reference])) - sec_dates = list(set([g.split('_')[3] for g in secondary])) + ref_dates = list(set(g.split('_')[3] for g in reference)) + sec_dates = list(set(g.split('_')[3] for g in secondary)) if len(ref_dates) > 1 or len(sec_dates) > 1: raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') diff --git a/tests/test_utils.py b/tests/test_utils.py index 3b7431a8..169c5eaa 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -25,15 +25,15 @@ def test_check_older_granule_is_reference(): secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], ) - with pytest.raises(ValueError, match='Reference granules must be from one date*'): + with pytest.raises(ValueError, match=r'.* granules must be from one date .*'): utils.check_older_granule_is_reference( reference=['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], ) - with pytest.raises(ValueError, match='Reference granules must be older*'): + with pytest.raises(ValueError, match=r'Reference granules must be older .*'): utils.check_older_granule_is_reference( - 'S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST' + 'S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VV_0000-BURST' ) From 8afb3d06d7b648840b0a5362ace9c961ff02509a Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 22 Aug 2024 07:21:46 -0500 Subject: [PATCH 62/81] combine validation utilities --- src/hyp3_isce2/burst.py | 62 +++++++++++++++++++----------- src/hyp3_isce2/insar_tops_burst.py | 15 +------- src/hyp3_isce2/utils.py | 29 ++------------ 3 files changed, 43 insertions(+), 63 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 03c05020..5f57199a 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path -from typing import Iterator, List, Optional, Tuple, Union +from typing import Iterable, Iterator, List, Optional, Tuple, Union import asf_search import numpy as np @@ -360,35 +360,51 @@ def get_burst_params(scene_name: str) -> BurstParams: ) -def validate_bursts(reference_scene: str, secondary_scene: str) -> None: - """Check whether the reference and secondary bursts are valid. +def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, Iterable[str]]) -> None: + """Check whether the reference and secondary bursts are a valid sets. Args: - reference_scene: The reference burst name. - secondary_scene: The secondary burst name. - - Returns: - None + reference: Reference granule(s) + secondary: Secondary granule(s) """ - ref_split = reference_scene.split('_') - sec_split = secondary_scene.split('_') + if isinstance(reference, str): + reference = [reference] + if isinstance(secondary, str): + secondary = [secondary] + + # Check number of bursts + if len(reference) < 1 or len(secondary) < 1: + ValueError('Must include at least 1 reference and 1 secondary burst') + if len(reference) != len(secondary): + ValueError('Must have the same number of reference and secondary bursts') + + # Check matching set of bursts + ref_num_swath_pol = sorted([g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference]) + sec_num_swath_pol = sorted([g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary]) + if ref_num_swath_pol != sec_num_swath_pol: + msg = 'The reference and secondary burst ID sets do not match.\n' + msg += f' Reference IDs: {ref_num_swath_pol}\n' + msg += f' Secondary IDs: {sec_num_swath_pol}' + raise ValueError(msg) + + # Check that only one valid polarization is present + pols = list(set([g.split('_')[4] for g in reference + secondary])) - ref_burst_id = ref_split[1] - sec_burst_id = sec_split[1] + if len(pols) > 1: + raise ValueError(f'All bursts must have a single polarization. Polarizations present: {" ".join(pols)}') - ref_polarization = ref_split[4] - sec_polarization = sec_split[4] + if pols[0] not in ['VV', 'HH']: + raise ValueError(f'{pols[0]} polarization is not currently supported, only VV and HH.') - if ref_burst_id != sec_burst_id: - raise ValueError(f'The reference and secondary burst IDs are not the same: {ref_burst_id} and {sec_burst_id}.') + # Check that the reference bursts are older + ref_dates = list(set([g.split('_')[3] for g in reference])) + sec_dates = list(set([g.split('_')[3] for g in secondary])) - if ref_polarization != sec_polarization: - raise ValueError( - f'The reference and secondary polarizations are not the same: {ref_polarization} and {sec_polarization}.' - ) + if len(ref_dates) > 1 or len(sec_dates) > 1: + raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') - if ref_polarization != 'VV' and ref_polarization != 'HH': - raise ValueError(f'{ref_polarization} polarization is not currently supported, only VV and HH.') + if ref_dates[0] >= sec_dates[0]: + raise ValueError('Reference granules must be older than secondary granules.') def load_burst_position(swath_xml_path: str, burst_number: int) -> BurstPosition: @@ -532,7 +548,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line: last_line, position.first_valid_sample: last_sample] = identity_value + mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value else: mask[:, :] = identity_value diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index a5b78b72..cb2e0f5f 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -28,7 +28,6 @@ from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( - check_older_granule_is_reference, image_math, isce2_copy, make_browse_image, @@ -145,7 +144,6 @@ def insar_tops_single_burst( bucket: Optional[str] = None, bucket_prefix: str = '', ): - check_older_granule_is_reference(reference, secondary) validate_bursts(reference, secondary) swath_number = int(reference[12]) range_looks, azimuth_looks = [int(value) for value in looks.split('x')] @@ -212,14 +210,7 @@ def insar_tops_multi_burst( bucket: Optional[str] = None, bucket_prefix: str = '', ): - ref_number_swath_pol = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference] - sec_number_swath_pol = [g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary] - - if ref_number_swath_pol != sec_number_swath_pol: - raise ValueError('The reference burst(s) and secondary burst(s) do not match') - - check_older_granule_is_reference(reference, secondary) - + validate_bursts(reference, secondary) reference_safe_path = burst2safe(reference) reference_safe = reference_safe_path.name.split('.')[0] secondary_safe_path = burst2safe(secondary) @@ -263,10 +254,6 @@ def main(): references = [item for sublist in args.reference for item in sublist] secondaries = [item for sublist in args.secondary for item in sublist] - if len(references) < 1 or len(secondaries) < 1: - parser.error('Must include at least 1 reference and 1 secondary') - if len(references) != len(secondaries): - parser.error('Must have the same number of references and secondaries') configure_root_logger() log.debug(' '.join(sys.argv)) diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index 88ece07d..d0c7c13b 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Iterable, Optional, Union +from typing import Optional import isceobj import numpy as np @@ -128,29 +128,6 @@ def write(self, out_path: Path): out_path.write_text(self.__str__()) -def check_older_granule_is_reference(reference: Union[str, Iterable], secondary: Union[str, Iterable]) -> None: - """Checks that the reference granule(s) are older than the secondary granule(s). - This is a convention which ensures that positive interferogram values represent motion away from the satellite. - - Args: - reference: Reference granule(s) - secondary: Secondary granule(s) - """ - if isinstance(reference, str): - reference = [reference] - - if isinstance(secondary, str): - secondary = [secondary] - - ref_dates = list(set(g.split('_')[3] for g in reference)) - sec_dates = list(set(g.split('_')[3] for g in secondary)) - if len(ref_dates) > 1 or len(sec_dates) > 1: - raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') - - if ref_dates[0] >= sec_dates[0]: - raise ValueError('Reference granules must be older than secondary granules.') - - def utm_from_lon_lat(lon: float, lat: float) -> int: """Get the UTM zone EPSG code from a longitude and latitude. See https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system @@ -220,7 +197,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i::image_obj.bands] + new_array[i, :, :] = array[i :: image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -385,7 +362,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i::image_obj.bands] = array[i, :, :] + new_array[i :: image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') From fd2e73e9da3d34e77916c36f159b1c3c6541991c Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 22 Aug 2024 07:45:59 -0500 Subject: [PATCH 63/81] update test for validate bursts --- src/hyp3_isce2/burst.py | 4 ++-- tests/test_burst.py | 39 ++++++++++++++++++++++++++++++++------- tests/test_utils.py | 23 ----------------------- 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 5f57199a..1a9507e1 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -374,9 +374,9 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, # Check number of bursts if len(reference) < 1 or len(secondary) < 1: - ValueError('Must include at least 1 reference and 1 secondary burst') + raise ValueError('Must include at least 1 reference and 1 secondary burst') if len(reference) != len(secondary): - ValueError('Must have the same number of reference and secondary bursts') + raise ValueError('Must have the same number of reference and secondary bursts') # Check matching set of bursts ref_num_swath_pol = sorted([g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference]) diff --git a/tests/test_burst.py b/tests/test_burst.py index 318d6c31..f996f929 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -173,18 +173,43 @@ def test_get_burst_params_multiple_results(): def test_validate_bursts(): - burst.validate_bursts('S1_030349_IW1_20230808T171601_VV_4A37-BURST', 'S1_030349_IW1_20230820T171602_VV_5AC3-BURST') - with pytest.raises(ValueError, match=r'.*polarizations are not the same.*'): + burst.validate_bursts('S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST') + burst.validate_bursts( + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], + ) + + with pytest.raises(ValueError, match=r'Must include at least 1.*'): + burst.validate_bursts(['a'], []) + + with pytest.raises(ValueError, match=r'Must have the same number.*'): + burst.validate_bursts(['a', 'b'], ['c']) + + with pytest.raises(ValueError, match=r'.*burst ID sets do not match.*'): + burst.validate_bursts( + ['S1_000000_IW1_20200101T000000_VV_0000-BURST'], ['S1_000000_IW2_20200201T000000_VV_0000-BURST'] + ) + + with pytest.raises(ValueError, match=r'.*burst ID sets do not match.*'): + burst.validate_bursts( + ['S1_000000_IW1_20200101T000000_VV_0000-BURST'], ['S1_000000_IW2_20200201T000000_VV_0000-BURST'] + ) + + with pytest.raises(ValueError, match=r'.*must have a single polarization.*'): burst.validate_bursts( - 'S1_215032_IW2_20230802T144608_VV_7EE2-BURST', 'S1_215032_IW2_20230721T144607_HH_B3FA-BURST' + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VH_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VH_0000-BURST'], ) - with pytest.raises(ValueError, match=r'.*burst IDs are not the same.*'): + + with pytest.raises(ValueError, match=r'.*polarization is not currently supported.*'): burst.validate_bursts( - 'S1_030349_IW1_20230808T171601_VV_4A37-BURST', 'S1_030348_IW1_20230820T171602_VV_5AC3-BURST' + ['S1_000000_IW1_20200101T000000_VH_0000-BURST', 'S1_000000_IW1_20200101T000000_VH_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VH_0000-BURST', 'S1_000000_IW1_20200201T000000_VH_0000-BURST'], ) - with pytest.raises(ValueError, match=r'.*only VV and HH.*'): + + with pytest.raises(ValueError, match=r'Reference granules must be older.*'): burst.validate_bursts( - 'S1_030349_IW1_20230808T171601_VH_4A37-BURST', 'S1_030349_IW1_20230820T171602_VH_5AC3-BURST' + 'S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VV_0000-BURST' ) diff --git a/tests/test_utils.py b/tests/test_utils.py index 169c5eaa..2322ba2b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,7 +6,6 @@ import isceobj # noqa import numpy as np -import pytest from osgeo import gdal import hyp3_isce2.utils as utils @@ -15,28 +14,6 @@ gdal.UseExceptions() -def test_check_older_granule_is_reference(): - utils.check_older_granule_is_reference( - 'S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST' - ) - - utils.check_older_granule_is_reference( - reference=['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], - secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], - ) - - with pytest.raises(ValueError, match=r'.* granules must be from one date .*'): - utils.check_older_granule_is_reference( - reference=['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], - secondary=['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], - ) - - with pytest.raises(ValueError, match=r'Reference granules must be older .*'): - utils.check_older_granule_is_reference( - 'S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VV_0000-BURST' - ) - - def test_utm_from_lon_lat(): assert utils.utm_from_lon_lat(0, 0) == 32631 assert utils.utm_from_lon_lat(-179, -1) == 32701 From 89dcfc21a34e2ffbd25d2affe16c8b6701cc4bb8 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 22 Aug 2024 07:54:29 -0500 Subject: [PATCH 64/81] fix flake8 --- src/hyp3_isce2/burst.py | 2 +- src/hyp3_isce2/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 1a9507e1..28d7308c 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -548,7 +548,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value + mask[position.first_valid_line:last_line, position.first_valid_sample:last_sample] = identity_value else: mask[:, :] = identity_value diff --git a/src/hyp3_isce2/utils.py b/src/hyp3_isce2/utils.py index d0c7c13b..df3ab55d 100644 --- a/src/hyp3_isce2/utils.py +++ b/src/hyp3_isce2/utils.py @@ -197,7 +197,7 @@ def load_isce2_image(in_path) -> tuple[isceobj.Image, np.ndarray]: shape = (image_obj.bands, image_obj.length, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i, :, :] = array[i :: image_obj.bands] + new_array[i, :, :] = array[i::image_obj.bands] array = new_array.copy() else: raise NotImplementedError('Non-BIL reading is not implemented') @@ -362,7 +362,7 @@ def write_isce2_image_from_obj(image_obj, array): shape = (image_obj.length * image_obj.bands, image_obj.width) new_array = np.zeros(shape, dtype=image_obj.toNumpyDataType()) for i in range(image_obj.bands): - new_array[i :: image_obj.bands] = array[i, :, :] + new_array[i::image_obj.bands] = array[i, :, :] array = new_array.copy() else: raise NotImplementedError('Non-BIL writing is not implemented') From 34d55e77d2bf0b705dbdff023750aa61de58dc61 Mon Sep 17 00:00:00 2001 From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:02:23 -0500 Subject: [PATCH 65/81] Apply suggestions from code review Co-authored-by: Jake Herrmann --- src/hyp3_isce2/burst.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 28d7308c..074da103 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -361,7 +361,7 @@ def get_burst_params(scene_name: str) -> BurstParams: def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, Iterable[str]]) -> None: - """Check whether the reference and secondary bursts are a valid sets. + """Check whether the reference and secondary bursts are valid. Args: reference: Reference granule(s) @@ -372,23 +372,20 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, if isinstance(secondary, str): secondary = [secondary] - # Check number of bursts if len(reference) < 1 or len(secondary) < 1: raise ValueError('Must include at least 1 reference and 1 secondary burst') if len(reference) != len(secondary): raise ValueError('Must have the same number of reference and secondary bursts') - # Check matching set of bursts - ref_num_swath_pol = sorted([g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference]) - sec_num_swath_pol = sorted([g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary]) + ref_num_swath_pol = sorted(g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference) + sec_num_swath_pol = sorted(g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary) if ref_num_swath_pol != sec_num_swath_pol: msg = 'The reference and secondary burst ID sets do not match.\n' msg += f' Reference IDs: {ref_num_swath_pol}\n' msg += f' Secondary IDs: {sec_num_swath_pol}' raise ValueError(msg) - # Check that only one valid polarization is present - pols = list(set([g.split('_')[4] for g in reference + secondary])) + pols = list(set(g.split('_')[4] for g in reference + secondary)) if len(pols) > 1: raise ValueError(f'All bursts must have a single polarization. Polarizations present: {" ".join(pols)}') @@ -396,9 +393,8 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, if pols[0] not in ['VV', 'HH']: raise ValueError(f'{pols[0]} polarization is not currently supported, only VV and HH.') - # Check that the reference bursts are older - ref_dates = list(set([g.split('_')[3] for g in reference])) - sec_dates = list(set([g.split('_')[3] for g in secondary])) + ref_dates = list(set(g.split('_')[3] for g in reference)) + sec_dates = list(set(g.split('_')[3] for g in secondary)) if len(ref_dates) > 1 or len(sec_dates) > 1: raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') From f4d1dad9f0a027d6137f2a340229057c1bb8c7cc Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 22 Aug 2024 16:07:10 -0500 Subject: [PATCH 66/81] add test case --- src/hyp3_isce2/burst.py | 2 +- tests/test_burst.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 074da103..39147b31 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -397,7 +397,7 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, sec_dates = list(set(g.split('_')[3] for g in secondary)) if len(ref_dates) > 1 or len(sec_dates) > 1: - raise ValueError('Reference granules must be from one date and secondary granules must be from one date.') + raise ValueError('Reference granules must be from one date and secondary granules must be another.') if ref_dates[0] >= sec_dates[0]: raise ValueError('Reference granules must be older than secondary granules.') diff --git a/tests/test_burst.py b/tests/test_burst.py index f996f929..2e8aefba 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -207,6 +207,12 @@ def test_validate_bursts(): ['S1_000000_IW1_20200201T000000_VH_0000-BURST', 'S1_000000_IW1_20200201T000000_VH_0000-BURST'], ) + with pytest.raises(ValueError, match=r'.*must be from one date.*'): + burst.validate_bursts( + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200202T000000_VV_0000-BURST'], + ) + with pytest.raises(ValueError, match=r'Reference granules must be older.*'): burst.validate_bursts( 'S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VV_0000-BURST' From bc35cfc51f68c2ac6a0a2b43a47c02ef7f5a4bb4 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Thu, 22 Aug 2024 16:43:09 -0500 Subject: [PATCH 67/81] readd granules argument --- src/hyp3_isce2/insar_tops_burst.py | 46 +++++++++++++++++++++--------- tests/test_burst.py | 5 ---- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index cb2e0f5f..18c031cd 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -3,6 +3,7 @@ import argparse import logging import sys +import warnings from pathlib import Path from shutil import copyfile, make_archive from typing import Iterable, Optional @@ -27,12 +28,7 @@ from hyp3_isce2.insar_tops import insar_tops_packaged from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal -from hyp3_isce2.utils import ( - image_math, - isce2_copy, - make_browse_image, - resample_to_radar_io, -) +from hyp3_isce2.utils import image_math, isce2_copy, make_browse_image, resample_to_radar_io from hyp3_isce2.water_mask import create_water_mask @@ -235,10 +231,16 @@ def insar_tops_multi_burst( log.info('ISCE2 TopsApp run completed successfully') +def oldest_granule_first(g1, g2): + if g1[14:29] <= g2[14:29]: + return [g1], [g2] + return [g2], [g1] + + def main(): """HyP3 entrypoint for the burst TOPS workflow""" parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - + parser.add_argument('granules', type=str.split, nargs='*', help='Reference and secondary scene names') parser.add_argument('--reference', type=str.split, nargs='+', help='List of reference scenes"') parser.add_argument('--secondary', type=str.split, nargs='+', help='List of secondary scenes"') parser.add_argument( @@ -252,16 +254,32 @@ def main(): args = parser.parse_args() - references = [item for sublist in args.reference for item in sublist] - secondaries = [item for sublist in args.secondary for item in sublist] + has_granules = args.granules is not None and len(args.granules) > 0 + has_ref_sec = args.reference is not None and args.secondary is not None + if has_granules and has_ref_sec: + parser.error('Either the positional granules argument, or --reference and --secondary must be specified.') + elif not has_granules and not has_ref_sec: + parser.error('Only the positional granules argument, or --reference and --secondary must be specified.') + elif has_granules: + warnings.warn( + 'The positional argument for granules is deprecated. Please use --reference and --secondary.', + DeprecationWarning, + ) + granules = [item for sublist in args.granules for item in sublist] + if len(granules) != 2: + parser.error('No more than two granules may be provided.') + reference, secondary = oldest_granule_first(granules[0], granules[1]) + else: + reference = [item for sublist in args.reference for item in sublist] + secondary = [item for sublist in args.secondary for item in sublist] configure_root_logger() log.debug(' '.join(sys.argv)) - if len(references) == 1: + if len(reference) == 1: insar_tops_single_burst( - reference=references[0], - secondary=secondaries[0], + reference=reference[0], + secondary=secondary[0], looks=args.looks, apply_water_mask=args.apply_water_mask, bucket=args.bucket, @@ -269,8 +287,8 @@ def main(): ) else: insar_tops_multi_burst( - reference=references, - secondary=secondaries, + reference=reference, + secondary=secondary, looks=args.looks, apply_water_mask=args.apply_water_mask, bucket=args.bucket, diff --git a/tests/test_burst.py b/tests/test_burst.py index 2e8aefba..1a513980 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -190,11 +190,6 @@ def test_validate_bursts(): ['S1_000000_IW1_20200101T000000_VV_0000-BURST'], ['S1_000000_IW2_20200201T000000_VV_0000-BURST'] ) - with pytest.raises(ValueError, match=r'.*burst ID sets do not match.*'): - burst.validate_bursts( - ['S1_000000_IW1_20200101T000000_VV_0000-BURST'], ['S1_000000_IW2_20200201T000000_VV_0000-BURST'] - ) - with pytest.raises(ValueError, match=r'.*must have a single polarization.*'): burst.validate_bursts( ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200101T000000_VH_0000-BURST'], From 4ed93022d1c0e2d360daf7162e644d076c35cb6c Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 22 Aug 2024 14:05:21 -0800 Subject: [PATCH 68/81] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d931c02..c2ad5aaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.0.0] +## [2.1.0] ### Added - The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. From d395cc820a5f76cf9e30f072d3fbaf302b09e82a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 22 Aug 2024 14:06:32 -0800 Subject: [PATCH 69/81] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ad5aaa..67c8d332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC. ### Changed -- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists instead of a single list of granules. +- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists. The positional `granules` argument is now optional and deprecated. - The `insar_tops_burst` workflow now exits with an error message if the reference granules are not older than the secondary granules, rather than silently re-ordering the granules. ## [2.0.0] From 37add8024704598aac2c0c820e85bc2fe2eb0b1b Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 22 Aug 2024 14:08:38 -0800 Subject: [PATCH 70/81] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67c8d332..13c58c27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists. The positional `granules` argument is now optional and deprecated. -- The `insar_tops_burst` workflow now exits with an error message if the reference granules are not older than the secondary granules, rather than silently re-ordering the granules. ## [2.0.0] ### Changed From 46697e2c46d0e1b796761b880c297f16a3270606 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 23 Aug 2024 07:50:20 -0500 Subject: [PATCH 71/81] fix date validation issue --- src/hyp3_isce2/burst.py | 6 +++--- tests/test_burst.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 39147b31..1776b8f5 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -393,8 +393,8 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, if pols[0] not in ['VV', 'HH']: raise ValueError(f'{pols[0]} polarization is not currently supported, only VV and HH.') - ref_dates = list(set(g.split('_')[3] for g in reference)) - sec_dates = list(set(g.split('_')[3] for g in secondary)) + ref_dates = list(set(g.split('_')[3][:7] for g in reference)) + sec_dates = list(set(g.split('_')[3][:7] for g in secondary)) if len(ref_dates) > 1 or len(sec_dates) > 1: raise ValueError('Reference granules must be from one date and secondary granules must be another.') @@ -544,7 +544,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line:last_line, position.first_valid_sample:last_sample] = identity_value + mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value else: mask[:, :] = identity_value diff --git a/tests/test_burst.py b/tests/test_burst.py index 1a513980..2fd04ff6 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -175,8 +175,8 @@ def test_get_burst_params_multiple_results(): def test_validate_bursts(): burst.validate_bursts('S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000000_IW1_20200201T000000_VV_0000-BURST') burst.validate_bursts( - ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000000_VV_0000-BURST'], - ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000000_VV_0000-BURST'], + ['S1_000000_IW1_20200101T000000_VV_0000-BURST', 'S1_000001_IW1_20200101T000001_VV_0000-BURST'], + ['S1_000000_IW1_20200201T000000_VV_0000-BURST', 'S1_000001_IW1_20200201T000001_VV_0000-BURST'], ) with pytest.raises(ValueError, match=r'Must include at least 1.*'): From 590605c3bb2f11446490d31d1eca103c36f64969 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 23 Aug 2024 07:56:17 -0500 Subject: [PATCH 72/81] include full year month day --- src/hyp3_isce2/burst.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 1776b8f5..1d5ff537 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -393,8 +393,8 @@ def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, if pols[0] not in ['VV', 'HH']: raise ValueError(f'{pols[0]} polarization is not currently supported, only VV and HH.') - ref_dates = list(set(g.split('_')[3][:7] for g in reference)) - sec_dates = list(set(g.split('_')[3][:7] for g in secondary)) + ref_dates = list(set(g.split('_')[3][:8] for g in reference)) + sec_dates = list(set(g.split('_')[3][:8] for g in secondary)) if len(ref_dates) > 1 or len(sec_dates) > 1: raise ValueError('Reference granules must be from one date and secondary granules must be another.') From 89fe40e787708c52ac37b60b7956ba1faf24a483 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Fri, 23 Aug 2024 07:56:52 -0500 Subject: [PATCH 73/81] fix flake8 --- src/hyp3_isce2/burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 1d5ff537..adec696f 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -544,7 +544,7 @@ def safely_multilook( if subset_to_valid: last_line = position.first_valid_line + position.n_valid_lines last_sample = position.first_valid_sample + position.n_valid_samples - mask[position.first_valid_line : last_line, position.first_valid_sample : last_sample] = identity_value + mask[position.first_valid_line:last_line, position.first_valid_sample:last_sample] = identity_value else: mask[:, :] = identity_value From d1bdd9ad1857f2188ccbc051b00233e863755b52 Mon Sep 17 00:00:00 2001 From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> Date: Fri, 23 Aug 2024 12:39:40 -0500 Subject: [PATCH 74/81] Update src/hyp3_isce2/insar_tops_burst.py Co-authored-by: Andrew Player --- src/hyp3_isce2/insar_tops_burst.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 18c031cd..fd36b1c8 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -257,9 +257,9 @@ def main(): has_granules = args.granules is not None and len(args.granules) > 0 has_ref_sec = args.reference is not None and args.secondary is not None if has_granules and has_ref_sec: - parser.error('Either the positional granules argument, or --reference and --secondary must be specified.') + parser.error('Provide either --reference and --secondary, or the positional granules argument, not both.') elif not has_granules and not has_ref_sec: - parser.error('Only the positional granules argument, or --reference and --secondary must be specified.') + parser.error('Either --reference and --secondary, or the positional granules argument, must be provided.') elif has_granules: warnings.warn( 'The positional argument for granules is deprecated. Please use --reference and --secondary.', From 5beca78449674db820e079916a693b87f2ba3aec Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 09:30:01 -0500 Subject: [PATCH 75/81] fix insar_tops workflow --- src/hyp3_isce2/insar_tops.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index dd988f42..ecdae45d 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -49,12 +49,9 @@ def insar_tops( aux_cal_dir = Path('aux_cal') dem_dir = Path('dem') - if download: - ref_dir = slc.get_granule(reference_scene) - sec_dir = slc.get_granule(secondary_scene) - else: - ref_dir = Path(reference_scene + '.SAFE') - sec_dir = Path(secondary_scene + '.SAFE') + ref_dir = slc.get_granule(reference_scene) + sec_dir = slc.get_granule(secondary_scene) + roi = slc.get_dem_bounds(ref_dir, sec_dir) log.info(f'DEM ROI: {roi}') @@ -133,8 +130,6 @@ def insar_tops_packaged( log.info('Begin ISCE2 TopsApp run') - do_download = os.path.exists(f'{reference}.SAFE') and os.path.exists(f'{secondary}.SAFE') - insar_tops( reference_scene=reference, secondary_scene=secondary, @@ -143,7 +138,6 @@ def insar_tops_packaged( azimuth_looks=azimuth_looks, range_looks=range_looks, apply_water_mask=apply_water_mask, - download=do_download ) log.info('ISCE2 TopsApp run completed successfully') From 381bd3bb8327678b1bf7f736542bbccab388a9fa Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 09:31:14 -0500 Subject: [PATCH 76/81] fix flake8 --- src/hyp3_isce2/insar_tops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index ecdae45d..68266e6c 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -2,7 +2,6 @@ import argparse import logging -import os import sys from pathlib import Path from shutil import copyfile, make_archive From 66cc546122babff609eb1e2e78acefd90471bf7d Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 10:39:44 -0500 Subject: [PATCH 77/81] fix pol in product name --- src/hyp3_isce2/insar_tops.py | 4 +++- src/hyp3_isce2/packaging.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 68266e6c..b0374b5a 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -141,7 +141,9 @@ def insar_tops_packaged( log.info('ISCE2 TopsApp run completed successfully') - product_name = packaging.get_product_name(reference, secondary, pixel_spacing=int(pixel_size)) + product_name = packaging.get_product_name( + reference, secondary, pixel_spacing=int(pixel_size), polarization=polarization, slc=True + ) product_dir = Path(product_name) product_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index b5e49565..43b06c25 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -47,14 +47,16 @@ def find_product(pattern: str) -> str: product = str(list(search)[0]) return product - -def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bool = True) -> str: +def get_product_name( + reference: str, secondary: str, pixel_spacing: int, polarization: Optional[str] = None, slc: bool = True +) -> str: """Get the name of the interferogram product. Args: reference: The reference burst name. secondary: The secondary burst name. pixel_spacing: The spacing of the pixels in the output image. + polarization: The polarization of the input data. Only required for SLCs. slc: Whether the input scenes are SLCs or bursts. Returns: @@ -69,7 +71,10 @@ def get_product_name(reference: str, secondary: str, pixel_spacing: int, slc: bo platform = reference_split[0] reference_date = reference_split[5][0:8] secondary_date = secondary_split[5][0:8] - polarization = os.path.basename(glob.glob(f'{reference}.SAFE/annotation/s1*')[0]).split('-')[3].upper() + if not polarization: + raise ValueError('Polarization is required for SLCs') + elif polarization not in ['VV', 'VH', 'HV', 'HH']: + raise ValueError('Polarization must be one of VV, VH, HV, or HH') ref_manifest_xml = etree.parse(f'{reference}.SAFE/manifest.safe', parser) metadata_path = './/metadataObject[@ID="measurementOrbitReference"]//xmlData//' relative_orbit_number_query = metadata_path + safe + 'relativeOrbitNumber' From 35604cdc511f9a4da235e8e2cb0d968e792ea8ae Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 10:44:56 -0500 Subject: [PATCH 78/81] fix flake8 --- src/hyp3_isce2/packaging.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/packaging.py b/src/hyp3_isce2/packaging.py index 43b06c25..09da4ec7 100644 --- a/src/hyp3_isce2/packaging.py +++ b/src/hyp3_isce2/packaging.py @@ -1,4 +1,3 @@ -import glob import os import subprocess from dataclasses import dataclass @@ -47,6 +46,7 @@ def find_product(pattern: str) -> str: product = str(list(search)[0]) return product + def get_product_name( reference: str, secondary: str, pixel_spacing: int, polarization: Optional[str] = None, slc: bool = True ) -> str: @@ -81,8 +81,13 @@ def get_product_name( orbit_number = ref_manifest_xml.find(relative_orbit_number_query).text.zfill(3) footprint = get_geometry_from_manifest(Path(f'{reference}.SAFE/manifest.safe')) lons, lats = footprint.exterior.coords.xy - def lat_string(lat): return ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') - def lon_string(lon): return ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') + + def lat_string(lat): + return ('N' if lat >= 0 else 'S') + f"{('%.1f' % np.abs(lat)).zfill(4)}".replace('.', '_') + + def lon_string(lon): + return ('E' if lon >= 0 else 'W') + f"{('%.1f' % np.abs(lon)).zfill(5)}".replace('.', '_') + lat_lims = [lat_string(lat) for lat in [np.min(lats), np.max(lats)]] lon_lims = [lon_string(lon) for lon in [np.min(lons), np.max(lons)]] name_parts = [platform, orbit_number, lon_lims[0], lat_lims[0], lon_lims[1], lat_lims[1]] From b327ef1e8006e6e0612f6ad22883c37a9899a759 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 13:38:37 -0500 Subject: [PATCH 79/81] remove unused parameter --- src/hyp3_isce2/insar_tops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index b0374b5a..fe4a3741 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -29,7 +29,6 @@ def insar_tops( azimuth_looks: int = 4, range_looks: int = 20, apply_water_mask: bool = False, - download: bool = True, ) -> Path: """Create a full-SLC interferogram @@ -40,6 +39,7 @@ def insar_tops( polarization: Polarization to use azimuth_looks: Number of azimuth looks range_looks: Number of range looks + apply_water_mask: Apply water mask to unwrapped phase Returns: Path to the output files @@ -118,7 +118,6 @@ def insar_tops_packaged( azimuth_looks: Number of azimuth looks range_looks: Number of range looks apply_water_mask: Apply water mask to unwrapped phase - download: Download the SLCs bucket: AWS S3 bucket to upload the final product to bucket_prefix: Bucket prefix to prefix to use when uploading the final product From 4c87af684f6ab3a12600edd8cef302296b267de0 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 15:52:09 -0500 Subject: [PATCH 80/81] improve naming --- src/hyp3_isce2/insar_tops.py | 10 +++++----- src/hyp3_isce2/insar_tops_burst.py | 12 ++++++------ src/hyp3_isce2/topsapp.py | 6 +++--- tests/test_topsapp.py | 14 +++++++------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index fe4a3741..98219248 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -63,7 +63,7 @@ def insar_tops( orbit_file = fetch_for_scene(granule, dir=orbit_dir) log.info(f'Got orbit file {orbit_file} from s1_orbits') - config = topsapp.TopsappBurstConfig( + config = topsapp.TopsappConfig( reference_safe=f'{reference_scene}.SAFE', secondary_safe=f'{secondary_scene}.SAFE', polarization=polarization, @@ -79,7 +79,7 @@ def insar_tops( config_path = config.write_template('topsApp.xml') if apply_water_mask: - topsapp.run_topsapp_burst(start='startup', end='filter', config_xml=config_path) + topsapp.run_topsapp(start='startup', end='filter', config_xml=config_path) water_mask_path = 'water_mask.wgs84' create_water_mask(str(dem_path), water_mask_path) multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) @@ -87,12 +87,12 @@ def insar_tops( resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') - topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) + topsapp.run_topsapp(start='unwrap', end='unwrap2stage', config_xml=config_path) isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') else: - topsapp.run_topsapp_burst(start='startup', end='unwrap2stage', config_xml=config_path) + topsapp.run_topsapp(start='startup', end='unwrap2stage', config_xml=config_path) copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') - topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) + topsapp.run_topsapp(start='geocode', end='geocode', config_xml=config_path) return Path('merged') diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index fd36b1c8..c9a52e74 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -96,7 +96,7 @@ def insar_tops_burst( orbit_file = fetch_for_scene(granule, dir=orbit_dir) log.info(f'Got orbit file {orbit_file} from s1_orbits') - config = topsapp.TopsappBurstConfig( + config = topsapp.TopsappConfig( reference_safe=f'{ref_params.granule}.SAFE', secondary_safe=f'{sec_params.granule}.SAFE', polarization=ref_params.polarization, @@ -111,10 +111,10 @@ def insar_tops_burst( ) config_path = config.write_template('topsApp.xml') - topsapp.run_topsapp_burst(start='startup', end='preprocess', config_xml=config_path) + topsapp.run_topsapp(start='startup', end='preprocess', config_xml=config_path) topsapp.swap_burst_vrts() if apply_water_mask: - topsapp.run_topsapp_burst(start='computeBaselines', end='filter', config_xml=config_path) + topsapp.run_topsapp(start='computeBaselines', end='filter', config_xml=config_path) water_mask_path = 'water_mask.wgs84' create_water_mask(str(dem_path), water_mask_path) multilook('merged/lon.rdr.full', outname='merged/lon.rdr', alks=azimuth_looks, rlks=range_looks) @@ -122,12 +122,12 @@ def insar_tops_burst( resample_to_radar_io(water_mask_path, 'merged/lat.rdr', 'merged/lon.rdr', 'merged/water_mask.rdr') isce2_copy('merged/phsig.cor', 'merged/unmasked.phsig.cor') image_math('merged/unmasked.phsig.cor', 'merged/water_mask.rdr', 'merged/phsig.cor', 'a*b') - topsapp.run_topsapp_burst(start='unwrap', end='unwrap2stage', config_xml=config_path) + topsapp.run_topsapp(start='unwrap', end='unwrap2stage', config_xml=config_path) isce2_copy('merged/unmasked.phsig.cor', 'merged/phsig.cor') else: - topsapp.run_topsapp_burst(start='computeBaselines', end='unwrap2stage', config_xml=config_path) + topsapp.run_topsapp(start='computeBaselines', end='unwrap2stage', config_xml=config_path) copyfile('merged/z.rdr.full.xml', 'merged/z.rdr.full.vrt.xml') - topsapp.run_topsapp_burst(start='geocode', end='geocode', config_xml=config_path) + topsapp.run_topsapp(start='geocode', end='geocode', config_xml=config_path) return Path('merged') diff --git a/src/hyp3_isce2/topsapp.py b/src/hyp3_isce2/topsapp.py index c233c0b3..7e7eecc5 100644 --- a/src/hyp3_isce2/topsapp.py +++ b/src/hyp3_isce2/topsapp.py @@ -41,7 +41,7 @@ ] -class TopsappBurstConfig: +class TopsappConfig: """Configuration for a topsApp.py run""" def __init__( @@ -135,8 +135,8 @@ def swap_burst_vrts(): del base -def run_topsapp_burst(dostep: str = '', start: str = '', end: str = '', config_xml: Path = Path('topsApp.xml')): - """Run topsApp.py for a burst pair with the desired steps and config file +def run_topsapp(dostep: str = '', start: str = '', end: str = '', config_xml: Path = Path('topsApp.xml')): + """Run topsApp.py for a granule pair with the desired steps and config file Args: dostep: The step to run diff --git a/tests/test_topsapp.py b/tests/test_topsapp.py index 272c0af8..fd1689df 100644 --- a/tests/test_topsapp.py +++ b/tests/test_topsapp.py @@ -1,10 +1,10 @@ import pytest -from hyp3_isce2.topsapp import TopsappBurstConfig, run_topsapp_burst, swap_burst_vrts +from hyp3_isce2.topsapp import TopsappConfig, run_topsapp, swap_burst_vrts def test_topsapp_burst_config(tmp_path): - config = TopsappBurstConfig( + config = TopsappConfig( reference_safe='S1A_IW_SLC__1SDV_20200604T022251_20200604T022318_032861_03CE65_7C85.SAFE', secondary_safe='S1A_IW_SLC__1SDV_20200616T022252_20200616T022319_033036_03D3A3_5D11.SAFE', polarization='VV', @@ -49,9 +49,9 @@ def test_swap_burst_vrts(tmp_path, monkeypatch): def test_run_topsapp_burst(tmp_path, monkeypatch): with pytest.raises(IOError): - run_topsapp_burst('topsApp.xml') + run_topsapp('topsApp.xml') - config = TopsappBurstConfig( + config = TopsappConfig( reference_safe='', secondary_safe='', polarization='', @@ -67,10 +67,10 @@ def test_run_topsapp_burst(tmp_path, monkeypatch): template_path = config.write_template(tmp_path / 'topsApp.xml') with pytest.raises(ValueError, match=r'.*not a valid step.*'): - run_topsapp_burst('notastep', config_xml=template_path) + run_topsapp('notastep', config_xml=template_path) with pytest.raises(ValueError, match=r'^If dostep is specified, start and stop cannot be used$'): - run_topsapp_burst('preprocess', 'startup', config_xml=template_path) + run_topsapp('preprocess', 'startup', config_xml=template_path) monkeypatch.chdir(tmp_path) - run_topsapp_burst('preprocess', config_xml=template_path) + run_topsapp('preprocess', config_xml=template_path) From 565250920189f3e8a0306aef1006f6663c13fbd5 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 26 Aug 2024 15:53:51 -0500 Subject: [PATCH 81/81] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13c58c27..133bfd5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed - The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists. The positional `granules` argument is now optional and deprecated. +- Moved HyP3 product packaging functionality out of `insar_tops_burst.py` and to a new `packaging.py` so that both `insar_tops` and `insar_tops_burst` can use it. ## [2.0.0] ### Changed