From f1cc5453931f144cf908c9c89869d0d8814c1f8c Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 12 Feb 2024 16:31:29 -0600 Subject: [PATCH 1/6] add create_readme function for merge workflow --- src/hyp3_isce2/merge_tops_bursts.py | 48 ++++++++++++++++--- .../insar_burst/insar_burst_base.md.txt.j2 | 8 +++- tests/test_merge_tops_bursts.py | 26 +++++++--- 3 files changed, 68 insertions(+), 14 deletions(-) diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index 421aaf8f..a51aa03a 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -16,6 +16,7 @@ from typing import Iterable, Optional, Tuple import asf_search +import isce import isceobj import lxml.etree as ET import numpy as np @@ -35,6 +36,7 @@ from stdproc.rectify.geocode.Geocodable import Geocodable from zerodop.geozero import createGeozero +import hyp3_isce2 import hyp3_isce2.burst as burst_utils from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.utils import ( @@ -983,6 +985,44 @@ def make_parameter_file( parameter_file.write(out_path) +def make_readme( + product_dir: Path, + product_name: str, + reference_scenes: Iterable[str], + secondary_scenes: Iterable[str], + range_looks: int, + azimuth_looks: int, + apply_water_mask: bool, +) -> None: + wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' + info = gdal.Info(str(wrapped_phase_path), format='json') + secondary_granule_datetime_str = secondary_scenes[0].split('_')[3] + + payload = { + 'processing_date': datetime.datetime.now(datetime.timezone.utc), + 'plugin_name': hyp3_isce2.__name__, + 'plugin_version': hyp3_isce2.__version__, + 'processor_name': isce.__name__.upper(), # noqa + 'processor_version': isce.__version__, # noqa + 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), + 'pixel_spacing': info['geoTransform'][1], + 'product_name': product_name, + 'reference_burst_name': ', '.join(reference_scenes), + 'secondary_burst_name': ', '.join(secondary_scenes), + 'range_looks': range_looks, + 'azimuth_looks': azimuth_looks, + 'secondary_granule_date': datetime.datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'), + 'dem_name': 'GLO-30', + 'dem_pixel_spacing': '30 m', + 'apply_water_mask': apply_water_mask, + } + content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_merge_readme.md.txt.j2', payload) + + output_file = product_dir / f'{product_name}_README.md.txt' + with open(output_file, 'w') as f: + f.write(content) + + def check_burst_group_validity(products) -> None: """Check that a set of burst products are valid for merging. This includes: All products have the same: @@ -1159,12 +1199,8 @@ def merge_tops_bursts(product_directory: Path, filter_strength: float, apply_wat """ range_looks, azimuth_looks = get_product_multilook(product_directory) prepare_products(product_directory) - run_isce2_workflow( - range_looks, azimuth_looks, filter_strength=filter_strength, apply_water_mask=apply_water_mask - ) - package_output( - product_directory, f'{range_looks}x{azimuth_looks}', filter_strength, water_mask=apply_water_mask - ) + run_isce2_workflow(range_looks, azimuth_looks, filter_strength=filter_strength, apply_water_mask=apply_water_mask) + package_output(product_directory, f'{range_looks}x{azimuth_looks}', filter_strength, water_mask=apply_water_mask) def main(): diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_base.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_base.md.txt.j2 index 7c5ddec3..a498c2f3 100644 --- a/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_base.md.txt.j2 +++ b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_base.md.txt.j2 @@ -91,6 +91,12 @@ The files generated in this process include: 8. Water Mask (GeoTIFF) 9. README.md.txt (Text File) +There are also four non-geocoded GeoTIFFs that remain in their native range-doppler coordinates. These four images compose +the image data needed to merge burst InSAR products together. These images include a range-doppler version of the wrapped +interferogram, a two-band range-doppler look vector image in the native ISCE2 format, and latitude/longitude images that +provide the information necessary to map range-doppler images into the geocoded domain. These images files are not +included in merged burst InSAR products. + *See below for detailed descriptions of each of the product files.* ---------------- @@ -263,4 +269,4 @@ Contact the HyP3 development team directly at: https://hyp3-docs.asf.alaska.edu/contact/ ------------- -Metadata version: {{ plugin_version }} \ No newline at end of file +Metadata version: {{ plugin_version }} diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 5386f581..41897584 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -6,18 +6,16 @@ from unittest.mock import patch import asf_search +import hyp3_isce2.burst as burst_utils +import hyp3_isce2.merge_tops_bursts as merge +import isceobj # noqa: I100 import lxml.etree as ET import numpy as np import pytest +from hyp3_isce2 import utils from osgeo import gdal, osr from requests import Session -import hyp3_isce2.burst as burst_utils -import hyp3_isce2.merge_tops_bursts as merge -from hyp3_isce2 import utils - -import isceobj # noqa: I100 - def mock_asf_search_results( slc_name: str, @@ -43,7 +41,7 @@ def mock_asf_search_results( return results -def create_test_geotiff(output_file, dtype='float32', n_bands=1): +def create_test_geotiff(output_file, dtype='float', n_bands=1): """Create a test geotiff for testing""" opts = {'float': (np.float64, gdal.GDT_Float64), 'cfloat': (np.complex64, gdal.GDT_CFloat32)} np_dtype, gdal_dtype = opts[dtype] @@ -426,3 +424,17 @@ def test_get_product_multilook(tmp_path): range_looks, azimuth_looks = merge.get_product_multilook(product_dir) assert range_looks == 20 assert azimuth_looks == 4 + + +def test_make_readme(tmp_path): + prod_name = 'foo' + # tmp_prod_dir = Path.cwd() / prod_name + tmp_prod_dir = tmp_path / prod_name + tmp_prod_dir.mkdir(exist_ok=True) + create_test_geotiff(str(tmp_prod_dir / f'{prod_name}_wrapped_phase.tif')) + reference_scenes = ['a_a_a_20200101T000000_a', 'b_b_b_20200101T000000_b'] + secondary_scenes = ['c_c_c_20210101T000000_c', 'd_d_d_20210101T000000_d'] + + merge.make_readme(tmp_prod_dir, prod_name, reference_scenes, secondary_scenes, 2, 10, True) + out_path = tmp_prod_dir / f'{prod_name}_README.md.txt' + assert out_path.exists() From 69e5b8ba8ad07a747e57e164784ec77d3cb61176 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 12 Feb 2024 16:31:52 -0600 Subject: [PATCH 2/6] add new template --- .../insar_burst_merge_readme.md.txt.j2 | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 new file mode 100644 index 00000000..8b72ac8c --- /dev/null +++ b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 @@ -0,0 +1,126 @@ +{% extends "insar_burst/insar_burst_base.md.txt.j2" %} + +Note to reader: This readme file includes text blocks to extend the insar burst base file. +Only text included in blocks called in the base file will be included in the output readme. +We have simple placeholders for readability in this file to indicate where the base file will have its own sections. + +{% block header %} +ASF Sentinel-1 Burst InSAR Data Package (ISCE2) +=============================================== + +This folder contains merged burst-based SAR Interferometry (InSAR) products and their associated files. The source data for +these products are Sentinel-1 bursts, extracted from Single Look Complex (SLC) products processed by ESA, +and they were processed using InSAR Scientific Computing Environment version 2 (ISCE2) software. + +Refer to +https://sentinels.copernicus.eu/web/sentinel/user-guides/sentinel-1-sar/acquisition-modes/interferometric-wide-swath +for more information on Sentinel-1 bursts. + +This data was processed by ASF DAAC HyP3 {{ processing_date.year }} using the {{ plugin_name }} plugin version +{{ plugin_version }} running {{ processor_name }} release {{ processor_version }}. +Files are projected to {{ projection }}, and the pixel spacing is {{ pixel_spacing|int }} m. + +The source bursts for this InSAR product are: + - Reference: {{ reference_burst_name }} + - Secondary: {{ secondary_burst_name }} + +Processing Date/Time: {{ processing_date.isoformat(timespec='seconds') }} + +The directory name for this product is: {{ product_name }} + +The output directory uses the following naming convention: + +S1_rrr__yyyymmdd_yyyymmdd_pp_INTzz_cccc + +rrr: Relative orbit ID values assigned by ESA. Merged burst InSAR products can contain many relative burst IDs, so the + relate orbit ID is used in lieu of relative burst IDs for these products + +yyyymmdd: Date of acquisition of the reference and secondary images, respectively. + +pp: Two character combination that represents the mode of radar orientation (polarization) for both signal + transmission and reception. The first position represents the transmit orientation mode and the second + position represents the receive orientation mode. + + HH: Horizontal Transmit - Horizontal Receive + HV: Horizontal Transmit - Vertical Receive + VH: Vertical Transmit - Horizontal Receive + VV: Vertical Transmit - Vertical Receive + +INT: The product type (always INT for InSAR). + +zz: The pixel spacing of the output image. + +cccc: 4-character unique product identifier. + +Files contained in the product directory are named using the directory name followed by a tag indicating the file type. +{% endblock %} +---------------- +(This is where the base file has the Pixel Spacing section) + +---------------- +(This is where the base file has the Using This Data section) + +*************** +(This is where the base file has parts 1-8 of the Product Contents) + +************* +{% block burst_insar_processing %} +# Burst InSAR Processing # + +The basic steps in Sentinel-1 Burst InSAR processing are as follows: + +*Pre-Processing* +1. Check that the input burst InSAR products are capable of being merged +2. Recreate the directory the post-interferogram generation ISCE2 directory structure +3. Reformat range-doppler burst InSAR product datasets to an ISCE2-compatible format +4. Create ISCE2 Sentinel-1 objects with the correct burst/multilook information + +*InSAR Processing* +5. Run topsApp step 'mergebursts' +6. Optionally apply the water mask to the wrapped image. +7. Run topsApp steps 'unwrap' and 'unwrap2stage' +8. Run step 'geocode' + +*Post-Processing* +9. translate output files to hyp3 format +10. write the README text file +11. write the metadata txt file + +---------------- +The detailed process, including the calls to ISCE2 software, is as follows: + +The prepare-processing and InSAR processing are combined in the insar_tps_burst function. + +## Pre-processing ## + - merge_tops_bursts.check_burst_group_validity:Check that the input burst InSAR products are capable of being merged + - merge_tops_bursts.download_metadata_xmls: Download metadata files for a set of burst InSAR products + - merge_tops_bursts.create_burst_cropped_s1_obj: Create ISCE2 `Sentinel1` objects for the swaths/bursts present + - merge_tops_bursts.spoof_isce2_setup: Recreate an ISCE2 setup post-`burstifg` + - merge_tops_bursts.download_dem_for_multiple_bursts: Download DEM for merge run + + +## InSAR processing ## +The ISCE2 InSAR processing this product uses includes the following ISCE2 topsApp steps: +- mergebursts +- filter +- unwrap +- unwrap2stage +- geocode + +These steps are run using these calls within hyp3-isce2: +- merge_tops_bursts.merge_bursts: Merge the wrapped burst interferograms +- merge_tops_bursts.goldstein_werner_filter: Apply the Goldstien-Werner Phase Filter +- merge_tops_bursts.mask_coherence: Optionally mask data before unwrapping +- merge_tops_bursts.snaphu_unwrap: Unwrap the merged interferogram using SNAPHU +- merge_tops_bursts.geocode_products: Geocode the output products + +## Post-Processing ## + - merge_tops_bursts.make_parameter_file: Produce metadata text file in the product + - merge_tops_bursts.translate_outputs: Convert the outputs of hyp3-isce2 to hyp3-gamma formatted geotiff files + - merge_tops_bursts.make_browse_image: Create a browse image for the dataset + - merge_tops_bursts.make_readme: Produce the readme.md.txt file in the product + {% endblock %} + + ----------- + (This is where the base file has a S1 Mission section) + (This is where the base file has a footer) From f442db87f546a3a6d54f260a6a94668ac88ebd36 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Feb 2024 12:49:10 -0600 Subject: [PATCH 3/6] fully integrate merge readme --- src/hyp3_isce2/merge_tops_bursts.py | 54 ++++++++++++++----- .../insar_burst_merge_readme.md.txt.j2 | 3 +- tests/test_merge_tops_bursts.py | 7 +-- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index a51aa03a..ae71429c 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -13,7 +13,7 @@ from secrets import token_hex from shutil import make_archive from tempfile import TemporaryDirectory -from typing import Iterable, Optional, Tuple +from typing import Iterable, List, Optional, Tuple import asf_search import isce @@ -903,9 +903,24 @@ def get_product_name(product: BurstProduct, pixel_size: int) -> str: ) +def get_product_metadata_info(base_dir: Path) -> List: + """Get the metadata for a set of ASF burst products + + Args: + base_dir: The directory containing UNZIPPED ASF burst products + + Returns: + A list of metadata dictionaries + """ + product_paths = list(Path(base_dir).glob('S1_??????_IW?_*')) + meta_file_paths = [path / f'{path.name}.txt' for path in product_paths] + metas = [read_product_metadata(path) for path in meta_file_paths] + return metas + + def make_parameter_file( out_path: Path, - product_directory: Path, + metas: List, range_looks: int, azimuth_looks: int, filter_strength: float, @@ -918,10 +933,10 @@ def make_parameter_file( Args: out_path: The path to write the parameter file to - product_directory: The path to the directory containing the ASF burst product directories + metas: A list of metadata dictionaries for the burst products + range_looks: The number of range looks azimuth_looks: The number of azimuth looks filter_strength: The Goldstein-Werner filter strength - range_looks: The number of range looks water_mask: Whether or not to use a water mask dem_name: The name of the source DEM dem_resolution: The resolution of the source DEM @@ -934,9 +949,6 @@ def make_parameter_file( SPACECRAFT_HEIGHT = 693000.0 EARTH_RADIUS = 6337286.638938101 - product_paths = list(product_directory.glob('S1_??????_IW?_*')) - meta_file_paths = [path / f'{path.name}.txt' for path in product_paths] - metas = [read_product_metadata(path) for path in meta_file_paths] reference_scenes = [meta['ReferenceGranule'] for meta in metas] secondary_scenes = [meta['SecondaryGranule'] for meta in metas] ref_orbit_number = metas[0]['ReferenceOrbitNumber'] @@ -987,13 +999,24 @@ def make_parameter_file( def make_readme( product_dir: Path, - product_name: str, - reference_scenes: Iterable[str], - secondary_scenes: Iterable[str], + reference_scenes: Iterable, + secondary_scenes: Iterable, range_looks: int, azimuth_looks: int, apply_water_mask: bool, ) -> None: + """Create a README file for the merged burst product and write it to product_dir + + Args: + product_dir: The path to the directory containing the merged burst product, + the directory name should be the product name + reference_scenes: A list of reference scenes + secondary_scenes: A list of secondary scenes + range_looks: The number of range looks + azimuth_looks: The number of azimuth looks + apply_water_mask: Whether or not a water mask was applied + """ + product_name = product_dir.name wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif' info = gdal.Info(str(wrapped_phase_path), format='json') secondary_granule_datetime_str = secondary_scenes[0].split('_')[3] @@ -1171,18 +1194,23 @@ def package_output( product_path = list(product_directory.glob('S1_??????_IW?_*'))[0] example_metadata = get_burst_metadata([product_path])[0] product_name = get_product_name(example_metadata, pixel_size) - product_dir = Path(product_name) - product_dir.mkdir(parents=True, exist_ok=True) + out_product_dir = Path(product_name) + out_product_dir.mkdir(parents=True, exist_ok=True) + metas = get_product_metadata_info(product_directory) make_parameter_file( Path(f'{product_name}/{product_name}.txt'), - product_directory, + metas, range_looks, azimuth_looks, filter_strength, water_mask, ) + translate_outputs(product_name, pixel_size=pixel_size, include_radar=False) + reference_scenes = [meta['ReferenceGranule'] for meta in metas] + secondary_scenes = [meta['SecondaryGranule'] for meta in metas] + make_readme(out_product_dir, reference_scenes, secondary_scenes, range_looks, azimuth_looks, water_mask) unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png') if archive: diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 index 8b72ac8c..9cbc475b 100644 --- a/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 +++ b/src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2 @@ -71,7 +71,7 @@ The basic steps in Sentinel-1 Burst InSAR processing are as follows: *Pre-Processing* 1. Check that the input burst InSAR products are capable of being merged -2. Recreate the directory the post-interferogram generation ISCE2 directory structure +2. Recreate the ISCE2 post-interferogram generation directory structure 3. Reformat range-doppler burst InSAR product datasets to an ISCE2-compatible format 4. Create ISCE2 Sentinel-1 objects with the correct burst/multilook information @@ -98,7 +98,6 @@ The prepare-processing and InSAR processing are combined in the insar_tps_burst - merge_tops_bursts.spoof_isce2_setup: Recreate an ISCE2 setup post-`burstifg` - merge_tops_bursts.download_dem_for_multiple_bursts: Download DEM for merge run - ## InSAR processing ## The ISCE2 InSAR processing this product uses includes the following ISCE2 topsApp steps: - mergebursts diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 41897584..041dfcc0 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -290,8 +290,10 @@ def test_make_parameter_file(test_data_dir, test_merge_dir, test_s1_obj, tmp_pat test_s1_obj.output = str(ifg_dir.parent / 'IW2') test_s1_obj.write_xml() + metas = merge.get_product_metadata_info(test_merge_dir) + out_file = tmp_path / 'test.txt' - merge.make_parameter_file(out_file, test_merge_dir, 20, 4, 0.6, True, base_dir=tmp_path) + merge.make_parameter_file(out_file, metas, 20, 4, 0.6, True, base_dir=tmp_path) assert out_file.exists() meta = utils.read_product_metadata(out_file) @@ -428,13 +430,12 @@ def test_get_product_multilook(tmp_path): def test_make_readme(tmp_path): prod_name = 'foo' - # tmp_prod_dir = Path.cwd() / prod_name tmp_prod_dir = tmp_path / prod_name tmp_prod_dir.mkdir(exist_ok=True) create_test_geotiff(str(tmp_prod_dir / f'{prod_name}_wrapped_phase.tif')) reference_scenes = ['a_a_a_20200101T000000_a', 'b_b_b_20200101T000000_b'] secondary_scenes = ['c_c_c_20210101T000000_c', 'd_d_d_20210101T000000_d'] - merge.make_readme(tmp_prod_dir, prod_name, reference_scenes, secondary_scenes, 2, 10, True) + merge.make_readme(tmp_prod_dir, reference_scenes, secondary_scenes, 2, 10, True) out_path = tmp_prod_dir / f'{prod_name}_README.md.txt' assert out_path.exists() From 3b2a7fefcaead4077ba5d5167dbe34b9522ad89a Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Feb 2024 12:53:16 -0600 Subject: [PATCH 4/6] fix flake8 --- tests/test_merge_tops_bursts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 041dfcc0..54ff6bb0 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -6,16 +6,17 @@ from unittest.mock import patch import asf_search -import hyp3_isce2.burst as burst_utils -import hyp3_isce2.merge_tops_bursts as merge import isceobj # noqa: I100 import lxml.etree as ET import numpy as np import pytest -from hyp3_isce2 import utils from osgeo import gdal, osr from requests import Session +import hyp3_isce2.burst as burst_utils +import hyp3_isce2.merge_tops_bursts as merge +from hyp3_isce2 import utils + def mock_asf_search_results( slc_name: str, From 39608c9ed343a3566ca9b2bb0260cf29ebec63c0 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 13 Feb 2024 12:54:17 -0600 Subject: [PATCH 5/6] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85132d03..ec4453d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added * `merge_tops_bursts.py` file and workflow for merge burst products created using insar_tops_bursts. * `merge_tops_bursts` entrypoint +* `merge_tops_bursts` README template and creation functionality * several classes and functions to `burst.py` and `utils.py` to support `merge_tops_burst`. * tests for the added functionality. * `tests/data/merge.zip` example data for testing merge workflow. From c8b5399fa3379510edc842474f0cbac4b88639e5 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 18:56:42 +0000 Subject: [PATCH 6/6] update coverage image --- images/coverage.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 1618ede7..6c15cace 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -9,13 +9,13 @@ - + coverage coverage - 74% - 74% + 75% + 75%