Merge pull request #198 from ASFHyP3/merge_readme

Create a merge workflow specific README file for merge products

CHANGELOG.md

@@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ### Added
 * `merge_tops_bursts.py` file and workflow for merge burst products created using insar_tops_bursts.
 * `merge_tops_bursts` entrypoint
+* `merge_tops_bursts` README template and creation functionality
 * several classes and functions to `burst.py` and `utils.py` to support `merge_tops_burst`.
 * tests for the added functionality.
 * `tests/data/merge.zip` example data for testing merge workflow.

src/hyp3_isce2/merge_tops_bursts.py

@@ -13,9 +13,10 @@
 from secrets import token_hex
 from shutil import make_archive
 from tempfile import TemporaryDirectory
-from typing import Iterable, Optional, Tuple
+from typing import Iterable, List, Optional, Tuple
 
 import asf_search
+import isce
 import isceobj
 import lxml.etree as ET
 import numpy as np
@@ -35,6 +36,7 @@
 from stdproc.rectify.geocode.Geocodable import Geocodable
 from zerodop.geozero import createGeozero
 
+import hyp3_isce2
 import hyp3_isce2.burst as burst_utils
 from hyp3_isce2.dem import download_dem_for_isce2
 from hyp3_isce2.utils import (
@@ -901,9 +903,24 @@ def get_product_name(product: BurstProduct, pixel_size: int) -> str:
     )
 
 
+def get_product_metadata_info(base_dir: Path) -> List:
+    """Get the metadata for a set of ASF burst products
+
+    Args:
+        base_dir: The directory containing UNZIPPED ASF burst products
+
+    Returns:
+        A list of metadata dictionaries
+    """
+    product_paths = list(Path(base_dir).glob('S1_??????_IW?_*'))
+    meta_file_paths = [path / f'{path.name}.txt' for path in product_paths]
+    metas = [read_product_metadata(path) for path in meta_file_paths]
+    return metas
+
+
 def make_parameter_file(
     out_path: Path,
-    product_directory: Path,
+    metas: List,
     range_looks: int,
     azimuth_looks: int,
     filter_strength: float,
@@ -916,10 +933,10 @@ def make_parameter_file(
 
     Args:
         out_path: The path to write the parameter file to
-        product_directory: The path to the directory containing the ASF burst product directories
+        metas: A list of metadata dictionaries for the burst products
+        range_looks: The number of range looks
         azimuth_looks: The number of azimuth looks
         filter_strength: The Goldstein-Werner filter strength
-        range_looks: The number of range looks
         water_mask: Whether or not to use a water mask
         dem_name: The name of the source DEM
         dem_resolution: The resolution of the source DEM
@@ -932,9 +949,6 @@ def make_parameter_file(
     SPACECRAFT_HEIGHT = 693000.0
     EARTH_RADIUS = 6337286.638938101
 
-    product_paths = list(product_directory.glob('S1_??????_IW?_*'))
-    meta_file_paths = [path / f'{path.name}.txt' for path in product_paths]
-    metas = [read_product_metadata(path) for path in meta_file_paths]
     reference_scenes = [meta['ReferenceGranule'] for meta in metas]
     secondary_scenes = [meta['SecondaryGranule'] for meta in metas]
     ref_orbit_number = metas[0]['ReferenceOrbitNumber']
@@ -983,6 +997,55 @@ def make_parameter_file(
     parameter_file.write(out_path)
 
 
+def make_readme(
+    product_dir: Path,
+    reference_scenes: Iterable,
+    secondary_scenes: Iterable,
+    range_looks: int,
+    azimuth_looks: int,
+    apply_water_mask: bool,
+) -> None:
+    """Create a README file for the merged burst product and write it to product_dir
+
+    Args:
+        product_dir: The path to the directory containing the merged burst product,
+            the directory name should be the product name
+        reference_scenes: A list of reference scenes
+        secondary_scenes: A list of secondary scenes
+        range_looks: The number of range looks
+        azimuth_looks: The number of azimuth looks
+        apply_water_mask: Whether or not a water mask was applied
+    """
+    product_name = product_dir.name
+    wrapped_phase_path = product_dir / f'{product_name}_wrapped_phase.tif'
+    info = gdal.Info(str(wrapped_phase_path), format='json')
+    secondary_granule_datetime_str = secondary_scenes[0].split('_')[3]
+
+    payload = {
+        'processing_date': datetime.datetime.now(datetime.timezone.utc),
+        'plugin_name': hyp3_isce2.__name__,
+        'plugin_version': hyp3_isce2.__version__,
+        'processor_name': isce.__name__.upper(),  # noqa
+        'processor_version': isce.__version__,  # noqa
+        'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']),
+        'pixel_spacing': info['geoTransform'][1],
+        'product_name': product_name,
+        'reference_burst_name': ', '.join(reference_scenes),
+        'secondary_burst_name': ', '.join(secondary_scenes),
+        'range_looks': range_looks,
+        'azimuth_looks': azimuth_looks,
+        'secondary_granule_date': datetime.datetime.strptime(secondary_granule_datetime_str, '%Y%m%dT%H%M%S'),
+        'dem_name': 'GLO-30',
+        'dem_pixel_spacing': '30 m',
+        'apply_water_mask': apply_water_mask,
+    }
+    content = hyp3_isce2.metadata.util.render_template('insar_burst/insar_burst_merge_readme.md.txt.j2', payload)
+
+    output_file = product_dir / f'{product_name}_README.md.txt'
+    with open(output_file, 'w') as f:
+        f.write(content)
+
+
 def check_burst_group_validity(products) -> None:
     """Check that a set of burst products are valid for merging. This includes:
     All products have the same:
@@ -1131,18 +1194,23 @@ def package_output(
     product_path = list(product_directory.glob('S1_??????_IW?_*'))[0]
     example_metadata = get_burst_metadata([product_path])[0]
     product_name = get_product_name(example_metadata, pixel_size)
-    product_dir = Path(product_name)
-    product_dir.mkdir(parents=True, exist_ok=True)
+    out_product_dir = Path(product_name)
+    out_product_dir.mkdir(parents=True, exist_ok=True)
 
+    metas = get_product_metadata_info(product_directory)
     make_parameter_file(
         Path(f'{product_name}/{product_name}.txt'),
-        product_directory,
+        metas,
         range_looks,
         azimuth_looks,
         filter_strength,
         water_mask,
     )
+
     translate_outputs(product_name, pixel_size=pixel_size, include_radar=False)
+    reference_scenes = [meta['ReferenceGranule'] for meta in metas]
+    secondary_scenes = [meta['SecondaryGranule'] for meta in metas]
+    make_readme(out_product_dir, reference_scenes, secondary_scenes, range_looks, azimuth_looks, water_mask)
     unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif'
     make_browse_image(unwrapped_phase, f'{product_name}/{product_name}_unw_phase.png')
     if archive:
@@ -1159,12 +1227,8 @@ def merge_tops_bursts(product_directory: Path, filter_strength: float, apply_wat
     """
     range_looks, azimuth_looks = get_product_multilook(product_directory)
     prepare_products(product_directory)
-    run_isce2_workflow(
-        range_looks, azimuth_looks, filter_strength=filter_strength, apply_water_mask=apply_water_mask
-    )
-    package_output(
-        product_directory, f'{range_looks}x{azimuth_looks}', filter_strength, water_mask=apply_water_mask
-    )
+    run_isce2_workflow(range_looks, azimuth_looks, filter_strength=filter_strength, apply_water_mask=apply_water_mask)
+    package_output(product_directory, f'{range_looks}x{azimuth_looks}', filter_strength, water_mask=apply_water_mask)
 
 
 def main():

src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_base.md.txt.j2

@@ -91,6 +91,12 @@ The files generated in this process include:
 8. Water Mask (GeoTIFF)
 9. README.md.txt (Text File)
 
+There are also four non-geocoded GeoTIFFs that remain in their native range-doppler coordinates. These four images compose
+the image data needed to merge burst InSAR products together. These images include a range-doppler version of the wrapped
+interferogram, a two-band range-doppler look vector image in the native ISCE2 format, and latitude/longitude images that
+provide the information necessary to map range-doppler images into the geocoded domain. These images files are not
+included in merged burst InSAR products.
+
 *See below for detailed descriptions of each of the product files.*
 
 ----------------
@@ -263,4 +269,4 @@ Contact the HyP3 development team directly at:
 https://hyp3-docs.asf.alaska.edu/contact/
 
 -------------
-Metadata version: {{ plugin_version }}
+Metadata version: {{ plugin_version }}

src/hyp3_isce2/metadata/templates/insar_burst/insar_burst_merge_readme.md.txt.j2

@@ -0,0 +1,125 @@
+{% extends "insar_burst/insar_burst_base.md.txt.j2" %}
+
+Note to reader: This readme file includes text blocks to extend the insar burst base file.
+Only text included in blocks called in the base file will be included in the output readme.
+We have simple placeholders for readability in this file to indicate where the base file will have its own sections.
+
+{% block header %}
+ASF Sentinel-1 Burst InSAR Data Package (ISCE2)
+===============================================
+
+This folder contains merged burst-based SAR Interferometry (InSAR) products and their associated files. The source data for
+these products are Sentinel-1 bursts, extracted from Single Look Complex (SLC) products processed by ESA,
+and they were processed using InSAR Scientific Computing Environment version 2 (ISCE2) software.
+
+Refer to
+https://sentinels.copernicus.eu/web/sentinel/user-guides/sentinel-1-sar/acquisition-modes/interferometric-wide-swath
+for more information on Sentinel-1 bursts.
+
+This data was processed by ASF DAAC HyP3 {{ processing_date.year }} using the {{ plugin_name }} plugin version
+{{ plugin_version }} running {{ processor_name }} release {{ processor_version }}.
+Files are projected to {{ projection }}, and the pixel spacing is {{ pixel_spacing|int }} m.
+
+The source bursts for this InSAR product are:
+ - Reference: {{ reference_burst_name }}
+ - Secondary: {{ secondary_burst_name }}
+
+Processing Date/Time: {{ processing_date.isoformat(timespec='seconds') }}
+
+The directory name for this product is: {{ product_name }}
+
+The output directory uses the following naming convention:
+
+S1_rrr__yyyymmdd_yyyymmdd_pp_INTzz_cccc
+
+rrr: Relative orbit ID values assigned by ESA. Merged burst InSAR products can contain many relative burst IDs, so the
+        relate orbit ID is used in lieu of relative burst IDs for these products
+
+yyyymmdd: Date of acquisition of the reference and secondary images, respectively.
+
+pp: Two character combination that represents the mode of radar orientation (polarization) for both signal
+        transmission and reception. The first position represents the transmit orientation mode and the second
+        position represents the receive orientation mode.
+
+    HH: Horizontal Transmit - Horizontal Receive
+    HV: Horizontal Transmit - Vertical Receive
+    VH: Vertical Transmit - Horizontal Receive
+    VV: Vertical Transmit - Vertical Receive
+
+INT: The product type (always INT for InSAR).
+
+zz: The pixel spacing of the output image.
+
+cccc: 4-character unique product identifier.
+
+Files contained in the product directory are named using the directory name followed by a tag indicating the file type.
+{% endblock %}
+----------------
+(This is where the base file has the Pixel Spacing section)
+
+----------------
+(This is where the base file has the Using This Data section)
+
+***************
+(This is where the base file has parts 1-8 of the Product Contents)
+
+*************
+{% block burst_insar_processing %}
+# Burst InSAR Processing #
+
+The basic steps in Sentinel-1 Burst InSAR processing are as follows:
+
+*Pre-Processing*
+1. Check that the input burst InSAR products are capable of being merged
+2. Recreate the ISCE2 post-interferogram generation directory structure
+3. Reformat range-doppler burst InSAR product datasets to an ISCE2-compatible format
+4. Create ISCE2 Sentinel-1 objects with the correct burst/multilook information
+
+*InSAR Processing*
+5. Run topsApp step 'mergebursts'
+6. Optionally apply the water mask to the wrapped image.
+7. Run topsApp steps 'unwrap' and 'unwrap2stage'
+8. Run step 'geocode'
+
+*Post-Processing*
+9. translate output files to hyp3 format
+10. write the README text file
+11. write the metadata txt file
+
+----------------
+The detailed process, including the calls to ISCE2 software, is as follows:
+
+The prepare-processing and InSAR processing are combined in the insar_tps_burst function.
+
+## Pre-processing ##
+ - merge_tops_bursts.check_burst_group_validity:Check that the input burst InSAR products are capable of being merged
+ - merge_tops_bursts.download_metadata_xmls: Download metadata files for a set of burst InSAR products
+ - merge_tops_bursts.create_burst_cropped_s1_obj: Create ISCE2 `Sentinel1` objects for the swaths/bursts present
+ - merge_tops_bursts.spoof_isce2_setup: Recreate an ISCE2 setup post-`burstifg`
+ - merge_tops_bursts.download_dem_for_multiple_bursts: Download DEM for merge run
+
+## InSAR processing ##
+The ISCE2 InSAR processing this product uses includes the following ISCE2 topsApp steps:
+- mergebursts
+- filter
+- unwrap
+- unwrap2stage
+- geocode
+
+These steps are run using these calls within hyp3-isce2:
+- merge_tops_bursts.merge_bursts: Merge the wrapped burst interferograms
+- merge_tops_bursts.goldstein_werner_filter: Apply the Goldstien-Werner Phase Filter
+- merge_tops_bursts.mask_coherence: Optionally mask data before unwrapping
+- merge_tops_bursts.snaphu_unwrap: Unwrap the merged interferogram using SNAPHU
+- merge_tops_bursts.geocode_products: Geocode the output products
+
+## Post-Processing ##
+ - merge_tops_bursts.make_parameter_file: Produce metadata text file in the product
+ - merge_tops_bursts.translate_outputs: Convert the outputs of hyp3-isce2 to hyp3-gamma formatted geotiff files
+ - merge_tops_bursts.make_browse_image: Create a browse image for the dataset
+ - merge_tops_bursts.make_readme: Produce the readme.md.txt file in the product
+ {% endblock %}
+
+ -----------
+ (This is where the base file has a S1 Mission section)
+ (This is where the base file has a footer)

tests/test_merge_tops_bursts.py

@@ -6,6 +6,7 @@
 from unittest.mock import patch
 
 import asf_search
+import isceobj  # noqa: I100
 import lxml.etree as ET
 import numpy as np
 import pytest
@@ -16,8 +17,6 @@
 import hyp3_isce2.merge_tops_bursts as merge
 from hyp3_isce2 import utils
 
-import isceobj  # noqa: I100
-
 
 def mock_asf_search_results(
     slc_name: str,
@@ -43,7 +42,7 @@ def mock_asf_search_results(
     return results
 
 
-def create_test_geotiff(output_file, dtype='float32', n_bands=1):
+def create_test_geotiff(output_file, dtype='float', n_bands=1):
     """Create a test geotiff for testing"""
     opts = {'float': (np.float64, gdal.GDT_Float64), 'cfloat': (np.complex64, gdal.GDT_CFloat32)}
     np_dtype, gdal_dtype = opts[dtype]
@@ -292,8 +291,10 @@ def test_make_parameter_file(test_data_dir, test_merge_dir, test_s1_obj, tmp_pat
     test_s1_obj.output = str(ifg_dir.parent / 'IW2')
     test_s1_obj.write_xml()
 
+    metas = merge.get_product_metadata_info(test_merge_dir)
+
     out_file = tmp_path / 'test.txt'
-    merge.make_parameter_file(out_file, test_merge_dir, 20, 4, 0.6, True, base_dir=tmp_path)
+    merge.make_parameter_file(out_file, metas, 20, 4, 0.6, True, base_dir=tmp_path)
     assert out_file.exists()
 
     meta = utils.read_product_metadata(out_file)
@@ -426,3 +427,16 @@ def test_get_product_multilook(tmp_path):
     range_looks, azimuth_looks = merge.get_product_multilook(product_dir)
     assert range_looks == 20
     assert azimuth_looks == 4
+
+
+def test_make_readme(tmp_path):
+    prod_name = 'foo'
+    tmp_prod_dir = tmp_path / prod_name
+    tmp_prod_dir.mkdir(exist_ok=True)
+    create_test_geotiff(str(tmp_prod_dir / f'{prod_name}_wrapped_phase.tif'))
+    reference_scenes = ['a_a_a_20200101T000000_a', 'b_b_b_20200101T000000_b']
+    secondary_scenes = ['c_c_c_20210101T000000_c', 'd_d_d_20210101T000000_d']
+
+    merge.make_readme(tmp_prod_dir, reference_scenes, secondary_scenes, 2, 10, True)
+    out_path = tmp_prod_dir / f'{prod_name}_README.md.txt'
+    assert out_path.exists()