Use new convention for validity mask (#38)

* Use new convention for validity mask in prepare + shadow mask

* Use new convention for validity mask in prepare + shadow mask

* Use new convention for validity mask in prepare + shadow mask

* New convention for valid_stack in stack_masks

* New convention in vegetation mask

* New convention in urbanmask

* New convention in input valid_stack

* Improve unitary test to add nodata in input image, and also a fake cloud mask to test validity mask

* Fix veg mask test

* fix validation and feature tests

* Update CHANGELOG

Author: ytanguy

CHANGELOG.md

@@ -12,7 +12,7 @@ A new section Unreleased is opened then for next dev phase.
 	- Add possibility to use a nearly void config files : almost all config files shall be initiated with default values #14
 
 ### Changed
-	- Change convention for validity mask : 0 values shall stand for "valid" pixels, and other values shall stand for user specified reasons (ex : 1 for NO_DATA in input VHR image, 2 for clouds, etc.)
+	- Change convention for validity mask : 0 values shall stand for "valid" pixels, and other values shall stand for user specified reasons (ex : 1 for NO_DATA in input VHR image, 2 for clouds, etc.) --> see #32
 
 ### Fixed
 	- fix precommit (#26)

docs/source/usage_api.rst

@@ -16,7 +16,7 @@ Then, you can import it in your Python scripts or notebooks.
 
 .. code-block:: python
 
-   import slurp-masks as slurp
+   import slurp
 
 Data preparation
 ----------------

pytest.ini

@@ -21,7 +21,7 @@
 [pytest]
 addopts = -ra
 markers =
-    ci: for CI ('pytest -m ci --config config_tests_ci.json --main_config main_config_tests_ci.json --cov=slurp' to launch CI tests and compute code coverage)
+    ci: for CI ('pytest -m ci --config config_tests_ci.json --main_config main_config_tests_ci.json --cov=slurp --cov-report html' to launch CI tests and compute code coverage (HTML report))
     validation: validate all algorithms (compute and compare to reference)
     features: validate specific features (launch pytest -m "features or validation" to launch all tests)
 testpaths = tests

slurp/masks/shadowmask.py

@@ -92,8 +92,10 @@ def compute_shadowmask(
 
     :param list input_buffers: 0 -> image, 1 -> valid_stack, 2 -> watermask
     :param list input_profiles: image profiles (not used but necessary for eoscale)
-    :param dict params: must contain the keys "thresholds", "binary_opening" and "small_objects"
-    :returns: valid_phr (boolean numpy array, True = valid data, False = no data)
+    :param dict params: must contain the keys "thresholds",
+                                              "binary_opening" and "small_objects"
+    :returns: valid_mask (int numpy array) with following legend
+              0: no shadows, 1: small shadows, 2: big shadows, + NODATA
     """
     raw_shadow_mask = np.zeros(input_buffers[0][0].shape, dtype=int)
     raw_shadow_mask.fill(1)
@@ -126,7 +128,9 @@ def compute_shadowmask(
     final_shadow_mask += raw_shadow_mask
 
     # apply NO_DATA mask
-    final_shadow_mask[np.logical_not(input_buffers[1][0])] = NODATA_INT8
+    final_shadow_mask = np.where(
+        input_buffers[1][0] == 0, final_shadow_mask, NODATA_INT8
+    )
 
     return final_shadow_mask
 

slurp/masks/stack_masks.py

@@ -293,7 +293,7 @@ def post_process(
         )
 
     # Apply NODATA
-    stack[0][np.logical_not(valid_stack[0])] = NODATA_INT8
+    stack[0][np.where(valid_stack[0] != 0)] = NODATA_INT8
 
     height_layer = np.zeros((1, input_image.shape[1], input_image.shape[2]))
 
@@ -310,12 +310,12 @@ def post_process(
     height_layer[0][watermask[0] == 1] = 0
     height_layer[0][shadowmask[0] == 2] = 0
 
-    height_layer[0][np.logical_not(valid_stack[0])] = NODATA_INT8
+    height_layer[0][np.where(valid_stack[0] != 0)] = NODATA_INT8
 
     markers_layer = np.zeros((1, input_image.shape[1], input_image.shape[2]))
     # Markers
     markers_layer[0] = markers
-    markers_layer[0][np.logical_not(valid_stack[0])] = NODATA_INT8
+    markers_layer[0][np.where(valid_stack[0] != 0)] = NODATA_INT8
 
     return [stack, height_layer, markers_layer]
 
@@ -547,7 +547,8 @@ def slurp_stackmask(
     argsdict, cli_params = utils.parse_args(keys, logs_to_file, main_config)
 
     for param in cli_params:
-        # If the parameter from the CLI is not None, we update argsdict with the value from the CLI
+        # If the parameter from the CLI is not None,
+        # we update argsdict with the value from the CLI
         if locals()[param] is not None:
             argsdict[param] = locals()[param]
 

slurp/masks/urbanmask.py

@@ -67,7 +67,7 @@ def apply_vegetationmask(
     non_veg_dilated = apply_morpho(
         non_veg[0], "binary_dilation", params["veg_binary_dilation"]
     )
-    valid_stack = np.logical_and(input_buffer[0], [non_veg_dilated])
+    valid_stack = np.logical_and(input_buffer[0] == 0, [non_veg_dilated])
 
     return valid_stack
 
@@ -83,9 +83,7 @@ def apply_watermask(
     :param dict params: dictionary of arguments (not used but necessary for eoscale)
     :returns: valid_phr (boolean numpy array, True = valid data, False = no data)
     """
-    valid_stack = np.logical_and(
-        input_buffer[0], np.where(input_buffer[1] == 0, True, False)
-    )
+    valid_stack = np.logical_and(input_buffer[0] == 0, input_buffer[1] == 0)
 
     return valid_stack
 
@@ -95,14 +93,14 @@ def get_grid_indexes_from_mask(nb_samples, valid_mask, mask_ground_truth):
     Recover of row and columns indices selected on the valid pixel of the image
 
     :param int nb_samples:
-    :param boolean numpy array valid_mask :
-    :param boolean numpy array mask_ground_truth :
+    :param boolean numpy array valid_mask :  shape (height, width)
+    :param boolean numpy array mask_ground_truth :  shape (height, width)
     :return: tuple of list , row indices and columns indices
     """
     valid_samples = np.logical_and(mask_ground_truth, valid_mask).astype(
         np.uint8
     )
-    _, rows, cols = np.where(valid_samples)
+    rows, cols = np.where(valid_samples)
 
     if 1 <= nb_samples <= len(rows):
         # np.arange(0, len(rows) -1, ...) : to be sure to exclude index len(rows)
@@ -130,6 +128,7 @@ def build_samples(
     """
     # Beware that WSF ground truth contains 0 (non building), 255 (building)
     # but sometimes 1 (invalid pixels ?)
+    validity_mask = (input_buffer[0] == 0)[0]
     mask_building_before_erosion = np.where(
         input_buffer[1] == params["value_classif"], True, False
     )
@@ -144,10 +143,10 @@ def build_samples(
 
     # Retrieve number of pixels for each class
     nb_built_subset = np.count_nonzero(
-        np.logical_and(mask_building, input_buffer[0])
+        np.logical_and(mask_building, validity_mask)
     )
     nb_other_subset = np.count_nonzero(
-        np.logical_and(mask_non_building, input_buffer[0])
+        np.logical_and(mask_non_building, validity_mask)
     )
     # Ratio of pixel class compare to the full image ratio
     urban_ratio = nb_built_subset / params["nb_valid_built_pixels"]
@@ -161,18 +160,18 @@ def build_samples(
     if nb_urban_subsamples > 0:
         # Building samples
         rows_b, cols_b = get_grid_indexes_from_mask(
-            nb_urban_subsamples, input_buffer[0][0], mask_building
+            nb_urban_subsamples, validity_mask, mask_building[0]
         )
 
         if nb_other_subsamples > 0:
             rows_nob, cols_nob = get_grid_indexes_from_mask(
-                nb_other_subsamples, input_buffer[0][0], mask_non_building
+                nb_other_subsamples, validity_mask, mask_non_building[0]
             )
     else:
 
         if nb_other_subsamples > 0:
             rows_nob, cols_nob = get_grid_indexes_from_mask(
-                nb_other_subsamples, input_buffer[0][0], mask_non_building
+                nb_other_subsamples, validity_mask, mask_non_building[0]
             )
 
     rows = np.concatenate((rows_b, rows_nob))
@@ -200,8 +199,9 @@ def rf_prediction(
     :returns: predicted mask (proba)
     """
     im_stack = np.concatenate((input_buffer[2:]), axis=0)
-    nodata_mask = (1 - input_buffer[0]).astype(bool)
-    valid_mask = input_buffer[1].astype(bool)
+    nodata_mask = input_buffer[0] != 0
+    valid_mask = input_buffer[1] == 0
+
     buffer_to_predict = np.transpose(im_stack[:, valid_mask[0]])
     # buffer_to_predict are non NODATA pixels, defined by all the primitives
     # (R-G-B-NIR-NDVI-NDWI-[+ features]
@@ -213,20 +213,19 @@ def rf_prediction(
         res_classif = classifier.classes_.take(np.argmax(proba, axis=1), axis=0)
         res_classif[res_classif == 255] = 1
 
-        prediction = np.zeros(valid_mask.shape)
-        prediction[0][valid_mask[0]] = res_classif
-        prediction[0][nodata_mask[0]] = NODATA_INT8
+        prediction = np.zeros(valid_mask[0].shape)
+        prediction[valid_mask[0]] = res_classif
+        prediction[nodata_mask[0]] = NODATA_INT8
 
-        proba_buildings = np.zeros(valid_mask.shape)
-        proba_buildings[0][valid_mask[0]] = (
+        proba_buildings = np.zeros(valid_mask[0].shape)
+        proba_buildings[valid_mask[0]] = (
             100 * proba[:, 1]
         )  # Proba for class 1 (buildings)
-        proba_buildings[0][nodata_mask[0]] = NODATA_INT8
-
+        proba_buildings[nodata_mask[0]] = NODATA_INT8
     else:
         # corner case : only NO_DATA !
-        prediction = np.full(valid_mask.shape, NODATA_INT8)
-        proba_buildings = np.full(valid_mask.shape, NODATA_INT8)
+        prediction = np.full(valid_mask[0].shape, NODATA_INT8)
+        proba_buildings = np.full(valid_mask[0].shape, NODATA_INT8)
 
     return [proba_buildings, prediction]
 
@@ -338,7 +337,7 @@ def nominal_case_urbanmask(
 
         key_predict = eoexe.n_images_to_m_images_filter(
             inputs=[key_original_valid_stack],
-            image_filter=add_nodata,
+            image_filter=fill_constant_mask,
             filter_parameters={"fill_value": 0},
             generate_output_profiles=eo_utils.single_uint8_profile,
             context_manager=eoscale_manager,
@@ -508,7 +507,7 @@ def samples_train_and_predict(
     return time_random_forest
 
 
-def add_nodata(
+def fill_constant_mask(
     input_buffer: list, input_profiles: list, params: dict
 ) -> np.ndarray:
     """
@@ -519,9 +518,10 @@ def add_nodata(
     :param dict params: dictionary of arguments
     :returns: updated predicted mask (proba)
     """
-    nodata_mask = (1 - input_buffer[0]).astype(bool)
-    proba_buildings = np.full(nodata_mask.shape, params["fill_value"])
-    proba_buildings[nodata_mask] = NODATA_INT8
+
+    proba_buildings = np.where(
+        input_buffer[0] == 0, params["fill_value"], NODATA_INT8
+    )
 
     return proba_buildings
 
@@ -752,9 +752,11 @@ def slurp_urbanmask(
             ]
 
             # Calculation of valid pixels
-            nb_valid_pixels = np.count_nonzero(valid_stack)
+            nb_valid_pixels = len(
+                np.where(valid_stack == 0)[0]
+            )  # np.count_nonzero(valid_stack)
             args.nb_valid_built_pixels = np.count_nonzero(
-                np.logical_and(local_gt, valid_stack)
+                np.logical_and(local_gt, valid_stack == 0)
             )
             args.nb_valid_other_pixels = (
                 nb_valid_pixels - args.nb_valid_built_pixels
@@ -791,7 +793,7 @@ def slurp_urbanmask(
 
                 key_predict = eoexe.n_images_to_m_images_filter(
                     inputs=[key_original_valid_stack],
-                    image_filter=add_nodata,
+                    image_filter=fill_constant_mask,
                     filter_parameters={"fill_value": 100},
                     generate_output_profiles=eo_utils.single_uint8_profile,
                     context_manager=eoscale_manager,
@@ -812,7 +814,7 @@ def slurp_urbanmask(
 
                 key_predict = eoexe.n_images_to_m_images_filter(
                     inputs=[key_original_valid_stack],
-                    image_filter=add_nodata,
+                    image_filter=fill_constant_mask,
                     filter_parameters={"fill_value": 0},
                     generate_output_profiles=eo_utils.single_uint8_profile,
                     context_manager=eoscale_manager,

slurp/masks/vegetationmask.py

@@ -39,7 +39,7 @@
 from slurp.post_process.morphology import apply_morpho
 from slurp.tools import eoscale_utils as eo_utils
 from slurp.tools import utils
-from slurp.tools.constant import NB_CLUSTERS, NODATA_INT8, NODATA_INT16
+from slurp.tools.constant import NB_CLUSTERS, NODATA_INT8
 
 logger = logging.getLogger("slurp")
 
@@ -108,8 +108,8 @@ def segmentation_task(
     segments = compute_segmentation(params, input_buffers[0], input_buffers[1])
 
     # minimum segment is 1, attribute 0 to no_data pixel
-    segments[np.logical_not(input_buffers[2])] = 0
-    segments[np.where(input_buffers[1] == NODATA_INT16)] = 0
+    # valid_stack contains valid pixels (0) and invalid pixels (any other value)
+    segments = np.where(input_buffers[2] == 0, segments, 0)
 
     return segments
 
@@ -533,7 +533,7 @@ def finalize_task(input_buffers: list, input_profiles: list, params: dict):
     final_mask = ts_stats.finalize(input_buffers[0], clustering)
 
     # Add nodata in final_mask (input_buffers[1] : valid mask)
-    final_mask[np.logical_not(input_buffers[1][0])] = NODATA_INT8
+    final_mask = np.where(input_buffers[1][0] == 0, final_mask, NODATA_INT8)
 
     return final_mask
 
@@ -599,7 +599,7 @@ def clean_task(
         im_classif,
     )
 
-    im_classif[np.logical_not(valid_stack)] = NODATA_INT8
+    im_classif = np.where(valid_stack == 0, im_classif, NODATA_INT8)
 
     return im_classif