Merge pull request #41 from quantifyearth/mwd-food-map-random-seed

mdales · web-flow · commit 1cf3b65eb80d · 2025-11-20T08:04:21.000Z
Add seed to random number generator used for hybrid Jung + GEAZ/HYDE habitat map.
diff --git a/.mypy.ini b/.mypy.ini
@@ -1,2 +1,3 @@
 [mypy]
-ignore_missing_imports = True
+ignore_missing_imports = True
+no_namespace_packages = True
diff --git a/.pylintrc b/.pylintrc
@@ -1,3 +1,6 @@
+[MAIN]
+init-hook='import sys; from pathlib import Path; script_dir = Path(".").resolve(); sys.path.extend([str(script_dir / "prepare_layers"), str(script_dir / "prepare_species")])'
+
 [FORMAT]
 max-line-length=120
 
diff --git a/prepare_layers/make_food_current_map.py b/prepare_layers/make_food_current_map.py
@@ -7,7 +7,7 @@
 from pathlib import Path
 from multiprocessing import Manager, Process, cpu_count
 from queue import Queue
-from typing import List, NamedTuple, Optional, Tuple
+from typing import NamedTuple
 
 import numpy as np
 import yirgacheffe as yg
@@ -32,8 +32,11 @@ def process_tile(
     current: yg.layers.RasterLayer,
     pnv: yg.layers.RasterLayer,
     tile: TileInfo,
+    random_seed: int,
 ) -> np.ndarray:
 
+    rng = np.random.default_rng(random_seed)
+
     data = current.read_array(tile.x_position, tile.y_position, tile.width, tile.height)
 
     diffs = [
@@ -61,7 +64,7 @@ def process_tile(
             continue
         required_points = min(required_points, possible_points)
 
-        selected_locations = np.random.choice(
+        selected_locations = rng.choice(
             len(valid_locations[0]),
             size=required_points,
             replace=False
@@ -90,13 +93,14 @@ def process_tile_concurrently(
     with yg.read_raster(current_lvl1_path) as current:
         with yg.read_raster(pnv_path) as pnv:
             while True:
-                tile: Optional[TileInfo] = input_queue.get()
-                if tile is None:
+                job : tuple[TileInfo, int] | None = input_queue.get()
+                if job is None:
                     break
+                tile, seed = job
                 if np.isnan(tile.crop_diff) and np.isnan(tile.pasture_diff):
                     result_queue.put((tile, None))
                 else:
-                    data = process_tile(current, pnv, tile)
+                    data = process_tile(current, pnv, tile, seed)
                     result_queue.put((tile, data.tobytes()))
 
     result_queue.put(None)
@@ -105,7 +109,7 @@ def build_tile_list(
     current_lvl1_path: Path,
     crop_adjustment_path: Path,
     pasture_adjustment_path: Path,
-) -> List[TileInfo]:
+) -> list[TileInfo]:
     tiles = []
     with yg.read_raster(current_lvl1_path) as current:
         current_dimensions = current.window.xsize, current.window.ysize
@@ -151,7 +155,7 @@ def assemble_map(
             band = output._dataset.GetRasterBand(1) # pylint: disable=W0212
 
             while True:
-                result : Optional[Tuple[TileInfo,Optional[bytearray]]] = result_queue.get()
+                result : tuple[TileInfo,bytearray | None] | None = result_queue.get()
                 if result is None:
                     sentinal_count -= 1
                     if sentinal_count == 0:
@@ -174,14 +178,18 @@ def pipeline_source(
     pasture_adjustment_path: Path,
     source_queue: Queue,
     sentinal_count: int,
+    random_seed: int,
 ) -> None:
+    rng = np.random.default_rng(random_seed)
+
     tiles = build_tile_list(
         current_lvl1_path,
         crop_adjustment_path,
         pasture_adjustment_path,
     )
-    for tile in tiles:
-        source_queue.put(tile)
+    seeds = rng.integers(2**63, size=len(tiles))
+    for tile, seed in zip(tiles, seeds):
+        source_queue.put((tile, seed))
     for _ in range(sentinal_count):
         source_queue.put(None)
 
@@ -190,6 +198,7 @@ def make_food_current_map(
     pnv_path: Path,
     crop_adjustment_path: Path,
     pasture_adjustment_path: Path,
+    random_seed: int,
     output_path: Path,
     processes_count: int,
 ) -> None:
@@ -210,7 +219,7 @@ def make_food_current_map(
             pnv_path,
             source_queue,
             result_queue,
-        )) for index in range(processes_count)]
+        )) for _ in range(processes_count)]
         for worker_process in workers:
             worker_process.start()
 
@@ -220,6 +229,7 @@ def make_food_current_map(
             pasture_adjustment_path,
             source_queue,
             processes_count,
+            random_seed,
         ))
         source_worker.start()
 
@@ -273,6 +283,13 @@ def main() -> None:
         help="Path of adjustment for pasture diff",
         dest="pasture_adjustment_path",
     )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        required=True,
+        help="Seed the random number generator",
+        dest="seed",
+    )
     parser.add_argument(
         '--output',
         type=Path,
@@ -295,6 +312,7 @@ def main() -> None:
         args.pnv_path,
         args.crop_adjustment_path,
         args.pasture_adjustment_path,
+        args.seed,
         args.output_path,
         args.processes_count,
     )
diff --git a/prepare_species/common.py b/prepare_species/common.py
@@ -20,6 +20,7 @@
 COLUMNS = [
     "id_no",
     "assessment_id",
+    "assessment_year",
     "season",
     "elevation_lower",
     "elevation_upper",
@@ -170,7 +171,7 @@ def process_habitats(
 
 
 def process_geometries(
-    geometries_data: List[Tuple[int,shapely.Geometry]],
+    geometries_data: List[Tuple[int,str]],
     report: SpeciesReport,
 ) -> Dict[int,shapely.Geometry]:
     if len(geometries_data) == 0:
@@ -226,7 +227,8 @@ def process_and_save(
     output_directory_path: Path,
 ) -> None:
 
-    id_no, assessment_id, elevation_lower, elevation_upper, scientific_name, family_name, threat_code = row
+    id_no, assessment_id, assessment_year, elevation_lower, elevation_upper, scientific_name, \
+        family_name, threat_code = row
 
     seasons = set(geometries.keys()) | set(habitats.keys())
 
@@ -237,6 +239,7 @@ def process_and_save(
             [[
                 id_no,
                 assessment_id,
+                int(assessment_year),
                 SEASON_NAME[1],
                 int(elevation_lower) if elevation_lower is not None else None,
                 int(elevation_upper) if elevation_upper is not None else None,
@@ -299,6 +302,7 @@ def process_and_save(
             [[
                 id_no,
                 assessment_id,
+                int(assessment_year),
                 SEASON_NAME[2],
                 int(elevation_lower) if elevation_lower is not None else None,
                 int(elevation_upper) if elevation_upper is not None else None,
@@ -318,6 +322,7 @@ def process_and_save(
             [[
                 id_no,
                 assessment_id,
+                int(assessment_year),
                 SEASON_NAME[3],
                 int(elevation_lower) if elevation_lower is not None else None,
                 int(elevation_upper) if elevation_upper is not None else None,
diff --git a/prepare_species/extract_species_psql.py b/prepare_species/extract_species_psql.py
@@ -22,6 +22,7 @@
 SELECT
     assessments.sis_taxon_id as id_no,
     assessments.id as assessment_id,
+    DATE_PART('year', assessments.assessment_date) as assessment_year,
     (assessment_supplementary_infos.supplementary_fields->>'ElevationLower.limit')::numeric AS elevation_lower,
     (assessment_supplementary_infos.supplementary_fields->>'ElevationUpper.limit')::numeric AS elevation_upper,
     taxons.scientific_name,
@@ -110,7 +111,16 @@ def process_row(
     register(connection)
     cursor = connection.cursor()
 
-    (id_no, assessment_id, _elevation_lower, _elevation_upper, scientific_name, _family_name, _threat_code) = row
+    (
+        id_no,
+        assessment_id,
+        _assessment_year,
+        _elevation_lower,
+        _elevation_upper,
+        scientific_name,
+        _family_name,
+        _threat_code,
+    ) = row
     report = SpeciesReport(id_no, assessment_id, scientific_name)
     if id_no in overrides:
         report.overriden = True
diff --git a/scripts/generate_food_map.sh b/scripts/generate_food_map.sh
@@ -6,6 +6,7 @@
 # downloaded and generated `current_raw.tif` from the original LIFE pipeline (see run.sh)
 
 set -e
+set -x
 
 if [ -z "${DATADIR}" ]; then
     echo "Please specify $DATADIR"
@@ -67,20 +68,29 @@ if [ ! -d "${DATADIR}"/food/current_layers ]; then
 fi
 
 # Combine GAEZ and HYDE data
-python3 ./prepare_layers/build_gaez_hyde.py --gaez "${DATADIR}"/food/GLCSv11_02_5m.tif \
-                                            --hyde "${DATADIR}"/food/modified_grazing2017AD.asc \
-                                            --output "${DATADIR}"/food/
+if [ ! -f "${DATADIR}"/food/crop.tif ] || [ ! -f "${DATADIR}"/food/pasture.tif ]; then
+    python3 ./prepare_layers/build_gaez_hyde.py --gaez "${DATADIR}"/food/GLCSv11_02_5m.tif \
+                                                --hyde "${DATADIR}"/food/modified_grazing2017AD.asc \
+                                                --output "${DATADIR}"/food/
+fi
 
-python3 ./utils/raster_diff.py --raster_a "${DATADIR}"/food/crop.tif \
-                               --raster_b "${DATADIR}"/food/current_layers/lcc_1401.tif \
-                               --output "${DATADIR}"/food/crop_diff.tif
+if [ ! -f "${DATADIR}"/food/crop_diff.tif ]; then
+    python3 ./utils/raster_diff.py --raster_a "${DATADIR}"/food/crop.tif \
+                                --raster_b "${DATADIR}"/food/current_layers/lcc_1401.tif \
+                                --output "${DATADIR}"/food/crop_diff.tif
+fi
 
-python3 ./utils/raster_diff.py --raster_a "${DATADIR}"/food/pasture.tif \
-                              --raster_b "${DATADIR}"/food/current_layers/lcc_1402.tif \
-                              --output "${DATADIR}"/food/pasture_diff.tif
+if [ ! -f "${DATADIR}"/food/pasture_diff.tif ]; then
+    python3 ./utils/raster_diff.py --raster_a "${DATADIR}"/food/pasture.tif \
+                                --raster_b "${DATADIR}"/food/current_layers/lcc_1402.tif \
+                                --output "${DATADIR}"/food/pasture_diff.tif
+fi
 
-python3 ./prepare_layers/make_food_current_map.py --current_lvl1 "${DATADIR}"/habitat/current_raw.tif \
-                                                  --pnv "${DATADIR}"/habitat/pnv_raw.tif \
-                                                  --crop_diff "${DATADIR}"/food/crop_diff.tif \
-                                                  --pasture_diff "${DATADIR}"/food/pasture_diff.tif \
-                                                  --output "${DATADIR}"/food/current_raw.tif
+if [ ! -f "${DATADIR}"/food/current_raw.tif ]; then
+    python3 ./prepare_layers/make_food_current_map.py --current_lvl1 "${DATADIR}"/habitat/current_raw.tif \
+                                                    --pnv "${DATADIR}"/habitat/pnv_raw.tif \
+                                                    --crop_diff "${DATADIR}"/food/crop_diff.tif \
+                                                    --pasture_diff "${DATADIR}"/food/pasture_diff.tif \
+                                                    --seed 42 \
+                                                    --output "${DATADIR}"/food/current_raw.tif
+fi
diff --git a/tests/test_food_layer.py b/tests/test_food_layer.py
diff --git a/utils/persistencegenerator.py b/utils/persistencegenerator.py
diff --git a/utils/speciesgenerator.py b/utils/speciesgenerator.py
