Update weatherquest configs. Clip bucket fields.

diable O3 timevarying, land periodic cal LAI

Use MODIS LAI climatology (land) and albedo from era5 (bucket) for historical runs. Initialize sea ice with sea ice temperatures from era5.

Fix sea ice (use surface temp for init). Use config outputs consistent with era5 val comparison.

lower ice min temp and revert CL grid

update config

Remove oceananigans and climaocean. Pin GPU compilers.

Add subseasonal calibration pipeline

Update calibration pipeline for weatherquest weekly calibrations on derecho

Add and use 1 day calibration. Fix some bugs in observations map and daily data handling.

add data normalizations and modify noise.

Add land mask, modify noise, add land parameter, write to scratch and deal with HDF errs

Add support and functionality for TransformInversion, Inversion. Precompute large matrices on cpu as inital step

add gravity wave to calibration, move to copies3

add options, add more parameters, log top level run script

fix normalization bug for multi variable option. Add ice albedo to toml.

fix precip units, add logging, start trying 7 day

weekly runs

Add NH average option, remove precip, all specifically of noise by var for NH option, add ensemble spatial plotting

update plots

Add script for analyzing parameters

Author: costachris

.gitignore

@@ -30,6 +30,7 @@ docs/src/generated/
 # Experiments
 !experiments/ClimaEarth/**/Manifest.toml
 !experiments/ClimaCore/**/Manifest.toml
+experiments/calibration/coarse_amip/output*/
 
 # Output
 *output*

Project.toml

@@ -29,14 +29,13 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 ClimaCoreMakie = "908f55d8-4145-4867-9c14-5dad1a479e4d"
 GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6"
 Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
-Oceananigans = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09"
 Poppler_jll = "759029c8-65f6-5f13-b93f-2d7e0a4cb77c"
 Printf = "366bfd08-7218-5ad0-9036-d27ba1f5fe54"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [extensions]
 ClimaCouplerMakieExt = ["CairoMakie", "ClimaCoreMakie", "GeoMakie", "Makie", "Poppler_jll", "Printf"]
-ClimaCouplerOceananigansMakieExt = ["CairoMakie", "ClimaCoreMakie", "GeoMakie", "Makie", "Oceananigans", "Poppler_jll", "Printf"]
+# ClimaCouplerOceananigansMakieExt = ["CairoMakie", "ClimaCoreMakie", "GeoMakie", "Makie", "Oceananigans", "Poppler_jll", "Printf"]  # Removed
 
 [compat]
 ArgParse = "1"

config/atmos_configs/climaatmos_wx_diagedmf.yml

@@ -8,24 +8,25 @@ topography: "Earth"
 rayleigh_sponge: true
 viscous_sponge: true
 hyperdiff: ClimaHyperdiffusion
-divergence_damping_factor: 50.0
+divergence_damping_factor: 20.0
 scalar_hyperdiffusion_coefficient: 0.929738
 vorticity_hyperdiffusion_coefficient: 0.1857
 insolation: "timevarying"
 rad: allskywithclear
 dt_cloud_fraction: "1hours"
 dt_rad: "1hours"
-aerosol_radiation: true
-time_varying_trace_gases: ["CO2", "O3"]
-prescribed_aerosols: ["CB1", "CB2", "DST01", "DST02", "DST03", "DST04", "DST05", "OC1", "OC2", "SO4", "SSLT01", "SSLT02", "SSLT03", "SSLT04", "SSLT05"]
+# time_varying_trace_gases: ["O3"]
+co2_model: fixed
+prescribe_ozone: true
+# prescribed_aerosols: ["CB1", "CB2", "DST01", "OC1", "OC2", "SO4", "SSLT01"]
 turbconv: diagnostic_edmfx
 implicit_diffusion: true
 approximate_linear_solve_iters: 2
 prognostic_tke: true
 edmfx_upwinding: first_order
-h_elem: 16
-z_max: 60000.0
-z_elem: 63
+h_elem: 15
+z_max: 70000.0
+z_elem: 70
 dz_bottom: 30.0
 edmfx_entr_model: "Generalized"
 edmfx_detr_model: "Generalized"
@@ -38,11 +39,12 @@ precip_model: 0M
 moist: equil
 toml: [toml/diagnostic_edmfx_era5_ic.toml]
 
-### 1M Microphysics ###
+# ### 1M Microphysics ###
 # precip_model: "1M"
 # moist: "nonequil"
-# toml: [toml/diagnostic_edmfx_1M.toml]
+# # toml: [toml/diagnostic_edmfx_1M.toml]
+# toml: [toml/longrun_aquaplanet_diagedmf.toml]
 
 cloud_model: "quadrature"
 use_itime: true
-deep_atmosphere: false
+deep_atmosphere: true

config/atmos_configs/climaatmos_wx_progedmf.yml

@@ -8,14 +8,16 @@ topography: "Earth"
 rayleigh_sponge: true
 viscous_sponge: true
 hyperdiff: ClimaHyperdiffusion
-divergence_damping_factor: 50.0
+divergence_damping_factor: 20.0
 scalar_hyperdiffusion_coefficient: 0.929738
 vorticity_hyperdiffusion_coefficient: 0.1857
 insolation: "timevarying"
 rad: allskywithclear
 dt_cloud_fraction: "1hours"
 dt_rad: "1hours"
 # time_varying_trace_gases: ["O3"]
+co2_model: fixed
+prescribe_ozone: true
 # prescribed_aerosols: ["CB1", "CB2", "DST01", "OC1", "OC2", "SO4", "SSLT01"]
 turbconv: "prognostic_edmfx"
 implicit_diffusion: true
@@ -28,8 +30,8 @@ approximate_linear_solve_iters: 2
 prognostic_tke: true
 edmfx_upwinding: first_order
 h_elem: 15
-z_max: 48000.0
-z_elem: 63
+z_max: 70000.0
+z_elem: 70
 dz_bottom: 30.0
 edmfx_entr_model: "Generalized"
 edmfx_detr_model: "Generalized"
@@ -50,4 +52,4 @@ moist: equil
 
 cloud_model: "quadrature"
 use_itime: true
-deep_atmosphere: false
+deep_atmosphere: true

config/subseasonal_configs/wxquest_diagedmf.yml

@@ -1,35 +1,71 @@
 FLOAT_TYPE: "Float32"
 albedo_model: "CouplerAlbedo"
 atmos_config_file: "config/atmos_configs/climaatmos_wx_diagedmf.yml"
-coupler_toml: ["toml/amip_diagedmf.toml"]
+coupler_toml: ["toml/wxquest_diagedmf.toml"]
+strict_params: false
+mode_name: "subseasonal"
+era5_initial_condition_dir: "/glade/campaign/univ/ucit0011/cchristo/wxquest_data/initial_conditions/initial_conditions_v1_dev"
+# era5_initial_condition_dir: "/net/sampo/data1/wxquest_data/initial_conditions/initial_conditions_v1_dev2"
 initial_condition: "WeatherModel"
 checkpoint_dt: "7days"
 dt: "120secs"
 dt_cpl: "120secs"
-dt_cloud_fraction: "1hours"
-dt_rad: "1hours"
+dt_seaice: "120secs"
 energy_check: false
-h_elem: 16
+h_elem: 32
 land_fraction_source: "era5"
 binary_area_fraction: false
-mode_name: "subseasonal"
-era5_initial_condition_dir: "/net/sampo/data1/wxquest_data/initial_conditions"
-land_model: "bucket"
-bucket_albedo_type: "map_temporal"
+
+### gravity waves 
+orographic_gravity_wave: "raw_topo"
+non_orographic_gravity_wave: true
+
+### integrated land ###
+land_model: "integrated"
+
+### bucket land ###
+# land_model: "bucket"
+# bucket_albedo_type: "map_temporal"
+# bucket_albedo_type: "era5"
+# bucket_initial_condition: "/net/sampo/data1/cchristo/WeatherQuest_data/archive_dev4/era5_bucket_processed_20250831_0000.nc"
+
 land_spun_up_ic: false
 land_temperature_anomaly: "nothing"
 radiation_reset_rng_seed: true
-start_date: "20180901"
+# start_date: "19750101"
+start_date: "20230115"
 surface_setup: "PrescribedSurface"
 topo_smoothing: true
 topography: "Earth"
+t_end: "8days"
 netcdf_output_at_levels: true
 output_default_diagnostics: false
 use_coupler_diagnostics: false
 use_land_diagnostics: false
-strict_params: false
-insolation: "timevarying"
 extra_atmos_diagnostics:
-  - short_name: [hfls, hfss, rsus, rlus]
-    period: 1months
+  - short_name: [tas, pr, mslp]
+    period: 1days
     reduction_time: average
+  # - short_name: [tas, ts, pr, mslp, ta, hus, hur, zg, orog, pfull, ua, va, wa, waup]
+  #   period: 1days
+  #   reduction_time: average
+  # - short_name: [hfss, hfls, evspsbl]
+  #   period: 1days
+  #   reduction_time: average
+  ###############
+  # - short_name: [tas, ts, pr, mslp, ta, hus, hur, zg, orog, pfull, ua, va, wa, waup]
+  #   period: 24hours
+  #   reduction_time: average
+  # - short_name: [hfss, hfls, evspsbl]
+  #   period: 24hours
+  #   reduction_time: average
+  # - short_name: [tas, ts, pr, mslp, ta, hus, hur, zg, orog, pfull, ua, va, wa, waup]
+  #   period: 24hours
+  #   reduction_time: average
+  # - short_name: [zg, orog, tas, ta, mslp, pr, ua, va, ts, rhoa, pfull, hur, hus, clw, cl, hfss, hfls, evspsbl, hussfc, rsds, rlds, rlus, rsus, rld, wa, waup]
+  #   period: 24hours
+  #   reduction_time: inst
+# extra_atmos_diagnostics:
+#   - short_name: [zg, orog, tas, ta, mslp, pr, ua, va, ts, rhoa, pfull, hur, hus, clw, cli, clivi, clwvi, cl, arup, tke, hfss, hfls, evspsbl, hussfc, rsds, rlds, rlus, rsus, rld, wa, waup]
+#     period: 3hours
+#     reduction_time: average

config/subseasonal_configs/wxquest_progedmf.yml

@@ -2,12 +2,14 @@ FLOAT_TYPE: "Float32"
 albedo_model: "CouplerAlbedo"
 atmos_config_file: "config/atmos_configs/climaatmos_wx_progedmf.yml"
 coupler_toml: ["toml/amip_progedmf.toml"]
+strict_params: false
 mode_name: "subseasonal"
 era5_initial_condition_dir: "/net/sampo/data1/wxquest_data/initial_conditions"
 initial_condition: "WeatherModel"
 checkpoint_dt: "7days"
 dt: "10secs"
 dt_cpl: "10secs"
+dt_seaice: "10secs"
 energy_check: false
 h_elem: 32
 land_fraction_source: "era5"
@@ -19,19 +21,20 @@ binary_area_fraction: false
 ### bucket land ###
 land_model: "bucket"
 bucket_albedo_type: "map_temporal"
-bucket_initial_condition: "/net/sampo/data1/wxquest_data/initial_conditions/era5_bucket_processed_20250907_0000.nc"
+bucket_initial_condition: "/net/sampo/data1/wxquest_data/initial_conditions/era5_bucket_processed_20250831_0000.nc"
 
 land_spun_up_ic: false
 land_temperature_anomaly: "nothing"
 use_land_diagnostics: true
 radiation_reset_rng_seed: true
-start_date: "20250907"
+start_date: "20250831"
 surface_setup: "PrescribedSurface"
 topo_smoothing: true
 topography: "Earth"
 t_end: "7days"
 netcdf_output_at_levels: true
+use_coupler_diagnostics: true
 extra_atmos_diagnostics:
-  - short_name: [tas, mslp, pr, ua, va, rhoa, pfull, hur, hus, clw, cli, clivi, clwvi, cl, arup, tke]
-    period: 1hours
+  - short_name: [zg, orog, tas, ta, mslp, pr, ua, va, ts, rhoa, pfull, hur, hus, clw, cli, clivi, clwvi, cl, arup, tke, hfss, hfls, evspsbl, hussfc, rsds, rlds, rlus, rsus, rld, wa, waup]
+    period: 3hours
     reduction_time: average

docs/src/input.md

@@ -163,8 +163,8 @@ specific timesteps should be specified, rather than only `dt`.
 | `--land_temperature_anomaly` | String | `"aquaplanet"` | `amip`, `aquaplanet`, `nothing` | Type of temperature anomaly for land model |
 | `--use_land_diagnostics` | Bool | `true` | `true`, `false` | Whether to compute and output land model diagnostics |
 | `--land_spun_up_ic` | Bool | `true` | `true`, `false` | Whether to use integrated land initial conditions from spun up state |
-| `--bucket_albedo_type` | String | `"map_static"` | `map_static`, `function`, `map_temporal` | Access bucket surface albedo information from data file |
-| `--bucket_initial_condition` | String | `""` | Any valid file path | File path for a NetCDF file (read documentation about requirements) |
+| `--bucket_albedo_type` | String | `"map_static"` | `map_static`, `function`, `map_temporal`, `era5` | Access bucket surface albedo information from data file. Use `era5` for ERA5-derived processed albedo files (requires `era5_initial_condition_dir`) |
+| `--bucket_initial_condition` | String | `""` | Any valid file path | File path for a NetCDF file (read documentation about requirements). In subseasonal mode, automatically inferred from `era5_initial_condition_dir` if not specified |
 | `--era5_initial_condition_dir` | String | `nothing` | Any valid directory path | Directory containing ERA5 initial condition files (subseasonal mode). Filenames inferred from `start_date`. Generated with `https://github.com/CliMA/WeatherQuest` |
 | `--land_fraction_source` | String | `"etopo"` | `etopo`, `era5` | Source for land fraction data. `etopo` uses ETOPO-derived landsea_mask artifact (binary), `era5` uses ERA5/IFS land fraction artifact (0.0 - 1.0), which includes large inland seas and lakes. |
 | `--binary_area_fraction` | Bool | `true` | `true`, `false` | Whether to use binary (thresholded) area fractions for land and ice. When true, land fraction > eps becomes 1, and ice fraction > 0.5 becomes 1 |