Merge pull request #3854 from LiteFarmOrg/integration

Release 3.8.0

Author: kathyavini

packages/api/.env.default

@@ -97,3 +97,15 @@ API_HOST=localhost
 OOO_MESSAGE_ENABLED=
 # Format YYYY/MM/DD
 OOO_END_DATE=
+
+
+# Debugging variables for working on specific features
+
+# Speed up the notifications timer (cycle through 1 hour every minute)
+# MOCK_TIMER=true
+
+# Specify which mock implementation to use for addon partners in development (GET irrigation prescription list)
+# MOCK_ADDON_PARTNER=ESCI
+
+# Mock Ensemble irrigation prescription details
+# USE_IP_MOCK_DETAILS=true

packages/api/db/migration/20250707200337_cleanup_legacy_ensemble_farm_addons.js

@@ -0,0 +1,71 @@
+/*
+ *  Copyright 2025 LiteFarm.org
+ *  This file is part of LiteFarm.
+ *
+ *  LiteFarm is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  LiteFarm is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const up = async (knex) => {
+  // Rows to be inserted to 'migration_deletion_logs' table
+  // When rolling back, rows will be restored backwards
+  const rowsToLog = [];
+  const addToLogs = (rows, tableName) => {
+    rows.forEach((row) => {
+      rowsToLog.push({ table_name: tableName, data: row });
+    });
+  };
+
+  const legacyEnsembleFarmAddons = await knex('farm_addon')
+    .where('addon_partner_id', 1)
+    .whereNull('org_pk');
+
+  addToLogs(legacyEnsembleFarmAddons, 'farm_addon');
+
+  // Delete logged records
+  await knex('farm_addon').where('addon_partner_id', 1).whereNull('org_pk').del();
+
+  if (rowsToLog.length) {
+    await knex('migration_deletion_logs').insert(
+      rowsToLog.map((row) => ({
+        migration_name: 'cleanup_legacy_ensemble_farm_addons',
+        ...row,
+      })),
+    );
+  }
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const down = async (knex) => {
+  // Restore deleted rows from migration_deletion_logs
+  const rows = await knex('migration_deletion_logs')
+    .where('migration_name', 'cleanup_legacy_ensemble_farm_addons')
+    .orderBy('id', 'desc');
+
+  for (const { table_name, data } of rows) {
+    try {
+      await knex(table_name).insert(data);
+    } catch (err) {
+      console.error(`Failed to restore row in ${table_name}:`, err);
+      throw err;
+    }
+  }
+
+  await knex('migration_deletion_logs')
+    .where('migration_name', 'cleanup_legacy_ensemble_farm_addons')
+    .del();
+};

packages/api/db/migration/20250709183357_add_columns_to_task.js

@@ -0,0 +1,36 @@
+/*
+ *  Copyright 2025 LiteFarm.org
+ *  This file is part of LiteFarm.
+ *
+ *  LiteFarm is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  LiteFarm is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const up = async (knex) => {
+  await knex.schema.alterTable('task', (table) => {
+    table.dateTime('revision_date');
+    table.string('revised_by_user_id').references('user_id').inTable('users');
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export const down = async (knex) => {
+  await knex.schema.table('task', (table) => {
+    table.dropColumn('revision_date');
+    table.dropColumn('revised_by_user_id');
+  });
+};

packages/api/db/user_data_requests/02-export_user_data.sql

@@ -0,0 +1,250 @@
+/*
+ *  Copyright 2025 LiteFarm.org
+ *  This file is part of LiteFarm.
+ *
+ *  LiteFarm is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  LiteFarm is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ *  GNU General Public License for more details, see <https://www.gnu.org/licenses/>.
+ */
+
+-- @description: Return new JSONB = base + (key→payload) if payload IS NOT NULL. A helper function to build the export data object
+CREATE OR REPLACE FUNCTION jsonb_merge(
+  base    JSONB,
+  key     TEXT,
+  payload JSONB
+) RETURNS JSONB
+  LANGUAGE sql AS $$
+    SELECT CASE
+      WHEN payload IS NULL THEN base
+      ELSE base || jsonb_build_object(key, payload)
+    END;
+$$;
+
+-- @description: Build a JSONB payload containing every user- and farm-scoped record for export
+CREATE OR REPLACE FUNCTION export_user_data()
+RETURNS JSONB
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    target_user_id            VARCHAR := 'user_id_here';
+    user_data                 user_data_collections;
+    farm_data                 farm_data_collections;
+    farm_count                INTEGER;
+
+    task_tables               RECORD;
+    task_prod_tables          RECORD;
+    pmp_tables                RECORD;
+    mp_tables                 RECORD;
+    plan_repetition_tables    RECORD;
+    figure_tables             RECORD;
+    animal_tables             RECORD;
+    animal_batch_tables       RECORD;
+    location_tables           RECORD;
+    secondary_tables          RECORD;
+    farm_tables               RECORD;
+    created_by_tables         RECORD;
+    user_tables               RECORD;
+
+    farm_json                 JSONB;
+    table_data                JSONB;
+    full_export               JSONB := '{}'::jsonb;
+BEGIN
+user_data := get_user_data(target_user_id);
+
+FOREACH farm_data IN ARRAY user_data.farms LOOP
+    farm_json := '{}'::jsonb;
+
+    -- ==== TASK PRODUCT-SCOPED TABLES ====
+    FOR task_prod_tables IN SELECT * FROM get_task_product_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(t)) FROM %I t WHERE t.task_products_id = ANY($1)',
+        task_prod_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.task_products_ids;
+
+      farm_json := jsonb_merge(farm_json, task_prod_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== TASK‐SCOPED TABLES ====
+    FOR task_tables IN SELECT * FROM get_task_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(t)) FROM %I t WHERE t.task_id = ANY($1)',
+        task_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.task_ids;
+
+      farm_json := jsonb_merge(farm_json, task_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== PLANTING MANAGEMENT PLAN TABLES ====
+    FOR pmp_tables IN SELECT * FROM get_pmp_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(p)) FROM %I p ' ||
+        'WHERE p.planting_management_plan_id = ANY($1)',
+        pmp_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.pmp_ids;
+
+      farm_json := jsonb_merge(farm_json, pmp_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== MANAGEMENT PLAN TABLES ====
+    FOR mp_tables IN SELECT * FROM get_management_plan_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(m)) FROM %I m ' ||
+        'WHERE m.management_plan_id = ANY($1)',
+        mp_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.management_plan_ids;
+
+      farm_json := jsonb_merge(farm_json, mp_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== MANAGEMENT PLAN GROUP TABLES ====
+    FOR plan_repetition_tables IN SELECT * FROM get_mp_repetition_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(r)) FROM %I r ' ||
+        'WHERE r.management_plan_group_id = ANY($1)',
+        plan_repetition_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.management_plan_group_ids;
+
+      farm_json := jsonb_merge(farm_json, plan_repetition_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== ANIMALS ====
+    FOR animal_tables IN SELECT * FROM get_animal_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(a)) FROM %I a ' ||
+        'WHERE a.animal_id = ANY($1)',
+        animal_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.animal_ids;
+
+      farm_json := jsonb_merge(farm_json, animal_tables.table_name, table_data);
+    END LOOP;
+
+    FOR animal_batch_tables IN SELECT * FROM get_animal_batch_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(ab)) FROM %I ab ' ||
+        'WHERE ab.animal_batch_id = ANY($1)',
+        animal_batch_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.animal_batch_ids;
+
+      farm_json := jsonb_merge(farm_json, animal_batch_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== LOCATIONS ====
+    FOR figure_tables IN SELECT * FROM get_figure_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(f)) FROM %I f ' ||
+        'WHERE f.figure_id = ANY($1)',
+        figure_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.figure_ids;
+
+      farm_json := jsonb_merge(farm_json, figure_tables.table_name, table_data);
+    END LOOP;
+
+    FOR location_tables IN SELECT * FROM get_location_export_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(l)) FROM %I l ' ||
+        'WHERE l.location_id = ANY($1)',
+        location_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.location_ids;
+
+      farm_json := jsonb_merge(farm_json, location_tables.table_name, table_data);
+    END LOOP;
+
+    -- ==== TABLES REQUIRING SECONDARY JOIN FOR FARM_ID ====
+    FOR secondary_tables IN SELECT * FROM get_secondary_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(c)) ' ||
+        'FROM %I c JOIN %I p ON c.%I = p.%I ' ||
+        'WHERE p.farm_id = $1',
+        secondary_tables.child_table,
+        secondary_tables.join_table,
+        secondary_tables.join_key,
+        secondary_tables.join_key
+      )
+      INTO table_data
+      USING farm_data.farm_id;
+
+      farm_json := jsonb_merge(farm_json, secondary_tables.child_table, table_data);
+    END LOOP;
+
+    -- ==== REMAINING FARM-SCOPED TABLES  ====
+    FOR farm_tables IN SELECT * FROM get_farm_export_tables() LOOP
+      EXECUTE format(
+        'SELECT json_agg(to_jsonb(fm)) ' ||
+        'FROM %I fm WHERE fm.farm_id = $1',
+        farm_tables.table_name
+      )
+      INTO table_data
+      USING farm_data.farm_id;
+
+      farm_json := jsonb_merge(farm_json, farm_tables.table_name, table_data);
+    END LOOP;
+
+  -- append this farm into the "farms" array
+  full_export := jsonb_merge(
+    full_export,
+    'farms', 
+    COALESCE(full_export->'farms','[]'::jsonb) || farm_json
+  );
+
+END LOOP;
+
+-- ==== USER DATA NOT SPECIFIC TO FARM ====
+FOR created_by_tables IN SELECT * FROM get_created_by_user_tables() LOOP
+    EXECUTE format(
+      'SELECT json_agg(to_jsonb(cb)) ' ||
+      'FROM %I cb WHERE cb.created_by_user_id = $1',
+      created_by_tables.table_name
+    )
+    INTO table_data
+    USING target_user_id;
+
+    full_export := jsonb_merge(full_export, created_by_tables.table_name, table_data);
+END LOOP;
+
+FOR user_tables IN SELECT * FROM get_user_scoped_tables() LOOP
+    EXECUTE format(
+      'SELECT json_agg(to_jsonb(u)) ' ||
+      'FROM %I u WHERE u.user_id = $1',
+      user_tables.table_name
+    )
+    INTO table_data
+    USING target_user_id;
+
+    full_export := jsonb_merge(full_export, user_tables.table_name, table_data);
+END LOOP;
+
+-- Format export object
+RETURN jsonb_build_object(
+  'user_id',       target_user_id,
+  'exported_at',   to_char(now(), 'YYYY-MM-DD"T"HH24:MI:SS"Z"'),
+  'export_type',   'all_user_data',
+  'data',          full_export
+);
+
+END $$;
+
+SELECT export_user_data() as export;