Count deleted datasets and properties

Author: ktuite

lib/model/query/analytics.js

@@ -742,6 +742,21 @@ FROM audits WHERE action = 'entity.bulk.delete'`);
 const getProjectsWithDescriptions = () => ({ all }) => all(sql`
 select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`);
 
+// Group deleted dataset and properties by project (by tracing actee ID parent)
+// Datasets can be deleted and purged so we can't look at their project IDs.
+// Projects can be archived but not deleted so this query should be fine.
+const countDeletedDatasetAndPropertiesByProject = () => ({ all }) =>
+  all(sql`SELECT
+      projects.id as "projectId",
+      SUM(CASE WHEN audits.action = 'dataset.delete' THEN 1 ELSE 0 END) AS dataset_count,
+      SUM(CASE WHEN audits.action = 'dataset.update.property.delete' THEN 1 ELSE 0 END) AS property_count
+    FROM audits
+    JOIN actees ON audits."acteeId" = actees."id"
+    JOIN projects ON actees."parent" = projects."acteeId"
+      WHERE audits.action IN ('dataset.delete', 'dataset.update.property.delete')
+        AND actees.species = 'dataset'
+    GROUP BY projects.id`);
+
 const projectMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.countUsersPerRole,
   Analytics.countAppUsers,
@@ -760,14 +775,15 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
   Analytics.countSubmissionsComments,
   Analytics.countSubmissionsByUserType,
   Analytics.getProjectsWithDescriptions,
+  Analytics.countDeletedDatasetAndPropertiesByProject,
   Analytics.getDatasets,
   Analytics.getDatasetEvents,
   Analytics.getDatasetProperties,
   Analytics.countEntitiesWithGeometry
 ]).then(([ userRoles, appUsers, deviceIds, pubLinks,
   forms, formGeoRepeats, formsEncrypt, formStates, webforms, reusedIds, entityForms,
   subs, subStates, subEdited, subComments, subUsers,
-  projWithDesc, datasets, datasetEvents, datasetProperties, entitiesWithGeometry ]) => {
+  projWithDesc, deletedDatasetCounts, datasets, datasetEvents, datasetProperties, entitiesWithGeometry ]) => {
   const projects = {};
 
   // users
@@ -935,6 +951,12 @@ const projectMetrics = () => (({ Analytics }) => runSequentially([
     project.other.description_length = row.description_length;
   }
 
+  for (const row of deletedDatasetCounts) {
+    const project = _getProject(projects, row.projectId);
+    project.other.num_datasets_deleted = row.dataset_count;
+    project.other.num_dataset_properties_deleted = row.property_count;
+  }
+
   // Order projects by ID
   const projArray = Object.entries(projects).sort((a, b) => a[0] - b[0]).map((k) => k[1]);
   return projArray;
@@ -1115,5 +1137,6 @@ module.exports = {
   countEntitiesWithGeometry,
   countMultiEntityErrors,
   countEntityBulkDeletes,
-  checkLoginCustomization
+  checkLoginCustomization,
+  countDeletedDatasetAndPropertiesByProject
 };

test/integration/other/analytics-queries.js

@@ -2498,6 +2498,65 @@ describe('analytics task queries', function () {
       const res = await container.Analytics.getProjectsWithDescriptions();
       res.should.eql([{ projectId: 1, description_length: 9 }, { projectId: projWithDesc, description_length: 13 }]);
     }));
+
+    it('should count number of datasets and properties deleted in each project', testService(async (service, { Analytics, Datasets }) => {
+      const asAlice = await service.login('alice');
+
+      // Create a dataset
+      await asAlice.post('/v1/projects/1/datasets')
+        .send({ name: 'trees' })
+        .expect(200);
+
+      await asAlice.post('/v1/projects/1/datasets/trees/properties')
+        .send({ name: 'height' })
+        .expect(200);
+
+      // Create another dataset
+      await asAlice.post('/v1/projects/1/datasets')
+        .send({ name: 'people' })
+        .expect(200);
+
+      // Delete the property
+      await asAlice.delete('/v1/projects/1/datasets/trees/properties/height')
+        .expect(200);
+
+      // Delete the datasets
+      await asAlice.delete('/v1/projects/1/datasets/trees')
+        .expect(200);
+
+      await asAlice.delete('/v1/projects/1/datasets/people')
+        .expect(200);
+
+      // Create another project
+      const newProjectId = await asAlice.post('/v1/projects')
+        .set('Content-Type', 'application/json')
+        .send({ name: 'Test Project' })
+        .expect(200)
+        .then(({ body }) => body.id);
+
+      // Create a dataset in new project to be deleted
+      await asAlice.post(`/v1/projects/${newProjectId}/datasets`)
+        .send({ name: 'people' })
+        .expect(200);
+
+      // Delete the dataset
+      await asAlice.delete(`/v1/projects/${newProjectId}/datasets/people`)
+        .expect(200);
+
+      // Force purge dataset in new project
+      await Datasets.purge(true, newProjectId);
+
+      const res = await Analytics.countDeletedDatasetAndPropertiesByProject();
+      for (const row in res) {
+        if (row.projectId === 1) {
+          row.dataset_count.should.equal(2);
+          row.property_count.should.equal(1);
+        } else if (row.projectId === newProjectId) {
+          row.dataset_count.should.equal(1);
+          row.property_count.should.equal(0);
+        }
+      }
+    }));
   });
 
   describe('combined analytics', () => {
@@ -2833,9 +2892,27 @@ describe('analytics task queries', function () {
     }));
 
     it('should fill in all project.other queries', testService(async (service, container) => {
-      await service.login('alice', (asAlice) =>
-        asAlice.patch('/v1/projects/1')
-          .send({ description: 'test desc' }));
+      const asAlice = await service.login('alice');
+
+      await asAlice.patch('/v1/projects/1')
+        .send({ description: 'test desc' });
+
+      // delete datasets and properties
+      await asAlice.post('/v1/projects/1/datasets')
+        .send({ name: 'trees' })
+        .expect(200);
+
+      await asAlice.post('/v1/projects/1/datasets/trees/properties')
+        .send({ name: 'height' })
+        .expect(200);
+
+      // Delete the property
+      await asAlice.delete('/v1/projects/1/datasets/trees/properties/height')
+        .expect(200);
+
+      // Delete the dataset
+      await asAlice.delete('/v1/projects/1/datasets/trees')
+        .expect(200);
 
       const res = await container.Analytics.previewMetrics();