Merge pull request #791 from motech-implementations/obd_file_bifurcation

Bifurcation of obd files to separate priority subscribers to enable b…

Author: gudipatiharitha

imi/src/main/java/org/motechproject/nms/imi/service/TargetFileService.java

@@ -4,6 +4,8 @@
 import org.motechproject.nms.imi.web.contract.FileProcessedStatusRequest;
 import org.motechproject.nms.kilkari.domain.SubscriptionOrigin;
 
+import java.util.HashMap;
+
 /**
  * Creating the targetFile: a csv file containing all the phone numbers to be called by the IVR system
  */
@@ -12,9 +14,7 @@ public interface TargetFileService {
      * Probably only to be called by an IT. This service's constructor sets the repeating
      * nms.obd.generate_target_file MOTECH event which triggers the daily generation of the targetFile.
      */
-    TargetFileNotification generateTargetFile();
-
-
+    HashMap<String, TargetFileNotification> generateTargetFile(boolean split);
     /**
      * The IVR system invoked the obdFileProcessedStatusNotification http endpoint signalling the completion of the
      * processing of the targetFile we generated
@@ -32,4 +32,6 @@ public interface TargetFileService {
      * @return the IMI ServiceID
      */
     String serviceIdFromOrigin(boolean freshCall, SubscriptionOrigin origin);
+
+    String serviceIdFromOriginJh(boolean freshCall, SubscriptionOrigin origin);
 }

imi/src/main/java/org/motechproject/nms/imi/service/impl/TargetFileServiceImpl.java

@@ -35,6 +35,7 @@
 import org.springframework.web.bind.annotation.ResponseStatus;
 
 import javax.servlet.http.HttpServletRequest;
+import java.util.HashMap;
 import java.util.regex.Pattern;
 
 /**
@@ -48,6 +49,7 @@ public class ImiController {
     public static final String INVALID_STATUS_ENUM = "Can not construct instance of " +
             "org.motechproject.nms.imi.domain.FileProcessedStatus from String value";
     public static final Pattern TARGET_FILENAME_PATTERN = Pattern.compile("OBD_NMS_[0-9]{14}\\.csv");
+    public static final Pattern TARGET_FILENAME_PATTERN_JH = Pattern.compile("OBD_NMS_[0-9]{14}JH\\.csv");
     public static final String IVR_INTERACTION_LOG = "IVR INTERACTION: %s";
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ImiController.class);
@@ -59,6 +61,8 @@ public class ImiController {
     private FileAuditRecordDataService fileAuditRecordDataService;
     private AlertService alertService;
 
+    public static final String generateJhFile = "imi.obd_bifurcate";
+
 
     @Autowired
     public ImiController(SettingsFacade settingsFacade, CdrFileService cdrFileService, AlertService alertService,
@@ -88,7 +92,8 @@ private static boolean validateFieldPresent(StringBuilder errors, String fieldNa
     // verify the passed targetFileName is valid
     private static boolean validateTargetFileName(StringBuilder errors, String targetFileName) {
         if (validateFieldPresent(errors, "fileName", targetFileName)) {
-            if (TARGET_FILENAME_PATTERN.matcher(targetFileName).matches()) {
+            if (TARGET_FILENAME_PATTERN.matcher(targetFileName).matches() ||
+                    TARGET_FILENAME_PATTERN_JH.matcher(targetFileName).matches() ) {
                 return true;
             } else {
                 errors.append(String.format(INVALID, "fileName"));
@@ -148,13 +153,13 @@ public String generateTargetFile() {
 
         LOGGER.debug("/generateTargetFile (GET)");
         try {
-            TargetFileNotification tfn = targetFileService.generateTargetFile();
-            LOGGER.debug("targetFileService.generateTargetFile() done");
+                HashMap<String, TargetFileNotification> tfn = targetFileService.generateTargetFile(Boolean.parseBoolean(settingsFacade.getProperty(generateJhFile)));
+                LOGGER.debug("targetFileService.generateTargetFile() done");
 
-            return tfn == null ? "null" : tfn.getFileName();
-        } catch(Exception e) {
-            LOGGER.error(e.getMessage(),e);
-            throw e;
+                return tfn == null ? "null" : tfn.values().toString();
+            }catch(Exception e){
+        LOGGER.error(e.getMessage(), e);
+        throw e;
         }
 
     }

imi/src/main/java/org/motechproject/nms/imi/web/ImiController.java

@@ -5,10 +5,10 @@ imi.target_file_time=05:00
 imi.target_file_sec_interval=86400
 
 #local directory where target files should be generated
-imi.local_obd_dir=obd-files-local
+imi.local_obd_dir=/usr/local/imi/obd
 
 #remote directory where target files should be copied to
-imi.remote_obd_dir=obd-files-remote
+imi.remote_obd_dir=/usr/local/imi/obd-remote
 
 #local directory where CDR files should be copied to
 imi.local_cdr_dir=cdr-files-local
@@ -44,6 +44,11 @@ imi.fresh_no_check_dnd=Retryonroute1noDND_1444132196827
 imi.retry_check_dnd=Retryonroute2noDND_1444132213332
 imi.retry_no_check_dnd= Retryonroute2noDND_1444132213332
 
+imi.fresh_check_dnd_jh= jhrRetryonroute1noDND_1560947525746
+imi.fresh_no_check_dnd_jh=jhrRetryonroute1noDND_1560947525746
+imi.retry_check_dnd_jh=jhrRetryonroute2noDND_1560947563925
+imi.retry_no_check_dnd_jh= jhrRetryonroute2noDND_1560947563925
+
 #maximum number of errors allowed in a CDR file after which all errors are ignored so as to not overwhelm the
 #audit table & the tomcat log file
 imi.max_cdr_error_count=100
@@ -85,4 +90,7 @@ imi.distributed_csr_processing=false
 
 # CSR chunk size - any number {n} greater than one will enable chunk processing, where sets (or chunks) of {n} CSRs are
 # distributed for processing, regardless of the value of imi.distributed_csr_processing
-imi.csr_chunk_size=1000
+imi.csr_chunk_size=1000
+
+#set to false if bifurcation of obd files is not required
+imi.obd_bifurcate=true

imi/src/main/resources/imi.properties

@@ -150,6 +150,7 @@ Subscription createSubscription(Subscriber subscriber, long callingNumber, Langu
      */
     List<Subscription> findActiveSubscriptionsForDay(DayOfTheWeek dayOfTheWeek, long offset, int rowCount);
 
+    List<String> findJhSubscriptionIds();
 
     /**
      * Get the list of pending subscriptions that starts after the specified date.

kilkari/src/main/java/org/motechproject/nms/kilkari/service/SubscriptionService.java

@@ -1011,6 +1011,31 @@ public List<Subscription> execute(Query query) {
         return subscriptions;
     }
 
+    @Override
+    public List<String> findJhSubscriptionIds() {
+        SqlQueryExecution<List<String>> queryExecution = new SqlQueryExecution<List<String>>() {
+            @Override
+            public String getSqlQuery() {
+                String query = "select subscriptionId from (select subscriptionId from nms_mcts_mothers a LEFT JOIN " +
+                        "nms_subscribers b on b.mother_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id where rchId like 'JH%' " +
+                        "and subscriptionPack_id_OID = 1 UNION ALL select subscriptionId from nms_mcts_children a LEFT JOIN " +
+                        "nms_subscribers b on b.child_id_oid=a.id LEFT JOIN nms_subscriptions c on c.subscriber_id_oid = b.id " +
+                        "where rchId like 'JH%' and subscriptionPack_id_OID = 2) as  a;";
+                LOGGER.debug(KilkariConstants.SQL_QUERY_LOG, query);
+                return query;
+            }
+
+            @Override
+            public List<String> execute(Query query) {
+                List<String> ids = (List<String>) query.execute();
+                return ids;
+            }
+        };
+        List<String> subscriptionIds = subscriptionDataService.executeSQLQuery(queryExecution);
+        return subscriptionIds;
+    }
+
+
 
     public List<Subscription> findPendingSubscriptionsFromDate(DateTime startDate, int page, int pageSize) {
         return subscriptionDataService.findByStatusAndStartDate(SubscriptionStatus.PENDING_ACTIVATION, startDate, new QueryParams(page, pageSize));

kilkari/src/main/java/org/motechproject/nms/kilkari/service/impl/SubscriptionServiceImpl.java

@@ -52,6 +52,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.security.NoSuchAlgorithmException;
+import java.util.HashMap;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -100,6 +101,8 @@ public class ImiControllerCdrBundleIT extends BasePaxIT {
     private String localObdDirBackup;
     private String remoteObdDirBackup;
     private CdrHelper helper;
+    private static final String Jh = "JH";
+    private static final String non_Jh = "NON-JH";
 
 
     @Before
@@ -511,12 +514,12 @@ public void verifyTargetFileNameRoundTrip() throws IOException, InterruptedExcep
                 SubscriptionOrigin.MCTS_IMPORT);
         transactionManager.commit(status);
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
-        assertNotNull(tfn);
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
+        assertNotNull(tfn.get(non_Jh));
 
         CdrHelper cdrHelper = new CdrHelper(settingsService, subscriberService,subscriptionService, subscriberDataService,
                 subscriptionPackDataService, languageDataService, languageService, circleDataService, stateDataService,
-                districtDataService, fileAuditRecordDataService, districtService, tfn.getFileName());
+                districtDataService, fileAuditRecordDataService, districtService, tfn.get(non_Jh).getFileName());
 
         helper.makeCdrs(1, 0, 0, 0);
         File remoteCdrFile = helper.makeRemoteCdrFile();

testing/src/test/java/org/motechproject/nms/testing/it/imi/ImiControllerCdrBundleIT.java

@@ -55,6 +55,7 @@
 import java.nio.file.Files;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -70,6 +71,8 @@ public class TargetFileServiceBundleIT extends BasePaxIT {
 
     private static final String LOCAL_OBD_DIR = "imi.local_obd_dir";
     private static final String REMOTE_OBD_DIR = "imi.remote_obd_dir";
+    private static final String Jh = "JH";
+    private static final String non_Jh = "NON-JH";
 
 
     private String localObdDirBackup;
@@ -175,9 +178,9 @@ public void createLargeFile() {
             ));
         }
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
-        assertNotNull(tfn);
-        getLogger().debug("Generated {}", tfn.getFileName());
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
+        assertNotNull(tfn.get(non_Jh));
+        getLogger().debug("Generated {}", tfn.values().toString());
     }
 
     // To check that target file should contain correct weekID according to LMP of the subscriber.
@@ -198,10 +201,10 @@ public void verifyFT151() throws NoSuchAlgorithmException, IOException {
         List<String> contents = new ArrayList<>();
         String line;
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
 
         File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
-        File targetFile = new File(targetDir, tfn.getFileName());
+        File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
         int recordCount = 0;
         boolean header = true;
         try (InputStream is = Files.newInputStream(targetFile.toPath());
@@ -218,8 +221,8 @@ public void verifyFT151() throws NoSuchAlgorithmException, IOException {
 
         String checksum = ChecksumHelper.checksum(targetFile);
 
-        assertEquals((int)tfn.getRecordsCount(), recordCount);
-        assertEquals(tfn.getChecksum(), checksum);
+        assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
+        assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
         assertTrue("w6_1".equals(contents.get(0)));
         assertEquals(1, recordCount);
     }
@@ -242,10 +245,10 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
         List<String> contents = new ArrayList<>();
         String line;
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
 
         File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
-        File targetFile = new File(targetDir, tfn.getFileName());
+        File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
         int recordCount = 0;
         boolean header = true;
         try (InputStream is = Files.newInputStream(targetFile.toPath());
@@ -261,8 +264,8 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
         }
 
         String checksum = ChecksumHelper.checksum(targetFile);
-        assertEquals((int)tfn.getRecordsCount(), recordCount);
-        assertEquals(tfn.getChecksum(), checksum);
+        assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
+        assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
         assertTrue("w5_1".equals(contents.get(0)));
 
         //update the date of birth of the subscriber
@@ -272,9 +275,9 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
         subscriberService.updateStartDate(subscriber2);
 
         // again generate the target file to check correct weekId is picked after DOB is changed.
-        tfn = targetFileService.generateTargetFile();
+        tfn = targetFileService.generateTargetFile(false);
         contents.clear();
-        targetFile = new File(targetDir, tfn.getFileName());
+        targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
         recordCount = 0;
         header = true;
         try (InputStream is = Files.newInputStream(targetFile.toPath());
@@ -289,8 +292,8 @@ public void verifyFT152() throws NoSuchAlgorithmException, IOException {
             }
         }
         checksum = ChecksumHelper.checksum(targetFile);
-        assertEquals((int)tfn.getRecordsCount(), recordCount);
-        assertEquals(tfn.getChecksum(), checksum);
+        assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
+        assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
         assertTrue("w4_1".equals(contents.get(0)));
         assertEquals(1, recordCount);
     }
@@ -313,10 +316,10 @@ public void verifyFT190() throws NoSuchAlgorithmException, IOException {
         List<String> contents = new ArrayList<>();
         String line;
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
 
         File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
-        File targetFile = new File(targetDir, tfn.getFileName());
+        File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
         int recordCount = 0;
         boolean header = true;
         try (InputStream is = Files.newInputStream(targetFile.toPath());
@@ -332,8 +335,8 @@ public void verifyFT190() throws NoSuchAlgorithmException, IOException {
         }
 
         String checksum = ChecksumHelper.checksum(targetFile);
-        assertEquals((int)tfn.getRecordsCount(), recordCount);
-        assertEquals(tfn.getChecksum(), checksum);
+        assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
+        assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
         assertTrue("w1_1.wav".equals(contents.get(0)));
     }
 
@@ -355,10 +358,10 @@ public void verifyFT191() throws NoSuchAlgorithmException, IOException {
         List<String> contents = new ArrayList<>();
         String line;
 
-        TargetFileNotification tfn = targetFileService.generateTargetFile();
+        HashMap<String,TargetFileNotification> tfn = targetFileService.generateTargetFile(false);
 
         File targetDir = new File(settingsService.getSettingsFacade().getProperty("imi.local_obd_dir"));
-        File targetFile = new File(targetDir, tfn.getFileName());
+        File targetFile = new File(targetDir, tfn.get(non_Jh).getFileName());
         int recordCount = 0;
         boolean header = true;
         try (InputStream is = Files.newInputStream(targetFile.toPath());
@@ -374,8 +377,8 @@ public void verifyFT191() throws NoSuchAlgorithmException, IOException {
         }
 
         String checksum = ChecksumHelper.checksum(targetFile);
-        assertEquals((int)tfn.getRecordsCount(), recordCount);
-        assertEquals(tfn.getChecksum(), checksum);
+        assertEquals((int)tfn.get(non_Jh).getRecordsCount(), recordCount);
+        assertEquals(tfn.get(non_Jh).getChecksum(), checksum);
         assertTrue("w1_1.wav".equals(contents.get(0)));
 
     }
@@ -395,9 +398,9 @@ public void testChecksumsVaryWithFileContent() throws NoSuchAlgorithmException,
 
         transactionManager.commit(status);
 
-        TargetFileNotification tfn1 = targetFileService.generateTargetFile();
-        assertNotNull(tfn1);
-        getLogger().debug(tfn1.toString());
+        HashMap<String,TargetFileNotification> tfn1 = targetFileService.generateTargetFile(false);
+        assertNotNull(tfn1.get(non_Jh));
+        getLogger().debug(tfn1.get(non_Jh).toString());
 
         // Sleep two seconds so the file names are different
         Thread.sleep(2000L);
@@ -413,11 +416,11 @@ public void testChecksumsVaryWithFileContent() throws NoSuchAlgorithmException,
 
         transactionManager.commit(status);
 
-        TargetFileNotification tfn2 = targetFileService.generateTargetFile();
-        assertNotNull(tfn2);
+        HashMap<String,TargetFileNotification> tfn2 = targetFileService.generateTargetFile(false);
+        assertNotNull(tfn2.get(non_Jh));
         getLogger().debug(tfn2.toString());
 
-        assertFalse(tfn1.getChecksum().equals(tfn2.getChecksum()));
+        assertFalse(tfn1.get(non_Jh).getChecksum().equals(tfn2.get(non_Jh).getChecksum()));
     }
 
 }