Skip to content

Commit 66747ef

Browse files
ndg63276Mark Williamsgfrn
authored
LIMS-2181: Improve speed of queueing and editing 1000s of samples (#1070)
* LIMS-2181: Improve speed of queueing and editing 1000s of samples * LIMS-2181: Do in chunks of 500 * LIMS-2181: Remove very slow validation * Update client/src/js/modules/imaging/views/queuecontainer.js Co-authored-by: Guilherme Francisco <guilherme.de-freitas@diamond.ac.uk> * LIMS-2181: Use placeholders rather than directly using ids --------- Co-authored-by: Mark Williams <mark.williams@diamond.ac.uk> Co-authored-by: Guilherme Francisco <guilherme.de-freitas@diamond.ac.uk>
1 parent 614b7d3 commit 66747ef

2 files changed

Lines changed: 132 additions & 30 deletions

File tree

‎api/src/Page/Sample.php‎

Lines changed: 103 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ class Sample extends Page
177177
array('/sub/:ssid', 'get', '_get_sub_sample'),
178178
array('/sub/:ssid', 'patch', '_update_sub_sample'),
179179
array('/sub/:ssid', 'put', '_update_sub_sample_full'),
180+
array('/sub/cid/:cid', 'post', '_update_container_sub_samples'),
180181
array('/sub', 'post', '_add_sub_sample'),
181182
array('/sub/:ssid', 'delete', '_delete_sub_sample'),
182183
array('/sub/queue/cid/:cid', 'post', '_queue_all_sub_samples'),
@@ -638,13 +639,49 @@ function _queue_all_sub_samples()
638639

639640
$this->db->wait_rep_sync(false);
640641

642+
if (!sizeof($subs))
643+
$this->_error('No subsamples found');
644+
645+
$sub_ids = array_column($subs, 'BLSUBSAMPLEID');
646+
$chunks = array_chunk($sub_ids, 500);
641647
$ret = array();
642-
foreach ($subs as $sub) {
643-
array_push($ret, array(
644-
'BLSUBSAMPLEID' => $sub['BLSUBSAMPLEID'],
645-
'CONTAINERQUEUESAMPLEID' => $this->_do_pre_q_sample(array('BLSUBSAMPLEID' => $sub['BLSUBSAMPLEID']))));
648+
649+
if ($this->has_arg('UNQUEUE')) {
650+
foreach ($chunks as $chunk) {
651+
$placeholders = implode(',', array_fill(0, count($chunk), '?'));
652+
$this->db->pq("DELETE FROM containerqueuesample
653+
WHERE containerqueueid IS NULL
654+
AND blsubsampleid IN ($placeholders)", $chunk);
655+
}
656+
657+
$ret = array_map(function($id) {
658+
return array('BLSUBSAMPLEID' => $id, 'CONTAINERQUEUESAMPLEID' => null);
659+
}, $sub_ids);
660+
661+
} else {
662+
foreach ($chunks as $chunk) {
663+
$placeholders = implode(',', array_fill(0, count($chunk), '?'));
664+
$this->db->pq("INSERT INTO containerqueuesample (blsubsampleid)
665+
SELECT ss.blsubsampleid
666+
FROM blsubsample ss
667+
INNER JOIN blsample s ON s.blsampleid = ss.blsampleid
668+
INNER JOIN container c ON c.containerid = s.containerid
669+
INNER JOIN dewar d ON d.dewarid = c.dewarid
670+
INNER JOIN shipping sh ON sh.shippingid = d.shippingid
671+
WHERE sh.proposalid = ?
672+
AND ss.blsubsampleid IN ($placeholders)",
673+
array_merge(array($this->proposalid), $chunk));
674+
675+
$chunk_ret = $this->db->pq("SELECT blsubsampleid, containerqueuesampleid
676+
FROM containerqueuesample
677+
WHERE blsubsampleid IN ($placeholders)
678+
AND containerqueueid IS NULL", $chunk);
679+
680+
$ret = array_merge($ret, $chunk_ret);
681+
}
646682
}
647683
$this->_output($ret);
684+
648685
}
649686

650687
function _sub_samples()
@@ -960,6 +997,68 @@ function _update_sub_sample_full()
960997
}
961998

962999

1000+
function _update_container_sub_samples()
1001+
{
1002+
if (!$this->arg('cid'))
1003+
$this->_error('No container specified');
1004+
1005+
if (!$this->arg('EXPERIMENTKIND'))
1006+
$this->_error('No experiment kind provided');
1007+
1008+
$rows = $this->db->pq("SELECT DISTINCT ss.diffractionplanid
1009+
FROM blsubsample ss
1010+
INNER JOIN blsample s ON s.blsampleid = ss.blsampleid
1011+
INNER JOIN diffractionplan dp ON dp.diffractionplanid = ss.diffractionplanid
1012+
INNER JOIN containerqueuesample cqs ON cqs.blsubsampleid = ss.blsubsampleid
1013+
WHERE s.containerid=:1
1014+
AND dp.experimentkind=:2
1015+
AND ss.diffractionplanid IS NOT NULL
1016+
AND cqs.containerqueuesampleid IS NOT NULL
1017+
AND cqs.containerqueueid IS NULL",
1018+
array($this->arg('cid'), $this->arg('EXPERIMENTKIND')));
1019+
1020+
if (!sizeof($rows)) {
1021+
$this->_output(array('TOTAL_UPDATED' => 0));
1022+
return;
1023+
}
1024+
1025+
$all_ids = array_column($rows, 'DIFFRACTIONPLANID');
1026+
$set_args = array();
1027+
foreach (array(
1028+
'REQUIREDRESOLUTION', 'EXPERIMENTKIND', 'PREFERREDBEAMSIZEX', 'PREFERREDBEAMSIZEY',
1029+
'EXPOSURETIME', 'BOXSIZEX', 'BOXSIZEY', 'AXISSTART', 'AXISRANGE', 'NUMBEROFIMAGES',
1030+
'TRANSMISSION', 'ENERGY', 'MONOCHROMATOR'
1031+
) as $f) {
1032+
array_push($set_args, $this->has_arg($f) ? $this->arg($f) : null);
1033+
}
1034+
1035+
$chunks = array_chunk($all_ids, 500);
1036+
$total_updated = 0;
1037+
1038+
foreach ($chunks as $chunk) {
1039+
$start_index = sizeof($set_args) + 1;
1040+
$placeholders = array();
1041+
1042+
foreach ($chunk as $index => $id) {
1043+
array_push($placeholders, ':' . ($start_index + $index));
1044+
}
1045+
1046+
$id_placeholders_list = implode(',', $placeholders);
1047+
1048+
$combined_args = array_merge($set_args, $chunk);
1049+
1050+
$this->db->pq("UPDATE diffractionplan
1051+
SET requiredresolution=:1, experimentkind=:2, preferredbeamsizex=:3, preferredbeamsizey=:4,
1052+
exposuretime=:5, boxsizex=:6, boxsizey=:7, axisstart=:8, axisrange=:9,
1053+
numberofimages=:10, transmission=:11, energy=:12, monochromator=:13
1054+
WHERE diffractionplanid IN ($id_placeholders_list)", $combined_args);
1055+
1056+
$total_updated += sizeof($chunk);
1057+
}
1058+
1059+
$this->_output(array('TOTAL_UPDATED' => $total_updated));
1060+
}
1061+
9631062

9641063
function _add_sub_sample()
9651064
{

‎client/src/js/modules/imaging/views/queuecontainer.js‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ define(['marionette',
683683
app.message({ message: 'Container Successfully Unqueued' })
684684
self.ui.unqueuebutton.hide()
685685
self.ui.queuebutton.show()
686-
self.model.set('CONTAINERQUEUEID', null)
686+
self.model.set('CONTAINERQUEUEID', null, { silent: true })
687687
},
688688
error: function() {
689689
app.alert({ message: 'Something went wrong unqueuing this container' })
@@ -704,23 +704,36 @@ define(['marionette',
704704

705705
var p = this.plans.findWhere({ DIFFRACTIONPLANID: this.ui.preset.val() })
706706
if (p) {
707-
self.ui.applyall.html('Applying...').prop('disabled', true)
708-
var promises = await this.applyModel(p, false)
709707

710-
const updateButtonAfterProcessing = () => {
711-
self.ui.applyall.html('<i class="fa fa-file-text-o"></i> Apply to All').prop('disabled', false)
708+
var updateParams = {
709+
EXPERIMENTKIND: p.get('EXPERIMENTKIND')
712710
}
713711

714-
if (promises && promises.length > 0) {
715-
Promise.allSettled(promises)
716-
.then(updateButtonAfterProcessing)
717-
.catch(error => {
718-
console.error("Error after promises settled: ", error);
719-
updateButtonAfterProcessing()
720-
})
721-
} else {
722-
updateButtonAfterProcessing()
723-
}
712+
const fields = [
713+
'REQUIREDRESOLUTION', 'PREFERREDBEAMSIZEX', 'PREFERREDBEAMSIZEY',
714+
'EXPOSURETIME', 'BOXSIZEX', 'BOXSIZEY', 'AXISSTART', 'AXISRANGE',
715+
'NUMBEROFIMAGES', 'TRANSMISSION', 'ENERGY', 'MONOCHROMATOR'
716+
]
717+
718+
fields.forEach(function(k) {
719+
if (p.get(k) !== null && p.get(k) !== undefined) {
720+
updateParams[k] = p.get(k)
721+
}
722+
})
723+
724+
Backbone.ajax({
725+
url: app.apiurl+'/sample/sub/cid/'+this.model.get('CONTAINERID'),
726+
method: 'POST',
727+
data: updateParams,
728+
success: function(json) {
729+
self.refreshSubSamples()
730+
app.message({ message: 'Applied preset to ' + json.TOTAL_UPDATED + ' samples' })
731+
},
732+
error: function() {
733+
app.alert({ message: 'Something went wrong applying to all' })
734+
}
735+
})
736+
724737
}
725738
},
726739

@@ -777,17 +790,7 @@ define(['marionette',
777790
return
778791
}
779792

780-
// need to validate all models here again in case they haven't been rendered
781-
this.qsubsamples.fullCollection.each(function(qs) {
782-
if (qs.get('_valid') !== undefined) return
783-
784-
const expcell = new ExperimentCell({model: qs, column: {beamlinesetups: this.beamlinesetups}});
785-
const val = expcell.checkIsValid();
786-
console.log({ experimentCell: val })
787-
}, this)
788-
789793
var invalid = this.typeselector.shadowCollection.where({ '_valid': false })
790-
console.log('queue', invalid, invalid.length > 0)
791794
if (invalid.length > 0) {
792795
app.alert({ message: 'There are '+invalid.length+' sub samples with invalid experimental plans, please either correct or remove these from the queue' })
793796
var inv = this.typeselector.collection.findWhere({ id: 'invalid' })
@@ -804,7 +807,7 @@ define(['marionette',
804807
app.message({ message: 'Container Successfully Queued' })
805808
self.ui.unqueuebutton.show()
806809
self.ui.queuebutton.hide()
807-
self.model.set('CONTAINERQUEUEID', json.CONTAINERQUEUEID)
810+
self.model.set('CONTAINERQUEUEID', json.CONTAINERQUEUEID, { silent: true })
808811
},
809812
error: function() {
810813
app.alert({ message: 'Something went wrong queuing this container' })

0 commit comments

Comments
 (0)