Skip to content

Commit f6a4dbc

Browse files
ehennestadCopilot
ehennestad
and
Copilot
authored
Support validation for unspecified dtype (#816)
* Create getBasicDTypeMap.m * Update mapType.m Added docstring Added arguments block with input validation Use type map for conversion to MATLAB type for basic dtype * Update mapType.m Cleanup/simplify * Update getBasicDTypeMap.m Use try/catch for better efficiency * Use uint->uint8, int->int8 in mapper * Regenerate core * Update +file/mapType.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Create MapTypeTest.m * Update types.util.checkDType to perform validation when any dtype is allowed * Added unit tests for checking dtype=any * Stricter compound value check plus tests * Update type generator to add type validation for dtype=any * Regenerate types * Fix test assigning invalid data to a VectorData object cell array of numerics are not supported data for a VectorData, export will fail with error id NWB:MapData:NonCellStr This test verifying a different error and should not assign invalid data * Add external link as valid type if any type is allowed Also suppress function output from validateAnyType * Fix tests with legacy ragged array assignment to vectordata * Fix unwrap and rewrap of nested values * Update checkDtype.m * Update +types/+util/rewrapValue.m Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update checkDtype.m Ensure validBasicTypes is a column vector --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 4715b87 commit f6a4dbc

18 files changed

Lines changed: 367 additions & 44 deletions

‎+file/Dataset.m‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
attributes;
1515
linkable;
1616
definesType;
17+
skipDtypeValidation;
1718
end
1819

1920
methods
@@ -28,6 +29,7 @@
2829
obj.readonly = false;
2930
obj.scalar = true;
3031
obj.definesType = false;
32+
obj.skipDtypeValidation = false;
3133

3234
obj.shape = {};
3335
obj.dimnames = {};
@@ -79,6 +81,10 @@
7981
if isKey(source, dataTypeKey)
8082
obj.dtype = file.mapType(source(dataTypeKey));
8183
end
84+
85+
if isKey(source, 'skip_dtype_validation')
86+
obj.skipDtypeValidation = logical(source('skip_dtype_validation'));
87+
end
8288

8389
% If a value key is specified, the resulting property is a
8490
% constant (and by definition required). Therefore we will only
@@ -167,4 +173,4 @@
167173
end
168174
end
169175
end
170-
end
176+
end

‎+file/fillValidators.m‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,11 @@
179179

180180
datasetAttributeValidationLines = ...
181181
fillTypedDatasetAttributeReferenceValidation(name, prop, namespaceReg);
182-
datasetValidationLines = [...
183-
{fillDtypeValidation(name, prop.dtype, namespaceReg)}, ...
184-
{fillDimensionValidation(name, prop.shape)} ];
182+
datasetValidationLines = {};
183+
if ~prop.skipDtypeValidation
184+
datasetValidationLines{end+1} = fillDtypeValidation(name, prop.dtype, namespaceReg); %#ok<AGROW>
185+
end
186+
datasetValidationLines{end+1} = fillDimensionValidation(name, prop.shape); %#ok<AGROW>
185187

186188
datasetAttributeValidationLines(cellfun('isempty', datasetAttributeValidationLines)) = [];
187189
datasetValidationLines(cellfun('isempty', datasetValidationLines)) = [];
@@ -317,12 +319,7 @@
317319
fdvstr = fillReferenceTypeValidation(name, type, namespaceReg);
318320
else
319321
fdvstr = '';
320-
if strcmp(type, 'any')
321-
fdvstr = '';
322-
return;
323-
else
324-
ts = strrep(type, '-', '_');
325-
end
322+
ts = strrep(type, '-', '_');
326323
fdvstr = [fdvstr ...
327324
'val = types.util.checkDtype(''' name ''', ''' ts ''', val);'];
328325
end

‎+spec/+internal/expandFieldsInheritedByInclusion.m‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,13 @@ function lockKeysInheritedThroughInclusion(spec)
1616
~(isKey(spec, 'neurodata_type_def') || isKey(spec, 'data_type_def'))
1717
% "inheritance" by inclusion
1818

19-
% If dtype and shape keys are not defined for a dataset, set these to
20-
% 'any' and 'null' respectively. This is done to ensure the class
21-
% generator will not create validation functions for a dataset using
22-
% default values and instead, the validation will be handled in the
23-
% "included" class.
19+
% If dtype and shape keys are not defined for a dataset, mark these so
20+
% the class generator will not create validation functions using
21+
% default values. Validation will instead be handled in the included
22+
% class when dtype is omitted on the including dataset.
2423

2524
if ~isKey(spec, 'dtype')
26-
spec('dtype') = 'any';
25+
spec('skip_dtype_validation') = true; %#ok<NASGU> spec is a handle object (containers.Map), so this assignment persists even if the variable is not returned.
2726
end
2827
if ~isKey(spec, 'shape')
2928
spec('shape') = nan; %#ok<NASGU> spec is a handle object (containers.Map), so this assignment persists even if the variable is not returned.

‎+tests/+system/DynamicTableTest.m‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ function addContainerUnevenColumns(~, file)
5252
), ...
5353
'randomvalues', types.hdmf_common.VectorData( ...
5454
'description', 'randomvalues column', ...
55-
'data', mat2cell(rand(25,2), repmat(5, 5, 1)) ...
55+
'data', rand(2, 5) ...
5656
) ...
5757
);
5858
end
@@ -76,7 +76,7 @@ function addContainerUnmatchedIDs(~, file)
7676
), ...
7777
'randomvalues', types.hdmf_common.VectorData( ...
7878
'description', 'randomvalues column', ...
79-
'data', mat2cell(rand(25,2), repmat(5, 5, 1)) ...
79+
'data', rand(2, 5) ...
8080
) ...
8181
);
8282
end
@@ -89,15 +89,15 @@ function addContainerUndefinedIDs(~, file)
8989
'colnames', colnames, ...
9090
'start_time', types.hdmf_common.VectorData( ...
9191
'description', 'start time column', ...
92-
'data', (1:5)' ...
92+
'data', (1:5) ...
9393
), ...
9494
'stop_time', types.hdmf_common.VectorData( ...
9595
'description', 'stop time column', ...
96-
'data', (2:6)' ...
96+
'data', (2:6) ...
9797
), ...
9898
'randomvalues', types.hdmf_common.VectorData( ...
9999
'description', 'randomvalues column', ...
100-
'data', mat2cell(rand(25,2), repmat(5, 5, 1)) ...
100+
'data', rand(2, 5) ...
101101
) ...
102102
);
103103
end
@@ -279,7 +279,7 @@ function DynamicTableCheckTest(testCase)
279279
@() testCase.addContainerUnevenColumns(testCase.file), ...
280280
'NWB:DynamicTable:CheckConfig:InvalidShape' ...
281281
)
282-
% 2. Defining a table with length of id's does not match
282+
% 2. Defining a table where length of id's does not match
283283
% the number of columns
284284
testCase.verifyError( ...
285285
@() testCase.addContainerUnmatchedIDs(testCase.file), ...

‎+tests/+unit/+schema/IncludedTypedDatasetValidationTest.m‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
classdef IncludedTypedDatasetValidationTest < matlab.unittest.TestCase
2+
3+
methods (Test)
4+
5+
function testExpandFieldsInheritedByInclusionMarksMissingDtype(testCase)
6+
datasetSpec = containers.Map();
7+
datasetSpec('data_type_inc') = 'AnyData';
8+
9+
node = containers.Map();
10+
node('datasets') = {datasetSpec};
11+
12+
spec.internal.expandFieldsInheritedByInclusion(node)
13+
14+
testCase.verifyFalse(isKey(datasetSpec, 'dtype'))
15+
testCase.verifyTrue(isKey(datasetSpec, 'skip_dtype_validation'))
16+
testCase.verifyTrue(datasetSpec('skip_dtype_validation'))
17+
testCase.verifyTrue(isKey(datasetSpec, 'shape'))
18+
testCase.verifyTrue(isnumeric(datasetSpec('shape')) && isnan(datasetSpec('shape')))
19+
end
20+
21+
function testExpandFieldsInheritedByInclusionDoesNotMarkExplicitDtype(testCase)
22+
datasetSpec = containers.Map();
23+
datasetSpec('data_type_inc') = 'AnyData';
24+
datasetSpec('dtype') = 'text';
25+
26+
node = containers.Map();
27+
node('datasets') = {datasetSpec};
28+
29+
spec.internal.expandFieldsInheritedByInclusion(node)
30+
31+
testCase.verifyFalse(isKey(datasetSpec, 'skip_dtype_validation'))
32+
testCase.verifyEqual(datasetSpec('dtype'), 'text')
33+
end
34+
35+
function testDatasetReadsSkipDtypeValidationFlag(testCase)
36+
datasetSpec = containers.Map();
37+
datasetSpec('data_type_inc') = 'AnyData';
38+
datasetSpec('skip_dtype_validation') = true;
39+
40+
datasetObj = file.Dataset(datasetSpec);
41+
42+
testCase.verifyTrue(datasetObj.skipDtypeValidation)
43+
testCase.verifyEqual(datasetObj.dtype, 'any')
44+
end
45+
end
46+
end

‎+tests/+unit/+types/+validators/CheckDtypeTest.m‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,85 @@ function testRejectsDatasetClassWhenRawDataIsExpected(testCase)
123123
'NWB:CheckDataType:InvalidConversion')
124124
end
125125

126+
function testAnyDtypeAllowsBasicAndTextTypes(testCase)
127+
stringValue = "hello world";
128+
cellstrValue = {'hello world'};
129+
charValue = 'hello world';
130+
datetimeValue = datetime('now');
131+
numericValue = uint8(1);
126132

133+
value = types.util.checkDtype('stringValue', 'any', stringValue);
134+
testCase.verifyClass(value, 'string')
127135

136+
value = types.util.checkDtype('cellstrValue', 'any', cellstrValue);
137+
testCase.verifyTrue(iscellstr(value))
138+
139+
value = types.util.checkDtype('charValue', 'any', charValue);
140+
testCase.verifyClass(value, 'char')
141+
142+
value = types.util.checkDtype('datetimeValue', 'any', datetimeValue);
143+
testCase.verifyClass(value, 'datetime')
144+
145+
value = types.util.checkDtype('numericValue', 'any', numericValue);
146+
testCase.verifyClass(value, 'uint8')
147+
end
148+
149+
function testAnyDtypeAllowsCompoundValues(testCase)
150+
[testValueStruct, testValueTable, testValueMap] = ...
151+
testCase.getCompoundValues(...
152+
'a', 1, ...
153+
'b', 'hello world');
154+
155+
value = types.util.checkDtype('structValue', 'any', testValueStruct);
156+
testCase.verifyClass(value, 'struct')
157+
158+
value = types.util.checkDtype('tableValue', 'any', testValueTable);
159+
testCase.verifyClass(value, 'table')
160+
161+
value = types.util.checkDtype('mapValue', 'any', testValueMap);
162+
testCase.verifyClass(value, 'containers.Map')
163+
end
164+
165+
function testAnyDtypeAllowsWrappedValue(testCase)
166+
wrappedValue = types.untyped.Anon('test', uint8(1));
167+
168+
value = types.util.checkDtype('wrappedValue', 'any', wrappedValue);
169+
170+
testCase.verifyClass(value, 'types.untyped.Anon')
171+
testCase.verifyEqual(value.value, uint8(1))
172+
end
173+
174+
function testAnyDtypeAllowsSoftLink(testCase)
175+
warningResetObj = types.untyped.SoftLink.disablePathDeprecationWarning(); %#ok<NASGU>
176+
177+
softLinkValue = types.untyped.SoftLink('/some/path');
178+
179+
value = types.util.checkDtype('softLinkValue', 'any', softLinkValue);
180+
181+
testCase.verifyClass(value, 'types.untyped.SoftLink')
182+
end
183+
184+
function testAnyDtypeRejectsInvalidCompoundFieldValue(testCase)
185+
invalidCompoundValue = struct('a', {{1, 2}});
186+
187+
testCase.verifyError(...
188+
@() types.util.checkDtype('invalidCompoundValue', 'any', invalidCompoundValue), ...
189+
'NWB:CheckDType:InvalidType')
190+
end
191+
192+
function testAnyDtypeRejectsNestedCompoundValue(testCase)
193+
nestedCompoundValue = struct('a', struct('b', 1));
194+
195+
testCase.verifyError(...
196+
@() types.util.checkDtype('nestedCompoundValue', 'any', nestedCompoundValue), ...
197+
'NWB:CheckDType:NestedCompoundNotSupported')
198+
end
199+
200+
function testAnyDtypeRejectsUnsupportedType(testCase)
201+
testCase.verifyError(...
202+
@() types.util.checkDtype('invalidValue', 'any', {struct()}), ...
203+
'NWB:CheckDType:InvalidType')
204+
end
128205
end
129206

130207
methods (Static)

‎+tests/+unit/+types/FunctionTests.m‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,5 +116,20 @@ function testCorrectType(testCase)
116116
@(varargin) types.util.correctType('5i', 'double'), ...
117117
'NWB:TypeCorrection:DataLoss')
118118
end
119+
120+
function testUnwrapAndRewrapNestedAnonDataset(testCase)
121+
datasetValue = types.hdmf_common.VectorData(...
122+
'data', int8(1), ...
123+
'description', 'test');
124+
wrappedValue = types.untyped.Anon('wrapped_data', datasetValue);
125+
126+
[value, originalValue] = types.util.unwrapValue(wrappedValue);
127+
testCase.verifyEqual(value, int8(1))
128+
129+
rewrappedValue = types.util.rewrapValue(value, originalValue);
130+
testCase.verifyClass(rewrappedValue, 'types.untyped.Anon')
131+
testCase.verifyClass(rewrappedValue.value, 'types.hdmf_common.VectorData')
132+
testCase.verifyEqual(rewrappedValue.value.data, int8(1))
133+
end
119134
end
120-
end
135+
end

‎+tests/+unit/dynamicTableTest.m‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,16 @@ function testEmptyTable(testCase)
2121
testCase.verifyError(@()types.core.TimeIntervals(...
2222
'description', 'test error', ...
2323
'start_time', types.hdmf_common.VectorData( ...
24-
'description', 'start time column', ...
25-
'data', (1:5)' ...
26-
)), ...
24+
'description', 'start time column', ...
25+
'data', (1:5)')), ...
2726
'NWB:DynamicTable:CheckConfig:ColumnNamesMismatch');
2827

2928
% validate that "vectordata" set checking works.
3029
testCase.verifyError(@()types.core.TimeIntervals(...
3130
'description', 'test error', ...
3231
'randomvalues', types.hdmf_common.VectorData( ...
33-
'description', 'random values', ...
34-
'data', mat2cell(rand(25,2), repmat(5, 5, 1)))), ...
32+
'description', 'random values', ...
33+
'data', rand(2,5))), ...
3534
'NWB:DynamicTable:CheckConfig:ColumnNamesMismatch');
3635
end
3736

‎+types/+core/BaseImage.m‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
%% VALIDATORS
5353

5454
function val = validate_data(obj, val)
55+
val = types.util.checkDtype('data', 'any', val);
5556
types.util.validateShape('data', {[1]}, val)
5657
end
5758
function val = validate_description(obj, val)

‎+types/+core/NWBData.m‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
%% VALIDATORS
4343

4444
function val = validate_data(obj, val)
45+
val = types.util.checkDtype('data', 'any', val);
4546
types.util.validateShape('data', {[1]}, val)
4647
end
4748
%% EXPORT

0 commit comments

Comments
 (0)