function [ data_processed ] = pnPreprocess(data, verboseFlag)
%PNPREPROCESS Output data after preprocessing
% [ data ] = pnPreprocess(data)
% Primary variables:
% data - Cell array data, rows = # of patients
% dataDesc - Cell array of data fields/descriptions
% dataFixed - Cell array of demographic data fields/descriptions
% tmp - Data for a single parameter, e.g. 'HR'
% idxRem - Index for data to be removed at end of the loop
% idxManip - Index for data to be manipulated some how in switch block
% high/low - Extracted data which is manipulated and re-imputed into tmp
% Copyright 2012 Alistair Johnson
% $LastChangedBy: alistair $
% $LastChangedDate: 2012-05-29 09:01:11 -0400 (Tue, 29 May 2012) $
% $Revision: 1 $
% Originally written on PCWIN64 by Alistair Johnson, 25-Apr-2012 01:37:49
% Contact: alistairewj@gmail.com
if nargin<2
verboseFlag = false;
end
data_processed = data;
[dataDesc,dataFixed] = pnDataDescriptions();
%=== Loop through the 3 fixed, demographic fields
for k=1:size(dataFixed,1)
fn = dataFixed{k,1};
if verboseFlag; fprintf('\n%%=== %s ===%%\n', fn); end;
[tmp,idx] = pnExtractField(data_processed,fn);
%=== Reset delete indices
idxRem = [];
switch fn
case 'RecordID'
continue;
case 'Age'
idxManip = cellfun(@(x) x>100, tmp(:,4),'UniformOutput',false);
high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
high = cellfun(@(x) x*0+105, high, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = high{m};
end
if verboseFlag; fprintf('Replaced %2.0f values of 200 with 105.\n',sum(cell2mat(idxManip))); end;
%=== Analyze residuals of data, check for bias
% idxManip = cellfun(@(x) numel(x), tmp(:,4),'UniformOutput',false);
% tmpDataAnalyze = cellfun(@(x,y) x(1:y-1) - x(y), tmp(:,4), idxManip, 'UniformOutput',false);
% tmpDataAnalyze = cell2mat(tmpDataAnalyze(cellfun(@(x) ~isempty(x), tmpDataAnalyze)));
% hist(tmpDataAnalyze,-10:1:10); xlabel('Age (1:end-1) - Age(end)');
idxManip = cellfun(@(x) [true(numel(x)-1,1);false], tmp(:,4),'UniformOutput',false);
%=== Impute 0s for vector values
tmp(:,4) = cellfun(@(x,y) x-x.*y, tmp(:,4), idxManip, 'UniformOutput',false);
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq'); % index ==0 for removal
if verboseFlag; fprintf('Deleted %2.0f vector values, leaving only the end value.\n',N); end;
case 'Gender'
idxManip = cellfun(@(x) (x==-1), tmp(:,4),'UniformOutput',false);
high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
high = cellfun(@(x) NaN, high, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = high{m};
end
if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(cell2mat(idxManip))); end;
case 'Height'
% 1 centimetre = 0.393700787 inches
% 1 foot = 30.48 centimetres
% 1 inch = 2.54 centimetres
idxManip = cellfun(@(x) x==-1, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) NaN, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) x<10, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*100, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multipled %2.0f values of <10 by 100 (1.8->180).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) x<25, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*10, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multipled %2.0f values of <25 by 10 (18->180).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) x<100, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*2.54, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multipled %2.0f values of <100 by 2.2 (81.8->180).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) x>1000, tmp(:,4),'UniformOutput',false);
high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
high = cellfun(@(x) x*0.1, high, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = high{m};
end
if verboseFlag; fprintf('Multipled %2.0f values of >1000 by 0.1 (1800->180).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) x>250, tmp(:,4),'UniformOutput',false);
high = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
high = cellfun(@(x) x/2.54, high, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = high{m};
end
if verboseFlag; fprintf('Multipled %2.0f values of >250 by 0.3937 (400->157).\n',sum(cell2mat(idxManip))); end;
end
%=== Replace data with tmp
for m=2:size(data,2)
data_processed(:,m) = cellfun(@pnPreprocessReplaceData, data_processed(:,m), tmp(:,m), idx, 'UniformOutput', false);
end
%=== Delete entries flagged for deletion
if ~isempty(idxRem)
for m=2:size(data,2)
data_processed(:,m) = cellfun(@pnPreprocessDeleteData, data_processed(:,m), idxRem, 'UniformOutput', false);
end
end
end
%=== Loop through all the fields, and process each appropriately
for k=1:size(dataDesc,1)
fn = dataDesc{k,1};
if verboseFlag; fprintf('\n%%=== %s ===%%\n', fn);
[tmp,idx] = pnExtractField(data_processed,fn);
idxRem=[];
switch fn
case 'Albumin'
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'ALP'
%=== Possible conversion errors in ALP, ALT, AST:
% Listed as measured in IU
% 1 Katal = 60,000,000 IU
% 1 micro Katal = 60 IU
% 1 nano Katal = 0.060 IU
% *** micro kats are sometimes used.
% Reference range: 30 to 120 IU/L
if verboseFlag; fprintf('No preprocessing performed - Note: highly skewed. Dependent on age and gender.\n'); end;
case 'ALT'
% Reference range: 7 to 56 IU/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'AST'
% Reference range: 5 to 40 IU/L
% Very heavy tailed: Normal range 5 to 40 IU/L, 10%% data > 1000 IU/L.
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Bilirubin'
% Reference range: 0.2 to 1.2 mg/dL
% 1 mg/dL == 17.1 µmol/L
% It is possible to have >36 mg/dL in cases of transplant, etc.
% Can't unilaterally fix this.
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'BUN'
% Reference range: 10-20 mg/dl (3.6-71 mmol/liter)
% 1 mg/dL == 0.357 mmol/L
% Can't really convert anything here.
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
% for m=2:size(tmp,2)
% tmp(:,m) = cellfun(@(x,y) x(y), tmp(:,m), idxManip,'UniformOutput',false);
% end
case 'Cholesterol'
% Reference range:
% Desirable <200 mg/dl <5.17 mmol/L
% Borderline high 200-239 mg/dl 5.17-6.18 mmol/L
% High ≥240 mg/dl ≥6.18 mmol/L
% 1 mg/dL == 0.0259 mmol/L
% Nothing bad here.
if verboseFlag; fprintf('No preprocessing needed.\n'); end;
case 'Creatinine'
% Reference range: 0.6-1.3 mg/dl
% <1.5 mg/dl (NEJM)
% 1 mg/dL == 88.4 µmol/L
%=== ~6 is a reasonable maximum, 7.6 is very confident
% Seems OK.
if verboseFlag; fprintf('No preprocessing needed.\n'); end;
case 'DiasABP'
%=== First, delete '0's since we don't know if it was badly
% converted or missing
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
[idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, -1, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which were -1.\n',N); end;
idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
idxManip = cellfun(@(x) x>170, tmp(:,4),'UniformOutput',false);
idxManip2 = cellfun(@(x) x<200, tmp(:,4),'UniformOutput',false);
idxManip = cellfun(@(x) sum(x), idxManip);
idxManip2 = cellfun(@(x) sum(~x), idxManip2);
if verboseFlag; fprintf('There exist %2.0f values between 170-200, which were left as is.\n',sum(idxManip)-sum(idxManip2)); end;
[idxRem2, N] = pnPreprocessRemovalIndices(tmp, idx, 200, 'gt');
[idxRem] = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false);
if verboseFlag; fprintf('Deleted %2.0f values above 200.\n',N); end;
case 'FiO2'
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'GCS'
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Glucose'
% Reference ranges:
% Fasting
% Normal 75-115 mg/dl 4.2-6.4 mmol/L
% Diabetes mellitus >125 mg/dl >7.0 mmol/L
% 2 Hr post-meal <120 mg/dl <6.7 mmol/L
% 1 mg/dL == 0.0555 mmol/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'HCO3'
% Reference ranges: 21-30 mEq/L 21-28 mmol/L
% 1 mEq/L == 1 mmol/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'HCT'
% Reference range: 41.0-53.0%
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'HR'
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
[idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, 299, 'gt');
idxRem = cellfun(@(x,y) x | y, idxRem, idxRem2, 'UniformOutput',false); % Combine removal indices
if verboseFlag; fprintf('Deleted %2.0f values which above 299.\n',N); end;
case 'K'
% Reference ranges: 3.5-5.0 mEq/L 3.5-5.0 mmol/L
% 1 mEq/L == 1 mmol/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Lactate' % mmol/L
% Reference ranges: 5-15 mg/dl 0.6-1.7 mmol/liter
% 1 mg/dL == 0.111 mmol/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Mg' % mmol/L
% Reference ranges: 1.8-3.0 mg/dl 0.8-1.2 mmol/L
%1 mg/dL == 0.411 mmol/L
%1 mEq/L == 0.50 mmol/L
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'MAP' % mmHg
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'MechVent'
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Na' % mEq/L
% Reference ranges: 136-145 mEq/L 136-145 mmol/L
% 1 mEq/L == 1 mmol/L
%=== Interesting spike at 150, possible rounding bias?
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'NIDiasABP' % mmHg
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'NIMAP' % mmHg
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'NISysABP' % mmHg
% Values below 1??
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which less than 1.\n',N); end;
case 'PaCO2' % mmHg
% Reference range:
% 4.7-6.0 kPa
% 35-45 mmHg
% 1 kPa == 7.5006 mmHg
idxManip = cellfun(@(x) x<10 & x>1, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*10, low, 'UniformOutput',false);
if verboseFlag; pnPreprocessPrintReplacedValues(tmp(:,4),low,idxManip); end;
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Replaced %2.0f values between [1,10] which were incorrectly recorded.\n',sum(cell2mat(idxManip))); end;
% Values below 1 must be wrong...
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
% idxManip = cellfun(@(x) x<1, tmp(:,4),'UniformOutput',false);
% low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
% low = cellfun(@(x) x*100, low, 'UniformOutput',false);
%
% for m=1:size(tmp,1)
% tmp{m,4}(idxManip{m}) = low{m};
% end
case 'PaO2' % mmHg
% Reference range:
% 11-13 kPa
% 75-100 mmHg
% 1 kPa == 7.5006 mmHg
idxManip = cellfun(@(x) x<20 & x>1, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*10, low, 'UniformOutput',false);
if verboseFlag; pnPreprocessPrintReplacedValues(tmp(:,4),low,idxManip); end;
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Replaced %2.0f values between [1,20] which were incorrectly recorded.\n',sum(cell2mat(idxManip))); end;
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'pH'
% Reference range:
% 7.34-7.45 units
%=== This is a mess
% There are 5 values between 94-100, not sure why. Probably put
% in the wrong field? Doesn't match with an [H+] conversion
idxManip = cellfun(@(x) (x>0.65 & x<0.8), tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*7.5006, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multiplied %2.0f values by 10 (0.7->7).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) (x>65 & x<80), tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*0.1, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multiplied %2.0f values by 0.1 (70->7).\n',sum(cell2mat(idxManip))); end;
idxManip = cellfun(@(x) (x>650 & x<800), tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*0.01, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multiplied %2.0f values by 0.01 (700->7).\n',sum(cell2mat(idxManip))); end;
%=== Values <6.5 and >0.8 are deleted
% Note: (there are no values around 0.6-0.75)
[idxRem] = pnPreprocessRemovalIndices(tmp, idx, 0.8, 'gt');
[idxRem2] = pnPreprocessRemovalIndices(tmp, idx, 6.5, 'lt');
idxRem = cellfun(@(x,y) x & y, idxRem, idxRem2, 'UniformOutput',false);
N = sum(cell2mat(cellfun(@(x) sum(x), idxRem, 'UniformOutput', false)));
if verboseFlag; fprintf('Deleted %2.0f values which were between [0.8,6.5].\n',N); end;
%=== Values >80 & <650 are deleted
% Note: (there are no values around 0.6-0.75)
[idxRem3] = pnPreprocessRemovalIndices(tmp, idx, 80, 'gt');
[idxRem4] = pnPreprocessRemovalIndices(tmp, idx, 650, 'lt');
idxRem = cellfun(@(x,y,z) (x & y) | z, idxRem3, idxRem4, idxRem, 'UniformOutput',false);
N = sum(cell2mat(cellfun(@(x,y) sum(x&y), idxRem3, idxRem4, 'UniformOutput', false)));
if verboseFlag; fprintf('Deleted %2.0f values which were between [80,650].\n',N); end;
case 'Platelets' % cells/nL
% Reference range: 150-350 (10e3)/mm^3
% 1 (10e3)/µL == 1 (10e3)/mm^3 == 1/nL
% Errors are probably in orders of 1000
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'RespRate' % bpm
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'SaO2' % %
% Reference range: 94-100
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'SysABP' % mmHg
% Reference range:
% 10-14 kPa
% 75-105 mmHg
% 1 kPa == 7.5006 mmHg
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'Temp'
%=== Check for Farenheit measurements and incorrect conversions
%=== There are 129 values which are too low...
%=== Assume some of the values are artefacts
%=== INCORRECTLY CONVERTED F->C
% 1x range should be ~ [1.5,8], use (1,10)
idxManip = cellfun(@(x) x<10 & x>1, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*9/5+32, low, 'UniformOutput',false);
%=== spooky action at a distance
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
% 2x range should be ~ [-16.8,-13], use (-17,-13)
idxManip2 = cellfun(@(x) x<=-13 & x>-17, low, 'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip2,'UniformOutput',false);
low = cellfun(@(x) (x*9/5+32)*9/5+32, low, 'UniformOutput',false);
%=== spooky action at a distance
for m=1:size(tmp,1)
tmp{m}(idxManip2{m}) = low{m};
end
%=== RECORDED AS F
% 1x range should be ~ [95,113]
idxManip3 = cellfun(@(x) x>90 & x<120, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip3,'UniformOutput',false);
low = cellfun(@(x) (x-32)*5/9, low, 'UniformOutput',false);
%=== spooky action at a distance
for m=1:size(tmp,1)
tmp{m}(idxManip3{m}) = low{m};
end
%=== DELETING VALUES NOW ===%
%=== Delete 0s and values below -17s (0s and converted 0s)
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which were equal to 0.\n',N); end;
[idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, -17, 'lt');
if verboseFlag; fprintf('%2.0f of those values were -17.8 (i.e. incorrectly converted from 0).\n',N); end;
%=== Negative values which don't seem erroneously converted
[idxRem3] = pnPreprocessRemovalIndices(tmp, idx, -13, 'ge');
[idxRem4] = pnPreprocessRemovalIndices(tmp, idx, 1, 'le');
idxRem3 = cellfun(@(x,y) x & y, idxRem3, idxRem4, 'UniformOutput',false);
N = sum(cell2mat(cellfun(@(x) sum(x), idxRem3, 'UniformOutput', false)));
if verboseFlag; fprintf('Deleted %2.0f values which were between [-13,1].\n',N); end;
%=== Positive values which don't seem erroneously converted
[idxRem4] = pnPreprocessRemovalIndices(tmp, idx, 50, 'ge');
[idxRem5] = pnPreprocessRemovalIndices(tmp, idx, 90, 'le');
idxRem4 = cellfun(@(x,y) x & y, idxRem4, idxRem5, 'UniformOutput',false);
N = sum(cell2mat(cellfun(@(x) sum(x), idxRem4, 'UniformOutput', false)));
if verboseFlag; fprintf('Deleted %2.0f values which were between [50,90].\n',N); end;
[idxRem5,N] = pnPreprocessRemovalIndices(tmp, idx, 119, 'gt');
if verboseFlag; fprintf('Deleted %2.0f values which were between above 119.\n',N); end;
idxRem = cellfun(@(l,m,f,a,o) l | m | f | a | o, idxRem, idxRem2, idxRem3, idxRem4, idxRem5, 'UniformOutput',false);
if verboseFlag; fprintf('Replaced %2.0f values which were incorrectly converted F->C.\n',sum(cell2mat(idxManip))); end;
if verboseFlag; fprintf('Replaced %2.0f values which were incorrectly converted F->C twice.\n',sum(cell2mat(idxManip2))); end;
if verboseFlag; fprintf('Replaced %2.0f values which were incorrectly recorded as F.\n',sum(cell2mat(idxManip2))); end;
case 'TroponinI'
% Reference ranges: 0-0.4 µg/L
% Could be off by orders of 10
idxManip = cellfun(@(x) (x>30), tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*0.1, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Multiplied %2.0f values by 0.11 (30->0.3).\n',sum(cell2mat(idxManip))); end;
case 'TroponinT'
% Reference ranges: 0-0.1 µg/L
% Could be off by orders of 10.
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'Urine'
if verboseFlag; fprintf('No preprocessing performed.\n'); end;
case 'WBC' % cells/nL
% Reference range: 4.5-11.0
% 1 (10^3)/µL == 1 (10^9)/L
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 1, 'lt');
idxManip = cellfun(@(x) x<1 & x>0, tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) x*10, low, 'UniformOutput',false);
if verboseFlag; pnPreprocessPrintReplacedValues(tmp(:,4),low,idxManip); end;
if verboseFlag; fprintf('Deleted %2.0f values which were less than 1.\n',N); end;
case 'Weight' % kg
% Reference range:
% 86.6 kg (190.9 lb) (males)
% 74.4 kg (164.0 lb) (females)
% 1 kilogram = 2.20462262 pounds, 1 pound = 0.45359237 kilograms
% 1 kilogram = 0.15747304 stones, 1 stone = 6.35029318 kilograms
%=== Impute NaN in the first weight, delete the others
idxManip = cellfun(@(x) ( (x+1)<1e-6 ), tmp(:,4),'UniformOutput',false);
low = cellfun(@(x,y) x(y), tmp(:,4),idxManip,'UniformOutput',false);
low = cellfun(@(x) NaN, low, 'UniformOutput',false);
for m=1:size(tmp,1)
tmp{m,4}(idxManip{m}) = low{m};
end
if verboseFlag; fprintf('Replaced %2.0f values which were -1 with NaN.\n',sum(cell2mat(idxManip))); end;
[idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
if verboseFlag; fprintf('Deleted %2.0f values which were 0.\n',N); end;
%=== The following values continually pop up:
% 0.6 is on patient 3889 (subid 142393)
% Their weight is constant at 70, then becomes 0.6 at 467 min
[idxRem2,N] = pnPreprocessRemovalIndices(tmp, idx, 35, 'lt');
if verboseFlag; fprintf('Deleted %2.0f values which were less than 35.\n',N); end;
[idxRem3,N] = pnPreprocessRemovalIndices(tmp, idx, 299, 'gt');
if verboseFlag; fprintf('Deleted %2.0f values which were greater than 299.\n',N); end;
idxRem = cellfun(@(f,a,t) f | a | t, idxRem, idxRem2, idxRem3, 'UniformOutput',false);
end
%=== Replace data with tmp
for m=2:size(data,2)
data_processed(:,m) = cellfun(@pnPreprocessReplaceData, data_processed(:,m), tmp(:,m), idx, 'UniformOutput', false);
end
%=== Remove deleted data from 'data'
if ~isempty(idxRem)
for m=2:size(data,2)
data_processed(:,m) = cellfun(@pnPreprocessDeleteData, data_processed(:,m), idxRem, 'UniformOutput', false);
end
end
end
end
end
function [data] = pnPreprocessDeleteData(data,idx)
%PNPREPROCESSDELETEDATA Deletes data in "data" at idx (used with cellfun)
% [ data ] = pnPreprocessDeleteData(data,idx)
%
%
% Inputs:
% data - Vector of data
% idx - Index for data to be removed
%
% Outputs:
% data - Data with elements at idx deleted
%
%
% Example
% data(:,m) = cellfun(@pnPreprocessDeleteData, data(:,m), idx, 'UniformOutput', false);
%
% See also PNPREPROCESSDATA
% Copyright 2012 Alistair Johnson
data(idx)=[];
end
function [idxOut, N] = pnPreprocessRemovalIndices(tmp, idx, val, equalityFcnStr)
%PNPREPROCESSDELETEDATA Generates indices to be used to remove data
% [idxOut] = pnPreprocessRemovalIndices(tmp, idx, val, equalityFcnStr)
% calculates the indices of data which satisfies some condition
% specified by equalityFcnStr and val. For example, if equalityFcnStr
% is 'eq' and val is 0, then the function locates the indices of tmp
% which contain data equal to 0. These indices are then translated
% into indices in the original data cell, to be used for data
% deletion at a later point in PNPREPROCESSDATA.
%
% [idxOut,N] = pnPreprocessRemovalIndices(tmp, idx, val, equalityFcnStr)
% also outputs the number of entries being deleted.
%
% Inputs:
% tmp - Cell array with data only from a given field
% idx - Indices that were used to extract tmp from the original
% data cell array
% val - A value used for comparison with tmp
% equalityFcnStr - The function used to compare val to data in tmp
%
% Outputs:
% idxOut - Indices of data cell array to be deleted
% N - Number of entries to be deleted
%
% Example:
% data = pnLoadTextFilesCell([bpath 'set-a']);
% [tmp,idx] = pnExtractField(data,'HR');
% [idxRem,N] = pnPreprocessRemovalIndices(tmp, idx, 0, 'eq');
% for m=2:4
% data(:,m) = cellfun(@pnPreprocessDeleteData,...
% data(:,m), idxRem, 'UniformOutput', false);
% end
% fprintf('Deleted %2.0f values which were 0.\n',N);
%
%
% See also PNPREPROCESSDATA
% Copyright 2012 Alistair Johnson
fcnTmp = str2func(equalityFcnStr);
idxRem = cellfun(@(x) fcnTmp(x,val), tmp(:,4),'UniformOutput',false);
idxOut = cellfun(@pnSubfcnRemIdx, idx, idxRem, 'UniformOutput',false);
N = sum(cell2mat(cellfun(@(x) sum(x), idxOut, 'UniformOutput', false)));
end
function [idx] = pnSubfcnRemIdx(idx, idxRem)
idx(idx) = idxRem;
end
function [data] = pnPreprocessReplaceData(data,tmp,idx)
%PNPREPROCESSREPLACEDATA Replace data in "data" with "tmp"
% [data] = pnPreprocessReplaceData(data,tmp,idx) replaces values in data
% with values in tmp using indices idx to map tmp to data. This function
% is used with cellfun in PNPREPROCESSDATA.
%
%
% Inputs:
% data - Vector of data
% tmp - Vector of data after some form of preprocessing
% idx - Indices mapping tmp into data
%
% Outputs:
% data - Vector of data with tmp imputed within
%
%
% Example
% data(:,m) = cellfun(@pnReplaceData, data, tmp, idx, 'UniformOutput', false);
%
% See also PNPREPROCESSDATA
data(idx)=tmp;
end
function [] = pnPreprocessPrintReplacedValues(orig,new,idx)
%PNPREPROCESSPRINTREPLACEDVALUES Prints values which were modified to
%command window.
% pnPreprocessReplaceData(new,orig,idx) prints values replaced in the
% given preprocessing step followed by their new value
%
% Inputs:
% new - Vector of data after some form of preprocessing
% orig - Vector of data
% idx - Indices mapping tmp into data
%
% Outputs:
% List of values which were changed
%
% Example
% data(:,m) = cellfun(@pnReplaceData, data, tmp, idx, 'UniformOutput', false);
%
% See also PNPREPROCESSDATA
%=== Extract cell indices which contain changed data
idxChanged = find(cellfun(@(x) ~isempty(x) && any(x), idx)==1);
for k=1:numel(idxChanged)
idxChangedTemp = find(idx{idxChanged(k)}==1);
for m=1:numel(idxChangedTemp)
fprintf('%3.2f --> %3.2f\n',...
orig{idxChanged(k)}(idxChangedTemp(m)),...
new{idxChanged(k)}(m));
end
end
end