Commit e7de2bf3 authored by R.W.Majeed's avatar R.W.Majeed

clean implementation of input value processing

parent 327d8715
Parsing of columns
------------------
Columns are parsed as follows:
1. If a @constant-value attribute is present, that value is used. Jump directly to 5 (@na processing)
2. Otherwise, the actual column value is used
3. Regular expression substitution is performed if present via @regex-replace (may change the value)
4. Map rules are executed if present via map/case and map/otherwise elements. These rules may change the value (and concept code)
5. If the @na attribute is present and equals the calculated value, the value is removed completely
......@@ -63,61 +63,90 @@ public abstract class Column<T> {
*/
public String getName(){return column;}
public abstract T valueOf(Object input) throws ParseException;
/**
* Create the column type value from a string representation.
* This will be used for the {@code constant-value} attribute is present
* and also to process data from string only sources like CSV or text tables.
* <p>
* The resulting type depends on the type attribute and can be one
* of Long, BigDecimal, String, DateTime or DateTimeAccuracy (for incomplete dates).
*
* @param input string value e.g. from text table column. This parameter is guaranteed to be non null.
* @return column output type representing the input value
* @throws ParseException if the string could not be parsed to the resulting data type
*/
public abstract T valueFromString(String input)throws ParseException;
/**
* Convert a string input value to the output data type. The resulting type depends
* on the type attribute and can be one of Long, BigDecimal, String, DateTime
* or DateTimeAccuracy (for incomplete dates).
* Convert another data type to the column data type.
*
* @param input input value e.g. from SQL result set.
* This parameter is guaranteed to be non null, since null values
* are handled before this method is called.
* <p>
* TODO: how to read SQL table data, which already contains types (e.g. sql.Integer)
* This parameter is also guaranteed not to be of String type,
* since strings are handled via {@link #valueFromString(String)}
*
* @param value input value. e.g. from text table column
* @return output type representing the input value
* @throws ParseException on errors with regular expressions
* @return column data type
* @throws ParseException if conversion of data types failed
*/
public Object preprocessValue(Object value)throws ParseException{
// use constant value if provided
if( constantValue != null ){
value = constantValue;
}
// apply regular expression replacements
if( value != null && regexReplace != null ){
value = applyRegexReplace((String)value);
}
// apply map rules
if( map != null ){
// TODO apply map rules
// TODO find way to communicate warnings
// TODO find way to set action (inplace/drop/generate)
}
// check for na result
if( na != null && value != null && na.equals(value) ){
value = null;
}
return value;
}
public abstract T valueOf(Object input) throws ParseException;
private String applyRegexReplace(String value){
// TODO apply replace
return value;
}
public T valueOf(ColumnMap map, Object[] row) throws ParseException{
if( column == null || column.isEmpty() ){
// use constant value if available
return valueOf(null);
public T valueOf(ColumnMap colMap, Object[] row) throws ParseException{
T ret;
// use constant value if available
if( constantValue != null ){
// check for NA
if( na != null && na.equals(constantValue) ){
ret = null; // will result in null value
}else{
ret = valueFromString(constantValue); // use constant value
}
}else if( column == null || column.isEmpty() ){
// no constant value and column undefined
// the column will always produce null values
ret = null;
}else{
// use actual row value
Objects.requireNonNull(colMap);
Objects.requireNonNull(row);
Integer index = colMap.indexOf(this);
Objects.requireNonNull(index);
Object rowval = row[index];
// string processing (na, regex-replace, mapping) only performed on string values
if( rowval == null ){
ret = null; // null value
}else if( rowval.getClass() == String.class ){
// non null string value
String val = (String)rowval;
// apply regular expression replacements
if( regexReplace != null ){
val = applyRegexReplace(val);
}
// TODO apply map rules
// check for NA
if( na != null && val != null && na.equals(val) ){
val = null;
}
// convert value
if( val != null ){
ret = valueFromString(val);
}else{
ret = null;
}
}else if( na != null || regexReplace != null || map != null ){
throw new ParseException("String operation (na/regexReplace/map) defined for column "+getName()+", but table provides type "+rowval.getClass().getName()+" instead of String");
}else{
// other non string value without string processing
ret = valueOf(rowval); // use value directly
}
}
Objects.requireNonNull(map);
Objects.requireNonNull(row);
Integer index = map.indexOf(this);
Objects.requireNonNull(index);
return this.valueOf(row[index]);
return ret;
}
public void validate()throws ParseException{
......
......@@ -59,6 +59,18 @@ public class Concept{
this.start = new DateTimeColumn(startColumn, format);
}
/**
* Create an observation for this concept with the given row data.
* TODO allow mapping actions to happen at this place, e.g. drop concept, log warning, change value
*
* @param patid patient id
* @param visit visit id
* @param factory observation factory
* @param map column map
* @param row row data
* @return fact
* @throws ParseException parse
*/
protected Observation createObservation(String patid, String visit, ObservationFactory factory, ColumnMap map, Object[] row) throws ParseException{
DateTimeAccuracy start = this.start.valueOf(map,row);
Observation o = factory.createObservation(patid, this.id, start);
......
......@@ -39,28 +39,28 @@ public class DateTimeColumn extends Column<DateTimeAccuracy>{
@Override
public DateTimeAccuracy valueOf(Object value) throws ParseException{
value = super.preprocessValue(value);
if( value == null ){
return null;
}else if( value instanceof String ){
// parse date according to format
if( formatter == null && format != null ){
formatter = DateTimeFormatter.ofPattern(format);
}
if( formatter == null ){
throw new ParseException("format must be specified for DateTime fields if strings are parsed");
}
// TODO parse
try{
return DateTimeAccuracy.parse(formatter,(String)value);
}catch( DateTimeParseException e ){
throw new ParseException("Unable to parse date: "+(String)value, e);
}
}else if( value instanceof Timestamp ){
if( value instanceof Timestamp ){
// convert from timestamp
return null;
}else{
throw new IllegalArgumentException("Don't know how to parse type "+value.getClass()+" to datetime");
}
}
@Override
public DateTimeAccuracy valueFromString(String input) throws ParseException {
// parse date according to format
if( formatter == null && format != null ){
formatter = DateTimeFormatter.ofPattern(format);
}
if( formatter == null ){
throw new ParseException("format must be specified for DateTime fields if strings are parsed");
}
// parse
try{
return DateTimeAccuracy.parse(formatter,input);
}catch( DateTimeParseException e ){
throw new ParseException("Unable to parse date: "+input, e);
}
}
}
\ No newline at end of file
......@@ -23,33 +23,33 @@ public class DecimalColumn extends Column<BigDecimal>{
@Override
public BigDecimal valueOf(Object input) throws ParseException {
Object value = preprocessValue(input);
if( value == null ){
return null;
}else if( value instanceof String ){
if( locale == null ){
// parse according to BigDecimal(String)
try{
return new BigDecimal((String)value);
}catch( NumberFormatException e ){
throw new ParseException("Unable to parse number: "+(String)value, e);
}
}else{
// use DecimalFormat for parsing
if( decimalFormat == null ){
decimalFormat = (DecimalFormat)NumberFormat.getNumberInstance(Locale.forLanguageTag(locale));
decimalFormat.setParseBigDecimal(true);
}
try {
return (BigDecimal)decimalFormat.parse((String)value);
} catch (java.text.ParseException e) {
throw new ParseException("Unable to parse number: "+(String)value, e);
}
if( input instanceof BigDecimal ){
return (BigDecimal)input;
}else{
throw new ParseException("Invalid type for decimal column: "+input.getClass().getName());
}
}
@Override
public BigDecimal valueFromString(String input) throws ParseException {
if( locale == null ){
// parse according to BigDecimal(String)
try{
return new BigDecimal(input);
}catch( NumberFormatException e ){
throw new ParseException("Unable to parse number '"+input+"'", e);
}
}else if( value instanceof BigDecimal ){
return (BigDecimal)value;
}else{
throw new ParseException("Invalid type for decimal column: "+value.getClass().getName());
// use DecimalFormat for parsing
if( decimalFormat == null ){
decimalFormat = (DecimalFormat)NumberFormat.getNumberInstance(Locale.forLanguageTag(locale));
decimalFormat.setParseBigDecimal(true);
}
try {
return (BigDecimal)decimalFormat.parse(input);
} catch (java.text.ParseException e) {
throw new ParseException("Unable to parse number '"+input+"'", e);
}
}
}
......
......@@ -9,13 +9,7 @@ public class IntegerColumn extends Column<Long> {
@Override
public Long valueOf(Object input) throws ParseException {
input = preprocessValue(input);
if( input == null ){
return null;
}else if( input instanceof String ){
// TODO: use integerformat for parsing
return Long.parseLong((String)input);
}else if( input instanceof Integer ){
if( input instanceof Integer ){
return new Long((Integer)input);
}else if( input instanceof Long ){
return (Long)input;
......@@ -23,5 +17,10 @@ public class IntegerColumn extends Column<Long> {
throw new ParseException("Unsupported input type "+input.getClass().getName());
}
}
@Override
public Long valueFromString(String input) throws ParseException {
return Long.parseLong(input);
}
}
......@@ -15,9 +15,11 @@ public class StringColumn extends Column<String>{
}
@Override
public String valueOf(Object input) throws ParseException {
Object value = preprocessValue(input);
if( value != null )return value.toString();
else return null;
return input.toString();
}
@Override
public String valueFromString(String input) throws ParseException {
return input; // nothing to do
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment