Commit 2c91b2b9 authored by R.W.Majeed's avatar R.W.Majeed

mapping of eav facts via virtual value columns

parent e7de2bf3
......@@ -22,6 +22,9 @@ public class EavRow implements FactRow {
public List<Observation> getFacts() {
return Arrays.asList(fact);
}
public Observation getFact(){
return fact;
}
@Override
public String getPatientId() {return fact.getPatientId();}
......
package de.sekmi.histream.etl;
public class MapFeedback {
private boolean drop;
private String concept;
private String value;
public MapFeedback(){
drop = false;
}
public void overrideConcept(String newConcept){
this.concept = newConcept;
}
public void overrideValue(String newValue){
this.value = newValue;
}
public void logWarning(String warning){
// TODO forward to parse warning handler
System.err.println("Map warning: "+warning);
}
// drop the fact, no observation will be generated
public void dropFact(){
drop = true;
}
public String getValueOverride(){
return value;
}
public boolean hasConceptOverride(){
return concept != null;
}
public String getConceptOverride(){
return concept;
}
public boolean isActionDrop(){
return drop;
}
}
......@@ -8,6 +8,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import de.sekmi.histream.etl.ColumnMap;
import de.sekmi.histream.etl.MapFeedback;
import de.sekmi.histream.etl.ParseException;
/**
......@@ -93,11 +94,92 @@ public abstract class Column<T> {
public abstract T valueOf(Object input) throws ParseException;
private String applyRegexReplace(String value){
// TODO apply replace
return value;
throw new UnsupportedOperationException("Not yet implemented");
}
private void applyMapRules(String value, MapFeedback action){
boolean match = false;
Objects.requireNonNull(map.cases);
for( MapCase mc : map.cases ){
Objects.requireNonNull(mc.value);
if( mc.value.equals(value) ){
match = true;
if( mc.setValue != null ){
action.overrideValue(mc.setValue);
}
// TODO check action
if( mc.setConcept != null ){
action.overrideConcept(mc.setConcept);
}
break;
}
}
if( match == false && map.otherwise != null ){
if( map.otherwise.setValue != null ){
action.overrideValue(map.otherwise.setValue);
}
if( map.otherwise.setConcept != null ){
action.overrideConcept(map.otherwise.setConcept);
}
// TODO check action
}
}
/**
* Process and return the column value from a table row without map rule processing.
* This method behaves as if {@link #valueOf(ColumnMap, Object[], MapFeedback)} was called
* with the last argument set to {@code null}.
*
* @see #valueOf(Object)
* @param colMap column map
* @param row table row
* @return value
* @throws ParseException parse errors
*/
public T valueOf(ColumnMap colMap, Object[] row) throws ParseException{
return valueOf(colMap, row, null);
}
private T processedValue(String val, MapFeedback mapFeedback) throws ParseException{
T ret;
// apply regular expression replacements
if( regexReplace != null ){
val = applyRegexReplace(val);
}
// apply map rules
if( map != null ){
if( mapFeedback == null ){
throw new ParseException("map element allowed for column "+getName());
}
applyMapRules(val, mapFeedback);
// use value override, if present
if( mapFeedback.getValueOverride() != null ){
val = mapFeedback.getValueOverride();
}
}
// check for NA
if( na != null && val != null && na.equals(val) ){
val = null;
}
// convert value
if( val != null ){
ret = valueFromString(val);
}else{
ret = null;
}
return ret;
}
/**
* Process and return the column value from a table row.
*
* @param colMap column map
* @param row table row
* @param mapFeedback map rule feedback, can be set to {@code null} if map rules forbidden for this column.
* @return final column value
* @throws ParseException parse errors
*/
public T valueOf(ColumnMap colMap, Object[] row, MapFeedback mapFeedback) throws ParseException{
T ret;
// use constant value if available
if( constantValue != null ){
......@@ -111,8 +193,10 @@ public abstract class Column<T> {
// no constant value and column undefined
// the column will always produce null values
ret = null;
// this should not happen -> concept neither constant value nor column name
}else{
// use actual row value
// TODO merge with valueOf(Object,MapFeedback), but colmap lookup shall not occur for constant values
Objects.requireNonNull(colMap);
Objects.requireNonNull(row);
Integer index = colMap.indexOf(this);
......@@ -121,26 +205,11 @@ public abstract class Column<T> {
// string processing (na, regex-replace, mapping) only performed on string values
if( rowval == null ){
ret = null; // null value
}else if( rowval.getClass() == String.class ){
}else if( rowval instanceof String ){
// non null string value
String val = (String)rowval;
// apply regular expression replacements
if( regexReplace != null ){
val = applyRegexReplace(val);
}
// TODO apply map rules
// check for NA
if( na != null && val != null && na.equals(val) ){
val = null;
}
// convert value
if( val != null ){
ret = valueFromString(val);
}else{
ret = null;
}
ret = processedValue((String)rowval, mapFeedback);
}else if( na != null || regexReplace != null || map != null ){
throw new ParseException("String operation (na/regexReplace/map) defined for column "+getName()+", but table provides type "+rowval.getClass().getName()+" instead of String");
throw new ParseException("String operation (na/regexReplace/map) defined for column "+getName()+", but table source provides type "+rowval.getClass().getName()+" instead of String");
}else{
// other non string value without string processing
ret = valueOf(rowval); // use value directly
......@@ -149,6 +218,40 @@ public abstract class Column<T> {
return ret;
}
/**
* Process and return the column value from a table row.
* Same as {@link #valueOf(ColumnMap, Object[], MapFeedback)} but without
* lookup with column map.
* TODO merge both methods
*
* @param rowval value from row
* @param mapFeedback mapping feedback
* @return processed value
* @throws ParseException parse error
*/
public T valueOf(Object rowval, MapFeedback mapFeedback) throws ParseException{
T ret;
// use constant value if available
if( constantValue != null ){
// check for NA
if( na != null && na.equals(constantValue) ){
ret = null; // will result in null value
}else{
ret = valueFromString(constantValue); // use constant value
}
}else if( rowval == null ){
ret = null;
}else if( rowval instanceof String ){
ret = processedValue((String)rowval, mapFeedback);
}else if( na != null || regexReplace != null || map != null ){
throw new ParseException("String operation (na/regexReplace/map) defined for column "+getName()+", but table source provides type "+rowval.getClass().getName()+" instead of String");
}else{
// other non string value without string processing
ret = valueOf(rowval); // use value directly
}
return ret;
}
public void validate()throws ParseException{
if( column == null && constantValue == null ){
throw new ParseException("Empty column name only allowed if constant-value is specified");
......
......@@ -11,6 +11,7 @@ import de.sekmi.histream.DateTimeAccuracy;
import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.etl.ColumnMap;
import de.sekmi.histream.etl.MapFeedback;
import de.sekmi.histream.etl.ParseException;
import de.sekmi.histream.impl.NumericValue;
import de.sekmi.histream.impl.StringValue;
......@@ -73,11 +74,21 @@ public class Concept{
*/
protected Observation createObservation(String patid, String visit, ObservationFactory factory, ColumnMap map, Object[] row) throws ParseException{
DateTimeAccuracy start = this.start.valueOf(map,row);
Observation o = factory.createObservation(patid, this.id, start);
String concept = this.id;
MapFeedback mf = new MapFeedback();
Object value = this.value.valueOf(map, row, mf);
if( mf.isActionDrop() ){
return null; // ignore this fact
}
if( mf.hasConceptOverride() ){
concept = mf.getConceptOverride();
}
Observation o = factory.createObservation(patid, concept, start);
if( visit != null ){
o.setEncounterId(visit);
}
Object value = this.value.valueOf(map, row);
String unit = null;
if( this.unit != null ){
unit = this.unit.valueOf(map, row);
......
package de.sekmi.histream.etl.config;
import java.math.BigDecimal;
import java.util.HashMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
import de.sekmi.histream.DateTimeAccuracy;
import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.Value;
import de.sekmi.histream.etl.ColumnMap;
import de.sekmi.histream.etl.EavRow;
import de.sekmi.histream.etl.MapFeedback;
import de.sekmi.histream.etl.ParseException;
import de.sekmi.histream.impl.NumericValue;
import de.sekmi.histream.impl.StringValue;
......@@ -25,6 +29,29 @@ public class EavTable extends Table<EavRow> {
@XmlElement
MDAT mdat;
@XmlTransient
Map<String,Column<?>> virtualColumnMap;
@XmlElementWrapper(name="virtual")
@XmlElement(name="value")
public void setVirtualValueColumns(Column<?>[] values){
if( values == null ){
virtualColumnMap = null;
}else{
virtualColumnMap = new HashMap<>();
for( Column<?> value : values ){
virtualColumnMap.put(value.column, value);
}
}
}
public Column<?>[] getVirtualValueColumns(){
if( virtualColumnMap == null ){
return null;
}else{
return virtualColumnMap.values().toArray(new Column<?>[virtualColumnMap.size()]);
}
}
@XmlType(name="eav-mdat")
@XmlAccessorType(XmlAccessType.FIELD)
public static class MDAT{
......@@ -78,42 +105,80 @@ public class EavTable extends Table<EavRow> {
return map;
}
private Column<?> getVirtualColumn(String concept){
return virtualColumnMap.get(concept);
}
@Override
public EavRow fillRecord(ColumnMap map, Object[] row, ObservationFactory factory) throws ParseException {
String patid = idat.patientId.valueOf(map, row);
DateTimeAccuracy start = mdat.start.valueOf(map,row);
String concept = mdat.concept.valueOf(map,row);
Observation fact = factory.createObservation(patid, concept, start);
String visit = idat.visitId.valueOf(map, row);
if( visit != null ){
fact.setEncounterId(visit);
}
String value = mdat.value.valueOf(map,row);
if( value != null ){
// generate/parse value
public EavRow fillRecord(ColumnMap colMap, Object[] row, ObservationFactory factory) throws ParseException {
String patid = idat.patientId.valueOf(colMap, row);
DateTimeAccuracy start = mdat.start.valueOf(colMap,row);
String concept = mdat.concept.valueOf(colMap,row);
String value = mdat.value.valueOf(colMap,row);
String unit = mdat.unit.valueOf(colMap,row);
Column<?> vcol = getVirtualColumn(concept);
Object vval;
if( vcol != null ){
// use virtual column for value processing
MapFeedback mf = new MapFeedback();
vval = vcol.valueOf(value, mf);
if( mf.hasConceptOverride() ){
concept = mf.getConceptOverride();
}
if( mf.isActionDrop() ){
return null; // ignore fact and row
}
}else if( value != null ){
// no virtual column provided, parse value directly
// use provided type info
String type = null;
if( mdat.type != null ){
type = mdat.type.valueOf(map,row);
type = mdat.type.valueOf(colMap,row);
}
Value factValue = null;
if( type == null ){
// for now, use string
// TODO determine type automatically from string representation
factValue = new StringValue(value);
vval = value;
}else if( type.equals(StringColumn.class.getAnnotation(XmlType.class).name()) ){
factValue = new StringValue(value);
}else if( type.equals(DecimalColumn.class.getAnnotation(XmlType.class).name())
|| type.equals(IntegerColumn.class.getAnnotation(XmlType.class).name()) ){
vval = value;
}else if( type.equals(DecimalColumn.class.getAnnotation(XmlType.class).name()) ){
try{
factValue = new NumericValue(new BigDecimal(value));
vval = new BigDecimal(value);
}catch( NumberFormatException e ){
throw new ParseException("Unable to parse number", e);
}
}else if( type.equals(IntegerColumn.class.getAnnotation(XmlType.class).name()) ){
try{
vval = Long.parseLong(value);
}catch( NumberFormatException e ){
throw new ParseException("Unable to parse integer", e);
}
}else{
throw new ParseException("Unsupported value type: "+type);
}
fact.setValue(factValue);
}else{
// null value
vval = null;
}
Observation fact = factory.createObservation(patid, concept, start);
String visit = idat.visitId.valueOf(colMap, row);
if( visit != null ){
fact.setEncounterId(visit);
}
if( vval != null ){
// convert native type to observation value
if( vval instanceof String ){
fact.setValue(new StringValue((String)vval));
}else if( vval instanceof BigDecimal ){
fact.setValue(new NumericValue((BigDecimal)vval,unit));
}else if( vval instanceof Long ){
fact.setValue(new NumericValue((Long)vval,unit));
}else{
throw new ParseException("Internal error: unsupported native value type: "+vval.getClass());
}
}// else fact without value
return new EavRow(fact);
}
......
......@@ -2,22 +2,24 @@ package de.sekmi.histream.etl.config;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlAttribute;
@XmlAccessorType(XmlAccessType.FIELD)
@XmlAccessorType(XmlAccessType.NONE)
public class MapCase {
@XmlAttribute
String value;
@XmlElement(name="set-value")
@XmlAttribute(name="set-value")
String setValue;
@XmlElement(name="set-concept")
@XmlAttribute(name="set-concept")
String setConcept;
// TODO use enum
@XmlAttribute
String action;
@XmlElement(name="log-warning")
@XmlAttribute(name="log-warning")
String logWarning;
}
package de.sekmi.histream.etl.config;
import java.util.Arrays;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
......@@ -9,8 +11,10 @@ import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.etl.ColumnMap;
import de.sekmi.histream.etl.ConceptTable;
import de.sekmi.histream.etl.MapFeedback;
import de.sekmi.histream.etl.ParseException;
import de.sekmi.histream.etl.PatientRow;
import de.sekmi.histream.ext.Patient.Sex;
/**
* Patient table. Contains patient id and other identifying information.
......@@ -73,12 +77,27 @@ public class PatientTable extends Table<PatientRow> implements ConceptTable{
@Override
public PatientRow fillRecord(ColumnMap map, Object[] row, ObservationFactory factory) throws ParseException {
PatientRow patient = new PatientRow();
patient.setId(idat.patientId.valueOf(map, row));
patient.setGivenName(idat.givenName.valueOf(map, row));
patient.setSurname(idat.surname.valueOf(map, row));
patient.setBirthDate(idat.birthdate.valueOf(map, row));
patient.setId(idat.patientId.valueOf(map, row, null));
patient.setGivenName(idat.givenName.valueOf(map, row, null));
patient.setSurname(idat.surname.valueOf(map, row, null));
patient.setBirthDate(idat.birthdate.valueOf(map, row, null));
if( idat.deathdate != null ){
patient.setDeathDate(idat.deathdate.valueOf(map, row));
patient.setDeathDate(idat.deathdate.valueOf(map, row, null));
}
if( idat.gender != null ){
MapFeedback mf = new MapFeedback();
String genderCode = idat.gender.valueOf(map, row, mf);
if( mf.isActionDrop() || mf.getConceptOverride() != null ){
throw new ParseException("concept override or drop not allowed for patient gender");
}
// gender may omitted
if( genderCode != null ){
try{
patient.setSex(Sex.valueOf(genderCode));
}catch( IllegalArgumentException e ){
throw new ParseException("Unsupported gender value '"+genderCode+"'. Use one of "+Arrays.toString(Sex.values()));
}
}
}
// concepts
if( concepts != null ){
......
......@@ -32,7 +32,8 @@ public class TestMarshall {
Assert.assertEquals("geschlecht",ds.patientTable.idat.gender.column);
// check gender mapping
Assert.assertNotNull(ds.patientTable.idat.gender.map);
Assert.assertEquals(2,ds.patientTable.idat.gender.map.cases.length);
Assert.assertEquals(3,ds.patientTable.idat.gender.map.cases.length);
Assert.assertEquals("W",ds.patientTable.idat.gender.map.cases[0].value);
Assert.assertEquals("vorname",ds.patientTable.idat.givenName.column);
Assert.assertEquals("nachname",ds.patientTable.idat.surname.column);
......@@ -58,6 +59,11 @@ public class TestMarshall {
Assert.assertEquals("natrium", c.id);
Assert.assertEquals("na", c.value.column);
Assert.assertEquals("mmol/l", c.unit.constantValue);
// check eav
Assert.assertEquals(1, ds.eavTables.length);
Assert.assertNotNull(ds.eavTables[0].virtualColumnMap);
Assert.assertNotNull(ds.eavTables[0].virtualColumnMap.get("f_eav_x"));
}
}
@Test
......
......@@ -84,6 +84,14 @@ public class TestReadTables {
Assert.assertNotNull(e);
Assert.assertEquals("test-1", e.getSourceId());
// skip to last
for( EavRow n = s.get(); n!=null; n=s.get() ){
r = n;
}
Observation f = r.getFact();
// should be processed by virtual column map
Assert.assertEquals("f_eav_x_1", f.getConceptId());
Assert.assertNull(f.getValue());
}
}
}
......@@ -16,10 +16,11 @@
<surname column="nachname"/>
<birthdate format="d.M.u" na="" column="geburtsdatum"/>
<deathdate format="d.M.u" na="" column="verstorben"/>
<gender column="geschlecht">
<gender column="geschlecht" na="">
<map> <!-- maps a column -->
<case value="W" set-value="F"/>
<case value="M" set-value="M"/>
<case value="W" set-value="female"/>
<case value="M" set-value="male"/>
<case value="X" set-value="indeterminate"/>
<otherwise set-value="" log-warning="Unexpected gender value"/>
</map>
</gender>
......@@ -107,17 +108,21 @@
<value column="value" na="@"/>
<unit column="unit" na="@"/>
</mdat>
<apply-mapping>
<!-- virtual value columns -->
<virtual>
<!-- for eav facts, the map always applies to the fact value -->
<map concept="f_eav_x">
<case value="1" set-value="" set-concept="f_eav_x_1"/>
<!-- action inplace is default -->
<case value="0" set-value="" set-concept="f_eav_x_0" action="inplace" />
<!-- action drop will not produce any fact -->
<otherwise log-warning="Unexpected value" action="drop-fact" />
<!-- action generate will produce a new concept with the given values -->
</map>
</apply-mapping>
<value column="f_eav_x" na="" xsi:type="string">
<map>
<case value="1" set-value="" set-concept="f_eav_x_1"/>
<!-- action inplace is default -->
<case value="0" set-value="" set-concept="f_eav_x_0"/>
<!-- action drop will not produce any fact -->
<otherwise log-warning="Unexpected value" action="drop-fact" />
<!-- action generate will produce a new concept with the given values -->
</map>
</value>
<!-- ... more value elements -->
</virtual>
</eav-table>
</datasource>
patid ignoriert1 nachname vorname geburtsdatum verstorben geschlecht patfakt1
p1 a n1 v1 01.02.2003 11.02.2003 F c
p1 a n1 v1 01.02.2003 11.02.2003 W c
p2 b n2 v2 02.03.2004 M d
p3 c n3 v3 03.04.2005 F e
\ No newline at end of file
p3 c n3 v3 03.04.2005 W e
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment