histream-core/examples/dwh-eav.xml

@@ -37,13 +37,14 @@
 		<birthdate>2001-01-01</birthdate>
 		<deathdate>2020</deathdate>
 		<sex>F</sex>
-		<encounter start="2014-01-01T10:30:00" end="2014-01-05T10:30:00">XXE12345</encounter>
+		<encounter start="2014-01-01T10:30:00" end="2014-01-05T10:30:00.123">XXE12345</encounter>
 		<location>Zuhause</location>
 		<!-- TODO inpatient/outpatient -->
 		<provider>xxxa</provider>
 		<facts>
 			<!--  test parsing of partial time stamps -->
 			<eav-item concept="T:date:secs" start="2014-09-07T10:40:03"/>
+			<eav-item concept="T:date:msec" start="2014-09-07T10:40:03.123"/>
 			<eav-item concept="T:date:mins" start="2014-09-07T10:40"/>
 			<eav-item concept="T:date:hours" start="2014-09-07T10"/>
 			<eav-item concept="T:date:day" start="2014-09-07"/>

histream-core/examples/dwh-flat.txt

@@ -23,9 +23,10 @@ XX12345	XXE12345	gest	dat	2020
 XX12345	XXE12345	sex	str	F
 XX12345	XXE12345	psurname	str	Dampf
 XX12345	XXE12345	pnames	str	A B
-XX12345	XXE12345	visit	@	@	@	2014-01-01T10:30:00	2014-01-05T10:30:00	xxxa	Zuhause
+XX12345	XXE12345	visit	@	@	@	2014-01-01T10:30:00	2014-01-05T10:30:00.123	xxxa	Zuhause
 # normale werte
 XX12345	XXE12345	T:date:secs	@	@	@	2014-09-07T10:40:03
+XX12345	XXE12345	T:date:msec	@	@	@	2014-09-07T10:40:03.123
 XX12345	XXE12345	T:date:mins	@	@	@	2014-09-07T10:40
 XX12345	XXE12345	T:date:hours	@	@	@	2014-09-07T10
 XX12345	XXE12345	T:date:day	@	@	@	2014-09-07

histream-core/src/main/java/de/sekmi/histream/DateTimeAccuracy.java

@@ -51,7 +51,7 @@ import de.sekmi.histream.xml.DateTimeAccuracyAdapter;
  */
 @XmlJavaTypeAdapter(DateTimeAccuracyAdapter.class)
 public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
-	static final String PARTIAL_FORMATTER_PATTERN = "u[-M[-d['T'H[:m[:s[.S]]][X]]]]";
+	static final String PARTIAL_FORMATTER_PATTERN = "u[-M[-d['T'H[:m[:s[.SSS]]][X]]]]";
 	static final DateTimeFormatter PARTIAL_FORMATTER  = DateTimeFormatter.ofPattern(PARTIAL_FORMATTER_PATTERN);
 
 	// TODO why not use instant, since we always calculate UTC? or Offset/ZonedDateTime?
@@ -67,33 +67,33 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 		this.accuracy = ChronoUnit.SECONDS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year) {
-		instant = LocalDateTime.of(year, 1, 1, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year) {
+		instant = LocalDateTime.of(year, 1, 1, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.YEARS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month) {
-		instant = LocalDateTime.of(year, month, 1, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month) {
+		instant = LocalDateTime.of(year, month, 1, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.MONTHS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day) {
-		instant = LocalDateTime.of(year, month, day, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day) {
+		instant = LocalDateTime.of(year, month, day, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.DAYS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours) {
-		instant = LocalDateTime.of(year, month, day, hours, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours) {
+		instant = LocalDateTime.of(year, month, day, hours, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.HOURS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours, int mins) {
-		instant = LocalDateTime.of(year, month, day, hours, mins).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours, int mins) {
+		instant = LocalDateTime.of(year, month, day, hours, mins).atZone(zone).toInstant();
 		accuracy = ChronoUnit.MINUTES;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours, int mins, int secs) {
-		instant = LocalDateTime.of(year, month, day, hours, mins, secs).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours, int mins, int secs) {
+		instant = LocalDateTime.of(year, month, day, hours, mins, secs).atZone(zone).toInstant();
 		accuracy = ChronoUnit.SECONDS;
 	}
 	
@@ -201,9 +201,9 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 			dt = instant.atOffset(ZoneOffset.UTC).toLocalDateTime();
 		}
 		
-		char[] prefixes = {0,'-','-','T',':',':'};
-		ChronoField[] fields = {ChronoField.YEAR, ChronoField.MONTH_OF_YEAR, ChronoField.DAY_OF_MONTH, ChronoField.HOUR_OF_DAY, ChronoField.MINUTE_OF_HOUR, ChronoField.SECOND_OF_MINUTE};
-		int[] digits = {4,2,2,2,2,2};
+		char[] prefixes = {0,'-','-','T',':',':','.'};
+		ChronoField[] fields = {ChronoField.YEAR, ChronoField.MONTH_OF_YEAR, ChronoField.DAY_OF_MONTH, ChronoField.HOUR_OF_DAY, ChronoField.MINUTE_OF_HOUR, ChronoField.SECOND_OF_MINUTE,ChronoField.MILLI_OF_SECOND};
+		int[] digits = {4,2,2,2,2,2,3};
 		int i;
 		for( i=0; i<fields.length; i++ ){
 			if( prefixes[i] != 0 )b.append(prefixes[i]);
@@ -285,8 +285,9 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 		if( a.isSupported(ChronoField.NANO_OF_SECOND) ){
 			// maximum accuracy of nanoseconds
 			// not supported yet, truncate to seconds
-			accuracy = ChronoUnit.NANOS;
-			dateTime = LocalDateTime.from(a);
+			accuracy = ChronoUnit.MILLIS;
+//			dateTime = LocalDateTime.from(a);
+			dateTime = LocalDateTime.of(a.get(ChronoField.YEAR), a.get(ChronoField.MONTH_OF_YEAR), a.get(ChronoField.DAY_OF_MONTH), a.get(ChronoField.HOUR_OF_DAY), a.get(ChronoField.MINUTE_OF_HOUR), a.get(ChronoField.SECOND_OF_MINUTE), a.get(ChronoField.NANO_OF_SECOND));
 		}else if( a.isSupported(ChronoField.SECOND_OF_MINUTE) ){
 			accuracy = ChronoUnit.SECONDS;
 			dateTime = LocalDateTime.of(a.get(ChronoField.YEAR), a.get(ChronoField.MONTH_OF_YEAR), a.get(ChronoField.DAY_OF_MONTH), a.get(ChronoField.HOUR_OF_DAY), a.get(ChronoField.MINUTE_OF_HOUR), a.get(ChronoField.SECOND_OF_MINUTE));
@@ -334,54 +335,64 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 	 * 
 	 * @param formatter formatter
 	 * @param text input text
+	 * @param zoneId time zone to use, if the parser doesn't supply a time zone or offset
 	 * @return date time with accuracy
 	 */
-	public static DateTimeAccuracy parse(DateTimeFormatter formatter, CharSequence text){
+	public static DateTimeAccuracy parse(DateTimeFormatter formatter, CharSequence text, ZoneId zoneId){
 		ParsePosition pos = new ParsePosition(0);
 		TemporalAccessor a = formatter.parseUnresolved(text, pos);
 		if( pos.getErrorIndex() != -1 ){
-			throw new DateTimeParseException("Text '"+String.valueOf(text.charAt(pos.getErrorIndex()))+"' could not be parsed at index "+pos.getErrorIndex(), text, pos.getErrorIndex());
+			throw new DateTimeParseException("Text '"+text+"' could not be parsed at index "+pos.getErrorIndex(), text, pos.getErrorIndex());
 		}else if( pos.getIndex() != text.length() ){
 			throw new DateTimeParseException("Unparsed text found at index "+pos.getIndex(), text, pos.getIndex());
 		}
+		
+		try{
+			int offset = a.get(ChronoField.OFFSET_SECONDS);
+			// explicit offset specified, use that information
+			zoneId = ZoneOffset.ofTotalSeconds(offset);
+		}catch( DateTimeException e ){
+			// no offset available
+			// use default specified in zoneId param
+		}
 		int year = a.get(ChronoField.YEAR);
 		// month
 		int month;
 		try{
 			month = a.get(ChronoField.MONTH_OF_YEAR);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year);
+			return new DateTimeAccuracy(zoneId, year);
 		}
 		
 		int day;
 		try{
 			day = a.get(ChronoField.DAY_OF_MONTH);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month);
+			return new DateTimeAccuracy(zoneId, year,month);
 		}
 
 		int hour;
 		try{
 			hour = a.get(ChronoField.HOUR_OF_DAY);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day);
+			return new DateTimeAccuracy(zoneId, year,month,day);
 		}
 
 		int minute;
 		try{
 			minute = a.get(ChronoField.MINUTE_OF_HOUR);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day, hour);
+			return new DateTimeAccuracy(zoneId, year,month,day, hour);
 		}
 		
 		int seconds;
 		try{
 			seconds = a.get(ChronoField.SECOND_OF_MINUTE);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day, hour, minute);
+			return new DateTimeAccuracy(zoneId, year,month,day, hour, minute);
 		}
 
-		return new DateTimeAccuracy(year,month,day, hour, minute, seconds);
+		return new DateTimeAccuracy(zoneId, year,month,day, hour, minute, seconds);
 		// milliseconds not supported for now
 	}
 	@Override

histream-core/src/main/java/de/sekmi/histream/impl/SimpleVisitExtension.java

@@ -47,7 +47,7 @@ public class SimpleVisitExtension implements Extension<VisitImpl>{
 		}
 		VisitImpl visit = new VisitImpl();
 		visit.setId((String)args[0]);
-		visit.setPatientId(((Patient)args[1]).getId());
+		visit.setPatient(((Patient)args[1]));
 		ExternalSourceType source = (ExternalSourceType)args[2];
 		visit.setSourceId(source.getSourceId());
 		visit.setSourceTimestamp(source.getSourceTimestamp());

histream-core/src/main/java/de/sekmi/histream/impl/VisitImpl.java

 package de.sekmi.histream.impl;
 
 
+import java.util.Objects;
+
 /*
  * #%L
  * histream
@@ -24,6 +26,7 @@ package de.sekmi.histream.impl;
 
 
 import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.ext.Patient;
 import de.sekmi.histream.ext.StoredExtensionType;
 import de.sekmi.histream.ext.Visit;
 
@@ -34,28 +37,36 @@ public class VisitImpl extends StoredExtensionType implements Visit {
 	private String patientId;
 	private String locationId;
 	
-	public VisitImpl(){
+	/**
+	 * Empty constructor protected, only
+	 * available to overriding classes.
+	 */
+	protected VisitImpl() {
 		
 	}
-	
-	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime, DateTimeAccuracy endTime, Status status){
+	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime){
 		setId(id);
 		this.patientId = patientId;
+		this.startTime = startTime;		
+		markDirty(true);
+	}
+	
+	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime, DateTimeAccuracy endTime, Status status){
+		this(id, patientId, startTime);
 		this.status = status;
-		this.startTime = startTime;
 		this.endTime = endTime;
-		markDirty(true);
 	}
 
 	public String getPatientId(){return patientId;}
 	
-	public void setPatientId(String patientId){
+	public void setPatient(Patient patient){
+		Objects.requireNonNull(patient);
 		// patient id should not be changed normally.
-		this.patientId = patientId;
+		this.patientId = patient.getId();
 		// TODO need to update dirty flag?
 		markDirty(true);
 	}
-	
+
 	@Override
 	public DateTimeAccuracy getStartTime() {
 		return startTime;

histream-core/src/main/java/de/sekmi/histream/io/GroupedXMLWriter.java

@@ -13,6 +13,8 @@ import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamWriter;
 import javax.xml.transform.Result;
 
+import org.w3c.dom.DOMException;
+
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationException;
 import de.sekmi.histream.ext.ExternalSourceType;
@@ -394,7 +396,7 @@ public class GroupedXMLWriter extends GroupedObservationHandler{
 			formatIndent();
 			marshalFactWithContext(observation, observation.getExtension(Visit.class), meta.source);
 			formatNewline();
-		} catch (JAXBException | XMLStreamException e) {
+		} catch (JAXBException | XMLStreamException | DOMException e) {
 			throw new ObservationException(e);
 		}
 		this.observationCount ++;

histream-core/src/main/java/de/sekmi/histream/io/XMLObservationSupplier.java

@@ -36,6 +36,7 @@ import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
 
+// TODO remove this class
 @Deprecated
 public class XMLObservationSupplier extends XMLObservationParser implements ObservationSupplier{
 	//private static final String namespaceURI = "http://sekmi.de/histream/dwh-eav";

histream-core/src/test/java/de/sekmi/histream/TestDateTimeAccuracy.java

@@ -23,7 +23,7 @@ public class TestDateTimeAccuracy {
 	public void testParseYYYYDD(){
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("M.u");
 		String text = "02.2003";
-		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, text);
+		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, text, ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.MONTHS,a.getAccuracy());
 		TemporalAccessor ac = a.toInstantMin().atOffset(ZoneOffset.UTC);
 		Assert.assertEquals(2, ac.get(ChronoField.MONTH_OF_YEAR));
@@ -35,10 +35,10 @@ public class TestDateTimeAccuracy {
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("d.M.u[ H[:m[:s]]]");
 		DateTimeAccuracy a;
 		ZoneId tz = ZoneId.of("Asia/Shanghai"); // China standard time
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003", ZoneOffset.UTC);
 		Assert.assertEquals("2003-02-01", a.toPartialIso8601(null));
 		Assert.assertEquals("2003-02-01", a.toPartialIso8601(tz));
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13", ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.HOURS, a.getAccuracy());
 		Assert.assertEquals("2003-02-01T13", a.toPartialIso8601(null));
 		Assert.assertEquals("2003-02-01T21+0800", a.toPartialIso8601(tz));
@@ -82,6 +82,8 @@ public class TestDateTimeAccuracy {
 		assertEquals(ChronoUnit.MINUTES, a.getAccuracy());
 		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06");
 		assertEquals(ChronoUnit.SECONDS, a.getAccuracy());
+		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06.789");
+		assertEquals(ChronoUnit.MILLIS, a.getAccuracy());
 		// verify zone offset
 		// for second accuracy
 		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06+0800");
@@ -103,13 +105,13 @@ public class TestDateTimeAccuracy {
 		formatter.withResolverStyle(ResolverStyle.STRICT);
 
 		DateTimeAccuracy a;
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003",  ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.DAYS,a.getAccuracy());
 		
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13",  ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.HOURS,a.getAccuracy());
 
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13:14");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13:14", ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.MINUTES,a.getAccuracy());
 	}
 
@@ -144,16 +146,16 @@ public class TestDateTimeAccuracy {
 		}
 		// verify same behavior for DateTimeAccurecy
 		// should not fail below
-		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, "2003");
+		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, "2003" ,ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.YEARS, a.getAccuracy());
 		// next two calls should throw exceptions
 		try{
-			DateTimeAccuracy.parse(formatter, "+");
+			DateTimeAccuracy.parse(formatter, "+", ZoneOffset.UTC);
 			Assert.fail("Exception unexpected input");
 		}catch( DateTimeParseException e ){
 		}
 		try{
-			DateTimeAccuracy.parse(formatter, "2003+");
+			DateTimeAccuracy.parse(formatter, "2003+", ZoneOffset.UTC);
 			Assert.fail("Exception expected for unparsed text at end of input");
 		}catch( DateTimeParseException e ){
 		}
@@ -165,6 +167,13 @@ public class TestDateTimeAccuracy {
 		assertEquals("2001-02-02T20Z", a.toPartialIso8601(ZoneId.of("UTC")));
 	}
 
+	@Test
+	public void verifyMillisecondParseAndToString() throws ParseException{
+		DateTimeAccuracy a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06.789", ZoneId.of("UTC"));
+		// date should be treated as if it had a +08:00 offset
+		assertEquals("2001-02-03T04:05:06.789Z", a.toPartialIso8601(ZoneId.of("UTC")));
+	}
+
 	@Test
 	public void verifyComparison() throws ParseException{
 		ZoneId zone = ZoneOffset.UTC.normalized();

histream-core/src/test/java/de/sekmi/histream/impl/ECMAEvaluatorTest.java

@@ -35,9 +35,10 @@ public class ECMAEvaluatorTest {
 		}
 
 		// skip to string value
-		for( int i=0; i<6; i++ ){
+		for( int i=0; i<7; i++ ){
 			o = s.get();
 		}
+		Assert.assertEquals("T:type:str", o.getConceptId());
 
 		// compare string value
 		Assert.assertEquals(true, eval.test("fact.value != null", o));

histream-core/src/test/java/de/sekmi/histream/impl/ObservationImplJAXBTest.java

@@ -8,6 +8,7 @@ import java.io.StringWriter;
 import java.math.BigDecimal;
 import java.net.URL;
 import java.time.Instant;
+import java.time.ZoneOffset;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.JAXB;
@@ -61,7 +62,7 @@ public class ObservationImplJAXBTest {
 		o.conceptId = "C"+index;
 		o.patientId = "P"+index;
 		o.encounterId = "E"+index;
-		o.startTime = new DateTimeAccuracy(2015,1,1,index);
+		o.startTime = new DateTimeAccuracy(ZoneOffset.UTC, 2015,1,1,index);
 		switch( index ){
 		case 0:
 			// string value

histream-core/src/test/java/de/sekmi/histream/impl/XPathEvaluatorTest.java

@@ -38,9 +38,10 @@ public class XPathEvaluatorTest {
 			Assert.assertEquals(expr, true, eval.test(expr, o));
 		}
 		// skip to string value
-		for( int i=0; i<6; i++ ){
+		for( int i=0; i<7; i++ ){
 			o = s.get();
 		}
+		Assert.assertEquals("T:type:str", o.getConceptId());
 
 		// compare string value
 		Assert.assertEquals(true, eval.test("f:fact/f:value='abc123'", o));

histream-core/src/test/java/de/sekmi/histream/io/FileObservationProviderTest.java

@@ -101,7 +101,7 @@ public class FileObservationProviderTest {
 				Assert.assertEquals("Zuhause", v.getLocationId());
 				try{
 					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-01T10:30:00"), v.getStartTime());
-					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-05T10:30:00"), v.getEndTime());
+					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-05T10:30:00.123"), v.getEndTime());
 				} catch (ParseException e) {
 					throw new RuntimeException(e);
 				}
@@ -111,6 +111,12 @@ public class FileObservationProviderTest {
 				ExternalSourceType s = o.getSource();
 				Assert.assertNotNull(s);
 			},
+			(Observation o) ->  {
+				Assert.assertEquals("T:date:msec", o.getConceptId());
+				// TODO store and calculate time in nanos
+				Assert.assertEquals(ChronoUnit.MILLIS, o.getStartTime().getAccuracy());
+				Assert.assertEquals(123, o.getStartTime().toInstantMin().atOffset(ZoneOffset.UTC).getLong(ChronoField.MILLI_OF_SECOND));
+			},
 			(Observation o) ->  {
 				Assert.assertEquals("T:date:mins", o.getConceptId());
 				Assert.assertEquals(ChronoUnit.MINUTES, o.getStartTime().getAccuracy());
@@ -232,6 +238,7 @@ public class FileObservationProviderTest {
 	
 	@Test
 	public void testStAXReader() throws FileNotFoundException, XMLStreamException, FactoryConfigurationError  {
+		// TODO delete XMLObservationSupplier
 		XMLObservationSupplier xos = new XMLObservationSupplier(factory, new FileInputStream("examples/dwh-eav.xml"));
 		validateExample(xos);
 		xos.close();

histream-core/src/test/java/de/sekmi/histream/io/TestXMLWriter.java

@@ -7,11 +7,17 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.text.ParseException;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.ZoneOffset;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.JAXB;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlValue;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
@@ -33,10 +39,19 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
 import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationException;
+import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
+import de.sekmi.histream.ext.ExternalSourceType;
+import de.sekmi.histream.ext.Patient;
+import de.sekmi.histream.ext.Visit;
 import de.sekmi.histream.impl.ExternalSourceImpl;
 import de.sekmi.histream.impl.Meta;
+import de.sekmi.histream.impl.ObservationFactoryImpl;
+import de.sekmi.histream.impl.SimplePatientExtension;
+import de.sekmi.histream.impl.SimpleVisitExtension;
+import de.sekmi.histream.impl.StringValue;
 import de.sekmi.histream.xml.XMLUtils;
 
 public class TestXMLWriter {
@@ -66,6 +81,7 @@ public class TestXMLWriter {
 		Document doc = builder.newDocument();
 		doc.getDomConfig().setParameter("namespaces", true);
 		doc.getDomConfig().setParameter("namespace-declarations", true);
+		// TODO check if DOM can be configured to allow newline characters in values
 
 // not suppoted by default implementation
 //		doc.getDomConfig().setParameter("canonical-form", true);
@@ -102,6 +118,19 @@ public class TestXMLWriter {
 
 		w.writeEndDocument();		
 	}
+
+	@Test
+	public void testMarshallNewlineInValues() throws ParserConfigurationException, ParseException {
+		DOMResult result = new DOMResult(createDocument());
+		ObservationFactory factory = new ObservationFactoryImpl();
+		factory.registerExtension(new SimplePatientExtension());
+		factory.registerExtension(new SimpleVisitExtension());
+		
+		Observation o = factory.createObservation("A", "B", DateTimeAccuracy.parsePartialIso8601("2018", ZoneOffset.UTC.normalized()));
+		o.setValue(new StringValue("1\n2"));
+		JAXB.marshal(o, result);
+		
+	}
 	@Test
 	public void testStreamWriterNamespaces() throws XMLStreamException, ParserConfigurationException{
 		XMLOutputFactory factory = XMLOutputFactory.newInstance();
@@ -164,6 +193,42 @@ public class TestXMLWriter {
 			reader.close();
 		}
 	}
+	@XmlRootElement
+	private static class ValType{
+		public ValType() {}
+		public ValType(String s) { this.value = s;}
+		@XmlValue 
+		String value;
+	}
+	@Test
+	public void testWriteDOM2() throws Exception{
+
+		// create document
+		DocumentBuilderFactory f = DocumentBuilderFactory.newInstance();
+//		f.setNamespaceAware(true);
+//		f.setCoalescing(true);
+//		f.setIgnoringComments(true);
+		DocumentBuilder builder = f.newDocumentBuilder();
+		Document doc = builder.newDocument();
+//		doc.getDomConfig().setParameter("namespaces", true);
+//		doc.getDomConfig().setParameter("namespace-declarations", true);
+
+		DOMResult dr = new DOMResult(doc);
+
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		// enable repairing namespaces to remove duplicate namespace declarations by JAXB marshal
+		// this does not work with the DOM stream writer
+
+//		factory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, Boolean.TRUE);
+		XMLStreamWriter writer = factory.createXMLStreamWriter(dr);
+		Marshaller marshaller = JAXBContext.newInstance(ValType.class).createMarshaller();
+//		marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE);
+		marshaller.marshal(new ValType("1\n2"), writer);
+		doc.normalizeDocument();
+		XMLUtils.printDOM(doc, debugLog);
+
+	}
+	
 	@Test
 	public void testWriteDOM() throws Exception{
 
@@ -175,6 +240,17 @@ public class TestXMLWriter {
 		w.setZoneId(ZoneId.of("Asia/Shanghai"));
 		Meta.transfer(s, w);
 		Streams.transfer(s, w);
+		
+		// manually create observation
+		Observation o = t.getFactory().createObservation("A", "B", DateTimeAccuracy.parsePartialIso8601("2018", ZoneOffset.UTC.normalized()));
+		o.setValue(new StringValue("1\n2"));
+		ExternalSourceType es = new ExternalSourceImpl("manual", Instant.now());
+		o.setSource(es);
+		Patient pat = t.getFactory().getExtension(Patient.class).createInstance("A",es);
+		o.setExtension(Patient.class, pat);
+		o.setExtension(Visit.class, t.getFactory().getExtension(Visit.class).createInstance("V",pat,es));
+		w.accept(o);
+
 		w.close();
 		s.close();
 

histream-core/src/test/resources/dwh.xml

@@ -40,13 +40,14 @@
 		<source timestamp="2015-09-28T09:41:10Z"/>
 		<encounter id="XXE12345">
 			<start>2014-01-01T10:30:00</start>
-			<end>2014-01-05T10:30:00</end>
+			<end>2014-01-05T10:30:00.123</end>
 			<location>Zuhause</location>
 			<!-- TODO inpatient/outpatient 
 			<provider>xxxa</provider>-->
 			<source timestamp="2015-09-28T08:41:10Z"/>
 <!-- no more <facts> group -->
 			<fact concept="T:date:secs" start="2014-09-07T10:40:03"/>
+			<fact concept="T:date:msec" start="2014-09-07T10:40:03.123"/>
 			<fact concept="T:date:mins" start="2014-09-07T10:40"/>
 			<fact concept="T:date:hours" start="2014-09-07T10"/>
 			<fact concept="T:date:day" start="2014-09-07"/>

histream-export/src/main/java/de/sekmi/histream/export/VisitFragmentParser.java

@@ -140,7 +140,7 @@ abstract class VisitFragmentParser extends GroupedXMLWriter {
 	/**
 	 * Called after each patient fragment was parsed.
 	 * The patient fragment does not contain any encounters,
-	 * these are provided to {@link #visitFragment(Node)}.
+	 * these are provided to {@link #visitFragment(Element)}.
 	 * @param patient patient node
 	 * @throws ObservationException error
 	 */

histream-export/src/main/java/de/sekmi/histream/export/csv/Table.java

@@ -14,8 +14,8 @@ class Table implements TableWriter{
 	private final CSVWriter export;
 	private PrintWriter out;
 	/**
-	 * @param csvWriter
-	 * @throws IOException 
+	 * @param csvWriter CSV writer
+	 * @throws IOException IO error
 	 */
 	Table(CSVWriter csvWriter, String filename) throws IOException {
 		export = csvWriter;

histream-export/src/test/java/de/sekmi/histream/export/TestExport.java

@@ -44,8 +44,8 @@ public class TestExport {
 		Assert.assertEquals("T:type:str", m.get(MemoryExportWriter.VISIT_TABLE, "byclass", 0));
 		m.dump();
 		// verify eav table
-		Assert.assertEquals(6, m.rowCount("eavtabletest"));
-		Assert.assertEquals("T:date:month", m.get("eavtabletest", "code", 4));
+		Assert.assertEquals(7, m.rowCount("eavtabletest"));
+		Assert.assertEquals("T:date:month", m.get("eavtabletest", "code", 5));
 		
 		// TODO something wrong with namespaces in xpath/dom
 	}

histream-export/src/test/java/de/sekmi/histream/export/config/TestVisitFragmentParser.java

@@ -46,7 +46,7 @@ public class TestVisitFragmentParser {
 //		System.out.println("XPath="+ret);
 		ret = (String)xp.evaluate("count(eav:fact)", visit, XPathConstants.STRING);
 //		System.out.println("Facts:"+ret);
-		Assert.assertEquals("12", ret);
+		Assert.assertEquals("13", ret);
 	}
 	
 }

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2Extractor.java

@@ -58,8 +58,8 @@ public abstract class I2b2Extractor implements ObservationSupplier {
 	 * <p>
 	 * 	
 	 * </p>
-	 * @param factory
-	 * @param dbc
+	 * @param factory extractor factory
+	 * @param dbc database connection
 	 * @throws SQLException error
 	 */
 	I2b2Extractor(I2b2ExtractorFactory factory, Connection dbc) throws SQLException {

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2Visit.java

@@ -2,6 +2,7 @@ package de.sekmi.histream.i2b2;
 
 import java.time.temporal.ChronoUnit;
 
+import de.sekmi.histream.ext.Patient;
 import de.sekmi.histream.ext.Visit;
 
 /*
@@ -64,7 +65,17 @@ public class I2b2Visit extends VisitImpl {
 
 	public int getNum(){return encounter_num;}
 	public int getPatientNum(){return patient_num;}
-	
+
+	@Override
+	public void setPatient(Patient patient) {
+		super.setPatient(patient);
+		if( patient instanceof I2b2Patient ) {
+			// also set the patient_num 
+			int patient_num = ((I2b2Patient)patient).getNum();
+			this.patient_num = patient_num;
+		}
+	}
+
 	@Override
 	public String toString(){
 		return "I2b2Visit(encounter_um="+encounter_num+")";

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/PostgresVisitStore.java

@@ -53,6 +53,7 @@ import de.sekmi.histream.ext.Visit.Status;
  * <p>
  * Some optional columns are used: active_status_cd, start_date, end_date, inout_cd, location_cd, sourcesystem_cd
  * <p>
+ * XXX after loading encounters, the String patientId not set anymore and always null. To determine the patientId, the patientStore is required for lookup of the patientNum
  * TODO use encounter_mapping table to map actual (source) patient_ide to internal patient_num for facts.
  * <p>
  * The variable argument list for {@link #createInstance(Object...)} requires the following arguments:
@@ -137,14 +138,14 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit> implements
 		db.setAutoCommit(true);
 		prepareStatements();
 		loadMaxEncounterNum();
-		batchLoad();
+		batchLoad(); /// XXX loading visits does not set the String patientId, for that, the patientStore would be needed
 	}
 	@Override
 	protected void prepareStatements() throws SQLException {
 		// TODO: use prefix from configuration to specify tablespace
 		insert = db.prepareStatement("INSERT INTO visit_dimension(encounter_num, patient_num, import_date, download_date, sourcesystem_cd) VALUES(?,?,current_timestamp,?,?)");
 		insertMapping = db.prepareStatement("INSERT INTO encounter_mapping(encounter_num, encounter_ide, encounter_ide_source, patient_ide, patient_ide_source, encounter_ide_status, project_id, import_date, download_date, sourcesystem_cd) VALUES(?,?,?,?,?,'A','"+projectId+"',current_timestamp,?,?)");
-		update = db.prepareStatement("UPDATE visit_dimension SET active_status_cd=?, start_date=?, end_date=?, inout_cd=?, location_cd=?, update_date=current_timestamp, download_date=?, sourcesystem_cd=? WHERE encounter_num=?");
+		update = db.prepareStatement("UPDATE visit_dimension SET patient_num=?, active_status_cd=?, start_date=?, end_date=?, inout_cd=?, location_cd=?, update_date=current_timestamp, download_date=?, sourcesystem_cd=? WHERE encounter_num=?");
 		//select = db.prepareStatement("SELECT encounter_num, patient_num, active_status_cd, start_date, end_date, inout_cd, location_cd, update_date, sourcesystem_cd FROM visit_dimension WHERE patient_num=?");
 		selectAll = db.prepareStatement("SELECT encounter_num, patient_num, active_status_cd, start_date, end_date, inout_cd, location_cd, download_date, sourcesystem_cd FROM visit_dimension", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
 		selectAll.setFetchSize(fetchSize);
@@ -310,16 +311,17 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit> implements
 	
 	private void updateStorage(I2b2Visit visit) throws SQLException {
 		synchronized( update ){
-			update.setString(1, visit.getActiveStatusCd());
-			update.setTimestamp(2, dialect.encodeInstantPartial(visit.getStartTime()));
-			update.setTimestamp(3, dialect.encodeInstantPartial(visit.getEndTime()));
-			update.setString(4, visit.getInOutCd());
-			update.setString(5, dialect.encodeLocationCd(visit.getLocationId()));
-			update.setTimestamp(6, dialect.encodeInstant(visit.getSourceTimestamp()));
-			update.setString(7, visit.getSourceId());
+			update.setInt(1, visit.getPatientNum());
+			update.setString(2, visit.getActiveStatusCd());
+			update.setTimestamp(3, dialect.encodeInstantPartial(visit.getStartTime()));
+			update.setTimestamp(4, dialect.encodeInstantPartial(visit.getEndTime()));
+			update.setString(5, visit.getInOutCd());
+			update.setString(6, dialect.encodeLocationCd(visit.getLocationId()));
+			update.setTimestamp(7, dialect.encodeInstant(visit.getSourceTimestamp()));
+			update.setString(8, visit.getSourceId());
 
 			// where encounter_num=visit.getNum()
-			update.setInt(8, visit.getNum());
+			update.setInt(9, visit.getNum());
 			int rows = update.executeUpdate();
 			if( rows == 0 ){
 				log.warning("UPDATE executed for visit_dimension.encounter_num="+visit.getNum()+", but no rows changed.");
@@ -368,7 +370,8 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit> implements
 	private I2b2Visit loadFromResultSet(ResultSet rs) throws SQLException{
 		int id = rs.getInt(1);
 		int patid = rs.getInt(2);
-		
+		// XXX String patientId is always null after loading from the database.
+
 		// load vital status code, which contains information about
 		// accuracy of birth and death dates.
 		String active_status_cd = rs.getString(3);
@@ -426,10 +429,12 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit> implements
 		I2b2Visit visit = idCache.get(encounterId);
 		
 		if( visit == null ){
+			// visit does not exist, create a new one
+
 			maxEncounterNum ++;
 			int encounter_num = maxEncounterNum;
 			visit = new I2b2Visit(encounter_num, patient.getNum());
-			visit.setPatientId(patient.getId());
+			visit.setPatient(patient);
 			
 			// created from observation, use source metadata
 			visit.setSourceId(source.getSourceId());
@@ -451,12 +456,17 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit> implements
 			// commonly, the item is modified after a call to this method,
 			// but changes are written later via a call to update.
 			// (otherwise, the instance would need to know whether to perform INSERT or UPDATE)
-		}else if( rejectPatientChange ){
+		}else {
 			// visit already existing
 			// verify that the patient number from the visit matches with the observation
 			if( visit.getPatientNum() != patient.getNum() ){
 				// throw exception to abort processing
-				throw new AssertionError("Patient_num mismatch between observation and visit", null);
+				if( rejectPatientChange ){
+					throw new AssertionError("Patient_num mismatch for visit "+encounterId+": history says "+visit.getPatientNum()+" while data says "+patient.getNum(), null);
+				}else {
+					log.info("Updating visit #"+visit.getNum()+" for patient change from #"+visit.getPatientNum()+" to #"+patient.getNum());
+					visit.setPatient(patient);
+				}
 			}
 		}
 		return visit;		

histream-import/src/main/java/de/sekmi/histream/etl/FileRowSupplier.java

@@ -6,6 +6,7 @@ import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.net.URL;
 import java.net.URLConnection;
+import java.nio.charset.Charset;
 import java.time.Instant;
 
 import com.opencsv.CSVParser;
@@ -19,7 +20,7 @@ public class FileRowSupplier extends RowSupplier {
 
 	private Instant timestamp;
 
-	public FileRowSupplier(URL location, String fieldSeparator) throws IOException{
+	public FileRowSupplier(URL location, String fieldSeparator, Charset charset) throws IOException{
 		if( fieldSeparator.length() > 1 ){
 			if( fieldSeparator.equals("\\t") ){
 				fieldSeparator = "\t";
@@ -28,7 +29,7 @@ public class FileRowSupplier extends RowSupplier {
 			}
 		}
 		this.url = location;
-		this.in = new CSVReader(new InputStreamReader(location.openStream()),fieldSeparator.charAt(0), CSVParser.DEFAULT_QUOTE_CHARACTER, (char)0);
+		this.in = new CSVReader(new InputStreamReader(location.openStream(), charset),fieldSeparator.charAt(0), CSVParser.DEFAULT_QUOTE_CHARACTER, (char)0);
 		
 		// TODO: check whether needed to close underlying InputStream
 		

histream-import/src/main/java/de/sekmi/histream/etl/VisitRow.java

@@ -3,13 +3,15 @@ package de.sekmi.histream.etl;
 import java.util.ArrayList;
 import java.util.List;
 
+import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.impl.VisitImpl;
 
 public class VisitRow extends VisitImpl implements FactRow{
 	List<Observation> facts;
 	
-	public VisitRow(){
+	public VisitRow(String visitId, String patientId, DateTimeAccuracy startTime){
+		super(visitId, patientId, startTime);
 		facts = new ArrayList<>();
 	}
 	@Override
@@ -21,6 +23,4 @@ public class VisitRow extends VisitImpl implements FactRow{
 	public String getVisitId() {
 		return this.getId();
 	}
-
-
 }

histream-import/src/main/java/de/sekmi/histream/etl/config/CsvFile.java

@@ -3,6 +3,7 @@ package de.sekmi.histream.etl.config;
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.util.regex.Pattern;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -32,7 +33,7 @@ public class CsvFile extends TableSource{
 	String url;
 	
 	/**
-	 * File encoding is not used yet.
+	 * Encoding to use for reading text files
 	 */
 	@XmlElement
 	String encoding;
@@ -44,11 +45,11 @@ public class CsvFile extends TableSource{
 	@XmlElement
 	String separator;
 	
-	@XmlElement
-	String quote;
-	
-	@XmlElement
-	char escape;
+//	@XmlElement
+//	String quote;
+//	
+//	@XmlElement
+//	char escape;
 	
 	private CsvFile(){
 	}
@@ -59,9 +60,18 @@ public class CsvFile extends TableSource{
 	}
 	@Override
 	public RowSupplier rows(Meta meta) throws IOException {
+		// resolve url relative to base url from metadata
 		URL base = meta.getLocation();
 		URL source = (base == null)?new URL(url):new URL(base, url);
-		return new FileRowSupplier(source, separator);
+		// determine charset
+		Charset charset;
+		if( encoding != null ) {
+			charset = Charset.forName(encoding);
+		}else{
+			// if not defined, use system charset
+			charset = Charset.defaultCharset();
+		}
+		return new FileRowSupplier(source, separator, charset);
 	}
 
 }

histream-import/src/main/java/de/sekmi/histream/etl/config/DateTimeColumn.java

 package de.sekmi.histream.etl.config;
 
 import java.sql.Timestamp;
+import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 
@@ -26,7 +27,9 @@ public class DateTimeColumn extends Column<DateTimeAccuracy>{
 	 */
 	@XmlAttribute
 	String format;
-	
+
+	@XmlAttribute
+	String zone;
 	
 	public DateTimeColumn(String name, String format){
 		super(name);
@@ -56,9 +59,15 @@ public class DateTimeColumn extends Column<DateTimeAccuracy>{
 		if( formatter == null ){
 			throw new ParseException("format must be specified for DateTime fields if strings are parsed");
 		}
+		ZoneId zoneId;
+		if( zone != null ){
+			zoneId = ZoneId.of(zone);
+		}else{
+			zoneId = ZoneId.systemDefault();
+		}
 		// parse
 		try{
-			return DateTimeAccuracy.parse(formatter,input);
+			return DateTimeAccuracy.parse(formatter,input, zoneId);
 		}catch( DateTimeParseException e ){
 			throw new ParseException("Unable to parse date '"+input+"' in column '"+this.column+"'", e);
 		}

histream-import/src/main/java/de/sekmi/histream/etl/config/VisitTable.java

@@ -84,19 +84,18 @@ public class VisitTable extends Table<VisitRow> implements ConceptTable{
 
 	@Override
 	public VisitRow fillRecord(ColumnMap map, Object[] row, ObservationFactory factory) throws ParseException {
-		VisitRow visit = new VisitRow();
-		visit.setId(idat.visitId.valueOf(map, row));
-		visit.setPatientId(idat.patientId.valueOf(map, row));
+		String vid = idat.visitId.valueOf(map, row);
+		String pid = idat.patientId.valueOf(map, row);
 		DateTimeAccuracy start = idat.start.valueOf(map, row);
-		if( start != null ){
-			visit.setStartTime(start);
-		}else{
+		if( start == null ){
 			// no start time specified for visit row
 			// any other way to retrieve a timestamp??
 			// ignore row
 			// TODO issue warning
 			return null;
 		}
+		VisitRow visit = new VisitRow(vid, pid, start);
+
 		if( idat.end != null ){
 			visit.setEndTime(idat.end.valueOf(map, row));
 		}

histream-import/src/main/java/de/sekmi/histream/etl/filter/DuplicateFactFilter.java

@@ -18,21 +18,29 @@ public class DuplicateFactFilter extends PostProcessingFilter{
 	@XmlElement
 	public String[] concept;
 
-	private static class FactComparator implements Comparator<Fact>{
+	static class FactComparator implements Comparator<Fact>{
 		@Override
 		public int compare(Fact o1, Fact o2) {
-			int cmp = o1.getObservation().getStartTime().compareTo(
-						o2.getObservation().getStartTime()	);
-			if( cmp == 0 ){
-				// if times are equal, sort by concept
-				cmp = o1.getConcept().compareTo(o2.getConcept());
-			}
-			return cmp;
+			return DuplicateFactFilter.compare(o1, o2);
+		}
+	}
+	public static int compare(Fact o1, Fact o2) {
+		int cmp = o1.getObservation().getStartTime().compareTo(
+					o2.getObservation().getStartTime()	);
+		if( true )return cmp;
+		if( cmp == 0 ){
+			// if times are equal, sort by concept
+			cmp = o1.getConcept().compareTo(o2.getConcept());
 		}
+		return cmp;
 	}
 	private void removeAllDuplicates(AbstractFacts facts){
 		// order by start and concept
-		facts.sort( new FactComparator() );
+		if( true ) {
+			throw new UnsupportedOperationException("Not yet implemented");
+		}
+		//facts.sort( new FactComparator() );
+		
 
 		ArrayList<Integer> duplicates = new ArrayList<>();
 		

histream-import/src/test/java/de/sekmi/histream/etl/TestETLSupplier.java

 package de.sekmi.histream.etl;
 
 import java.io.IOException;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -11,13 +13,11 @@ import org.junit.Test;
 
 import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
-import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
 import de.sekmi.histream.ext.ExternalSourceType;
 import de.sekmi.histream.ext.Patient;
 import de.sekmi.histream.ext.Visit;
 import de.sekmi.histream.impl.Meta;
-import de.sekmi.histream.impl.ObservationFactoryImpl;
 import de.sekmi.histream.io.GroupedXMLWriter;
 import de.sekmi.histream.io.Streams;
 
@@ -103,8 +103,9 @@ public class TestETLSupplier {
 		Patient p = fact.getExtension(Patient.class);
 		Assert.assertNotNull(p);
 		Assert.assertEquals("p1", p.getId());
-		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-01"), p.getBirthDate());
-		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-11"), p.getDeathDate());
+		ZoneId zone = ZoneId.systemDefault();
+		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-01",zone), p.getBirthDate());
+		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-11",zone), p.getDeathDate());
 
 		// TODO verify other patient information
 		Assert.assertEquals("v1", p.getGivenName());
@@ -128,7 +129,7 @@ public class TestETLSupplier {
 		Assert.assertEquals("v1", v.getId());
 		// TODO make sure custom partial date format is parsed correctly for missing seconds
 		//Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2013-03-20T09:00"), v.getStartTime());
-		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2013-03-21T13:00:21"), v.getEndTime());
+		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2013-03-21T13:00:21", ZoneId.systemDefault()), v.getEndTime());
 
 		Assert.assertEquals("v1", v.getId());
 		Assert.assertEquals(null, v.getLocationId());

histream-import/src/test/java/de/sekmi/histream/etl/TestRowSupplier.java

 package de.sekmi.histream.etl;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 import org.junit.Assert;
 import org.junit.Test;
@@ -9,7 +10,7 @@ public class TestRowSupplier {
 	
 	@Test
 	public void testLoadRows() throws IOException{
-		try( FileRowSupplier r = new FileRowSupplier(getClass().getResource("/data/test-1-patients.txt"), "\t") ){
+		try( FileRowSupplier r = new FileRowSupplier(getClass().getResource("/data/test-1-patients.txt"), "\t", StandardCharsets.ISO_8859_1) ){
 			String[] h = r.getHeaders();
 			Assert.assertEquals("patid", h[0]);
 			Assert.assertEquals("nachname", h[2]);

histream-import/src/test/java/de/sekmi/histream/etl/config/TestMarshall.java

@@ -69,10 +69,13 @@ public class TestMarshall {
 			
 			// check post processing
 			Assert.assertNotNull(ds.postProcessing);
-			Assert.assertEquals(3, ds.postProcessing.filter.length);
-			Assert.assertEquals(DuplicateFactFilter.class, ds.postProcessing.filter[0].getClass());
-			DuplicateFactFilter f = (DuplicateFactFilter)ds.postProcessing.filter[0];
+			Assert.assertEquals(2, ds.postProcessing.filter.length);
+// duplicate fact filter removed for now
+//			Assert.assertEquals(DuplicateFactFilter.class, ds.postProcessing.filter[0].getClass());
+//			DuplicateFactFilter f = (DuplicateFactFilter)ds.postProcessing.filter[0];
+//			Assert.assertEquals(1, f.concept.length);
 			ScriptFilter sf = (ScriptFilter)ds.postProcessing.filter[1];
+			Assert.assertNotNull(sf);
 			
 			// check script
 			/* Assert.assertEquals(2,  ds.scripts.length);

histream-import/src/test/java/de/sekmi/histream/etl/filter/TestDuplicateFactFilter.java

+package de.sekmi.histream.etl.filter;
+
+import java.text.ParseException;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import javax.script.ScriptException;
+
+import org.junit.Test;
+
+import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.impl.ObservationFactoryImpl;
+import de.sekmi.histream.impl.SimplePatientExtension;
+import de.sekmi.histream.impl.SimpleVisitExtension;
+import de.sekmi.histream.scripting.AbstractFacts;
+import de.sekmi.histream.scripting.EncounterScriptEngine;
+import de.sekmi.histream.scripting.Fact;
+
+import static org.junit.Assert.*;
+
+public class TestDuplicateFactFilter {
+
+	// XXX implement filter first
+	//@Test
+	public void verifyComparator() throws ScriptException, ParseException{
+		ObservationFactory of = new ObservationFactoryImpl(new SimplePatientExtension(), new SimpleVisitExtension());
+		DuplicateFactFilter filter = new DuplicateFactFilter();
+		EncounterScriptEngine e = new EncounterScriptEngine();
+		e.setObservationFactory(of);
+		AbstractFacts facts = e.wrapEncounterFacts("P1", "E1", DateTimeAccuracy.parsePartialIso8601("2001-02-03", ZoneId.systemDefault()), new ArrayList<>());
+		Fact a = facts.add("lala");
+		Fact b = facts.add("xx");
+//		assertTrue( DuplicateFactFilter.compare(a, b) < 0 );
+		b.start("2000-01-02T03:04:05Z");
+//		assertTrue( DuplicateFactFilter.compare(a, b) > 0 );
+		Fact c = facts.add("xx").start("2000-01-02T03:05Z");
+		Fact d = facts.add("lala"); // add duplicate
+//		assertTrue( DuplicateFactFilter.compare(c, b) > 0 );
+//		assertTrue( DuplicateFactFilter.compare(a, d) == 0 );
+//		assertTrue( DuplicateFactFilter.compare(a, c) > 0 );
+//		assertTrue( DuplicateFactFilter.compare(d, c) > 0 );
+		
+		filter.processVisit(facts);
+		assertEquals(3, facts.size());
+		// TODO compare with timestamps
+	}
+}

histream-import/src/test/java/de/sekmi/histream/etl/validation/TestValidator.java

@@ -5,7 +5,6 @@ import org.junit.Test;
 
 import de.sekmi.histream.ObservationSupplier;
 import de.sekmi.histream.etl.ETLObservationSupplier;
-import de.sekmi.histream.etl.config.DataSource;
 import de.sekmi.histream.io.Streams;
 
 public class TestValidator {
@@ -70,7 +69,8 @@ public class TestValidator {
 		}
 		Assert.fail("Exception expected");
 	}
-	@Test
+	// XXX implement duplicate fact filter first
+	//@Test
 	public void validateData4WithDuplicateFilter() throws Exception{
 		// duplicate concepts
 		try( ObservationSupplier os = ETLObservationSupplier.load(getClass().getResource("/data/test-4-datasource2.xml")) ){

histream-import/src/test/resources/data/p21khg/AUSBILDUNG.csv

+IK;Ausbildungsstätte;Ausbildungsstätten-Typ;Ausbildungsplätze-insgesamt;Ausbildungsplätze-des-KH;Ausbildungsplätze-für-andere-KH;Ausbildende;Auszubildende-im-eigenen-KH;Azubis-J1;Azubis-J2;Azubis-J3;Auszubildende-an-anderen-KH;Ausbildungsvergütungen;Personalkosten-je-VK;Kosten-Unterricht;Kosten-praktische-Ausbildung;Sachaufwand-Ausbildungsstätte;Gemeinkosten-Ausbildungsstätte;Vereinbarte-Gesamtkosten-Ausbildungsstätte
+260100012;A05;1;30;30;0;2,00;29,00;12,00;9,00;8,00;0,00;290000,00;45532,78;60555,21;30672,10;10321,11;22081,20;20000,00
+260100012;A06;1;20;20;0;2,00;20,00;7,00;6,00;7,00;0,00;220000,00;45556,18;60000,01;20214,20;4567,69;17238,08;22000,00

histream-import/src/test/resources/data/p21khg/Abrechnung.csv

+IK;DRG-Flle-vereinbart;DRG-Flle-abgerechnet;Bewertungsrelationen-vereinbart;Bewertungsrelationen-abgerechnet;Erlsausgleich-4-Abs-3
+260100012;12000;11970;11876,32;10957,45;475,22

histream-import/src/test/resources/data/p21khg/Entgelte.csv

+IK;Entlassender-Standort;Entgeltbereich;KH-internes-Kennzeichen;IK-Krankenkasse;Entgeltart;Entgeltbetrag;Abrechnung-von;Abrechnung-bis;Entgeltanzahl;Tage-ohne-Berechnung-Behandlung;Tag-der-Behandlung
+260100012;01;PSY;6;101556856;01032900;185,65;20170113;20170120;8;0;
+260100012;01;PSY;6;101556856;01032900;185,65;20170108;20170110;3;0;
+260100012;01;PSY;6;101556856;01032900;185,65;20170101;20170105;5;0;
+260100012;01;PSY;6;101556856;00000000;2970,40;20170101;20170120;1;4;
+260100012;01;DRG;1;111556856;7010I47B;6803,08;20170519;20170601;1;0;
+260100012;01;DRG;1;111556856;00000000;6803,08;20170519;20170601;1;0;
+260100012;01;DRG;2;111556856;7010I47B;6803,08;20170519;20170601;1;2;
+260100012;01;DRG;2;111556856;00000000;6803,08;20170519;20170601;1;2;
+260100012;01;DRG;3;101556856;8500A90A;178,50;20170101;20170101;1;0;
+260100012;01;DRG;3;101556856;8500A90A;178,50;20170102;20170102;1;0;
+260100012;01;DRG;3;101556856;8500A90A;178,50;20170105;20170105;1;0;
+260100012;01;DRG;3;101556856;00000000;535,50;20170101;20170105;1;2;
+260100012;01;PSY;4;101556856;00000000;5427,35;20170101;20170119;1;0;
+260100012;01;PSY;4;101556856;01012900;185,65;20170101;20170119;19;0;
+260100012;01;PSY;4;101556856;01000001;100,00;20170101;20170119;19;0;
+260100012;01;PSY;5;101556856;00000000;4865,05;20170101;20170119;1;2;
+260100012;01;PSY;5;101556856;01012900;185,65;20170101;20170119;17;2;
+260100012;01;PSY;5;101556856;01000001;100,00;20170101;20170119;17;2;
+260100012;01;PSY;7;101556856;A1PA02A5;305,80;20170101;20170105;5;0;
+260100012;01;PSY;7;101556856;00000000;1529,00;20170101;20170105;1;0;
+260100012;01;PIA;11;101556856;31000000;185,65;;;1;0;20170410
+260100012;01;PIA;11;101556856;00000000;185,65;;;1;0;20170410
\ No newline at end of file

histream-import/src/test/resources/data/p21khg/FAB.csv

+IK;Entlassender-Standort;Entgeltbereich;KH-internes-Kennzeichen;FAB;FAB-Aufnahmedatum;FAB-Entlassungsdatum
+260100012;1;DRG;1;HA1600;201705190800;201706020800
+260100012;1;DRG;2;HA1600;201705190800;201705260800
+260100012;1;DRG;2;HA0001;201705260801;201705280800
+260100012;1;DRG;2;HA1600;201705280801;201706020800
+260100012;1;DRG;3;HA0200;201701010800;201701050800
+260100012;1;PSY;4;HA2900;201701010800;201701200800
+260100012;1;PSY;5;HA2900;201701010800;201701121800
+260100012;1;PSY;5;HA0003;201701121801;201701150700
+260100012;1;PSY;5;HA2900;201701150701;201701200800
+260100012;1;PSY;6;HA2900;201701010800;201701200800
+260100012;1;PSY;7;HA2900;201701010800;201701051000

histream-import/src/test/resources/data/p21khg/FALL.csv

+IK;Entlassender-Standort;Entgeltbereich;KH-internes-Kennzeichen;Versicherten-ID;Vertragskennzeichen-64b-Modellvorhaben;IK-der-Krankenkasse;Geburtsjahr;Geburtsmonat;Geschlecht;PLZ;Wohnort;Aufnahmedatum;Aufnahmeanlass;Aufnahmegrund;Fallzusammenfhrung;Fallzusammenfhrungsgrund;Aufnahmegewicht;Entlassungsdatum;Entlassungsgrund;Alter-in-Tagen-am-Aufnahmetag;Alter-in-Jahren-am-Aufnahmetag;Patientennummer;interkurrente-Dialysen;Beatmungsstunden;Behandlungsbeginn-vorstationr;Behandlungstage-vorstationr;Behandlungsende-nachstationr;Behandlungstage-nachstationr;IK-Verlegungs-KH;Belegungstage-in-anderem-Entgeltbereich;Beurlaubungstage-PSY;Kennung Besonderer Fall Modellvorhaben
+260100012;01;DRG;1;9999999999;;111556856;1945;01;w;53121;Bonn;201705190800;E;0101;N;;;201706020800;019;;66;Pat7412;;;;;;;;;;
+260100012;01;DRG;2;9999999999;;111556856;1945;01;w;53121;Bonn;201705190800;E;0101;J;OG;;201706020800;019;;66;Pat7413;;;;;;;;2;;
+260100012;01;DRG;3;P000000004;;101556856;1945;01;w;53121;Bonn;201701010800;E;0301;N;;;201701050800;019;;66;Pat7414;;;;;;;;;;
+260100012;01;PSY;4;P000000004;;101556856;1945;01;w;53121;Bonn;201701010800;E;0101;N;;;201701200800;019;;66;Pat7415;;;;;;;;;;
+260100012;01;PSY;5;P000000004;;101556856;1945;01;w;53121;Bonn;201701010800;E;0101;N;;;201701200800;019;;66;Pat7415;;;;;;;;;2;
+260100012;01;PSY;6;P000000004;;101556856;1945;01;w;53121;Bonn;201701010800;E;0301;N;;;201701200800;019;;66;Pat7415;;;;;;;;;;
+260100012;01;PSY;7;P000000004;M17AC002;101556856;1945;01;w;53121;Bonn;201701010800;E;0101;N;;;201701051000;019;;66;Pat7415;;;;;;;;;;
+260100012;01;PIA;11;P000000004;;101556856;1945;01;w;53121;Bonn;201704100800;E;0101;N;;;201704101200;019;;66;Pat7415;;;;;;;;;;

histream-import/src/test/resources/data/p21khg/FUSIONEN.csv

+IK;IKF;Falldaten von;Falldaten bis
+260100012;260100001;20170701;20171231
+260100012;260100002;20170701;20171231

histream-import/src/test/resources/data/p21khg/ICD.csv

+IK;Entlassender-Standort;Entgeltbereich;KH-internes-Kennzeichen;Diagnoseart;ICD-Version;ICD-Kode;Lokalisation;Diagnosensicherheit;Sekundr-Kode;Lokalisation2;Diagnosensicherheit2
+260100012;01;PSY;6;ND;2017;F10.1;;;;;
+260100012;01;DRG;1;HD;2017;M16.7;L;;;;
+260100012;01;DRG;2;HD;2017;M16.7;L;;;;
+260100012;01;DRG;3;HD;2017;R26.8;;;;;
+260100012;01;PSY;4;HD;2017;F20.1;;;;;
+260100012;01;PSY;4;ND;2017;G40.1;;;;;
+260100012;01;PSY;5;HD;2017;F20.0;;;;;
+260100012;01;PSY;5;ND;2017;F10.0;;;;;
+260100012;01;DRG;1;ND;2017;K66.0;;;;;
+260100012;01;PSY;4;ND;2017;F10.1;;;;;
+260100012;01;PSY;6;HD;2017;F20.1;;;;;
+260100012;01;PSY;6;ND;2017;F10.1;;;;;
+260100012;01;PSY;7;HD;2017;F20.0;;;;;
+260100012;01;PSY;7;ND;2017;F05.8;;;;;
+260100012;01;PIA;11;ND;2017;F12.2;;A;;;

histream-import/src/test/resources/data/p21khg/INFO.csv

+IK;Datenerhebung;Datum-der-Erstellung;E-Mail-Adresse;DRG-Grouper;(Softwarelösung);Versionskennung;E-Mail-Adresse2
+260100012;2017;201803011027;info@domain.de;;;20180101;max.mustermann@domain.de

histream-import/src/test/resources/data/p21khg/Kosten.csv

+IK;Entlassender-Standort;Entgeltbereich;KH-internes-Kennzeichen;Kostenstellengruppe;Kostenartengruppe;Kostenwert;Pflegetag
+260100012;3;DRG;1;1;1;536,13;20170101
+260100012;3;DRG;1;1;2;1019,02;20170101
+260100012;3;DRG;1;1;3;32,06;20170101
+260100012;3;DRG;1;1;4a;55,29;20170101
+260100012;3;DRG;1;1;4b;7,4;20170101
+260100012;3;DRG;1;1;6a;71,88;20170101
+260100012;3;DRG;1;1;6b;1,35;20170101
+260100012;3;DRG;1;1;6c;11,33;20170101
+260100012;3;DRG;1;1;7;263,16;20170101
+260100012;3;DRG;1;1;8;813,2;20170101
+260100012;3;DRG;1;4;1;431,4;20170101
+260100012;3;DRG;1;4;3;290,61;20170101
+260100012;3;DRG;1;4;4a;10,12;20170101
+260100012;3;DRG;1;4;4b;9,84;20170101
+260100012;3;DRG;1;4;5;922,48;20170101
+260100012;3;DRG;1;4;6a;158,95;20170101
+260100012;3;DRG;1;4;6b;92,86;20170101
+260100012;3;DRG;1;4;6c;4,17;20170101
+260100012;3;DRG;1;4;7;181,4;20170101
+260100012;3;DRG;1; 4;8;219,16;20170101
+260100012;3;DRG;1;5;1;268,53;20170101
+260100012;3;DRG;1;5;3;179,63;20170101
+260100012;3;DRG;1;5;4a;15,6;20170101
+260100012;3;DRG;1;5;4b;0,72;20170101
+260100012;3;DRG;1;5;6a;52,14;20170101
+260100012;3;DRG;1;5;6b;1,7;20170101
+260100012;3;DRG;1;5;6c;0,53;20170101
+260100012;3;DRG;1;5;7;35,03;20170101
+260100012;3;DRG;1;5;8;76,64;20170101
+260100012;3;DRG;1;9;1;34,4;20170101
+260100012;3;DRG;1;9;3;44,56;20170101
+260100012;3;DRG;1;9;4a;0,4;20170101
+260100012;3;DRG;1;9;4b;5,07;20170101
+260100012;3;DRG;1;9;6a;5,85;20170101
+260100012;3;DRG;1;9;6b;6,18;20170101
+260100012;3;DRG;1;9;6c;21,14;20170101
+260100012;3;DRG;1;9;7;15,45;20170101
+260100012;3;DRG;1;9;8;28,19;20170101
+260100012;3;DRG;1;10;1;6,3;20170101
+260100012;3;DRG;1;10;3;40,01;20170101
+260100012;3;DRG;1;10;4a;1,79;20170101
+260100012;3;DRG;1;10;4b;41,44;20170101
+260100012;3;DRG;1;10;6a;29,68;20170101
+260100012;3;DRG;1;10;6b;6,74;20170101
+260100012;3;DRG;1;10;6c;29,92;20170101
+260100012;3;DRG;1;10;7;39,45;20170101
+260100012;3;DRG;1;10;8;16,01;20170101
+260100012;3;DRG;1;11;1;7,94;20170101
+260100012;3;DRG;1;11;3;7,17;20170101
+260100012;3;DRG;1;11;4a;0,12;20170101
+260100012;3;DRG;1;11;6a;1,29;20170101
+260100012;3;DRG;1;11;6c;0,51;20170101
+260100012;3;DRG;1;11;7;1,46;20170101
+260100012;3;DRG;1;11;8;3,92;20170101
+260100012;3;DRG;1;12;1;5,55;20170101
+260100012;3;DRG;1;12;3;141,62;20170101
+260100012;3;DRG;1;12;4a;0,16;20170101
+260100012;3;DRG;1;12;6a;1,86;20170101
+260100012;3;DRG;1;12;7;5,71;20170101
+260100012;3;DRG;1;12;8;52,79;20170101
+260100012;3;DRG;1;13;1;35,7;20170101
+260100012;3;DRG;1;13;2;6,84;20170101
+260100012;3;DRG;1;13;3;34,74;20170101
+260100012;3;DRG;1;13;4a;1,34;20170101
+260100012;3;DRG;1;13;6a;4,88;20170101
+260100012;3;DRG;1;13;7;5,93;20170101
+260100012;3;DRG;1;13;8;21,46;20170101
+260100012;0;PSY;4;21;1;23,11;20170101
+260100012;0;PSY;4;21;2;63,06;20170101
+260100012;0;PSY;4;21;3a;4,61;20170101
+260100012;0;PSY;4;21;3b;3,67;20170101
+260100012;0;PSY;4;21;3c;2,81;20170101
+260100012;0;PSY;4;21;3;1,1;20170101
+260100012;0;PSY;4;21;4a;1,32;20170101
+260100012;0;PSY;4;21;6a;1,11;20170101
+260100012;0;PSY;4;21;7;0,13;20170101
+260100012;0;PSY;4;21;8;10,7;20170101
+260100012;0;PSY;4;10;1;3,61;20170101
+260100012;0;PSY;4;10;3;2,81;20170101
+260100012;0;PSY;4;10;4a;1,32;20170101
+260100012;0;PSY;4;10;6a;1,11;20170101
+260100012;0;PSY;4;10;7;0,06;20170101
+260100012;0;PSY;4;10;8;0,19;20170101
+260100012;0;PSY;4;13;1;12,11;20170101
+260100012;0;PSY;4;13;2;23,06;20170101
+260100012;0;PSY;4;13;3;9,1;20170101
+260100012;0;PSY;4;13;4a;0,32;20170101
+260100012;0;PSY;4;13;6a;0,11;20170101
+260100012;0;PSY;4;13;7;0,07;20170101
+260100012;0;PSY;4;13;8; 0,23;20170101
+260100012;0;PSY;4;21;1;23,11;20170102
+260100012;0;PSY;4;21;2;63,06;20170102
+260100012;0;PSY;4;21;3a;4,61;20170102
+260100012;0;PSY;4;21;3b;3,67;20170102
+260100012;0;PSY;4;21;3c;2,81;20170102
+260100012;0;PSY;4;21;3;1,1;20170102
+260100012;0;PSY;4;21;4a;1,32;20170102
+260100012;0;PSY;4;21;6a;1,11;20170102
+260100012;0;PSY;4;21;7;0,13;20170102
+260100012;0;PSY;4;21;8;10,7;20170102
+260100012;0;PSY;4;23;1;23,11;20170102
+260100012;0;PSY;4;23;2;63,06;20170102
+260100012;0;PSY;4;23;4a;1,32;20170102
+260100012;0;PSY;4;23;6a;1,11;20170102
+260100012;0;PSY;4;21;1;23,11;20170103
+260100012;0;PSY;4;21;2;63,06;20170103
+260100012;0;PSY;4;21;3a;4,61;20170103
+260100012;0;PSY;4;21;3b;3,67;20170103
+260100012;0;PSY;4;21;3c;2,81;20170103
+260100012;0;PSY;4;21;3;1,1;20170103
+260100012;0;PSY;4;21;4a;1,32;20170103
+260100012;0;PSY;4;21;6a;1,11;20170103
+260100012;0;PSY;4;21;7;0,13;20170103
+260100012;0;PSY;4;21;8;10,7;20170103
+260100012;0;PSY;4;22;1;32,11;20170104
+260100012;0;PSY;4;22;2;126,06;20170104
+260100012;0;PSY;4;22;3a;28,61;20170104
+260100012;0;PSY;4;22;4a;2,64;20170104
+260100012;0;PSY;4;22;6a;2,22;20170104
+260100012;0;PSY;4;22;7;1,53;20170104
+260100012;0;PSY;4;22;8;6,59;20170104
+260100012;0;PSY;4;23;1;23,11;20170104
+260100012;0;PSY;4;23;2;63,06;20170104
+260100012;0;PSY;4;23;4a;1,32;20170104
+260100012;0;PSY;4;23;6a;1,11;20170104
+260100012;0;PSY;4;22;1;32,11;20170105
+260100012;0;PSY;4;22;2;126,06;20170105
+260100012;0;PSY;4;22;3a;28,61;20170105
+260100012;0;PSY;4;22;4a;2,64;20170105
+260100012;0;PSY;4;22;6a;2,22;20170105
+260100012;0;PSY;4;22;7;1,53;20170105
+260100012;0;PSY;4;22;8;6,59;20170105
+260100012;0;PSY;4;21;1;23,11;20170106
+260100012;0;PSY;4;21;2;63,06;20170106
+260100012;0;PSY;4;21;3a;4,61;20170106
+260100012;0;PSY;4;21;3b;3,67;20170106
+260100012;0;PSY;4;21;3c;2,81;20170106
+260100012;0;PSY;4;21;3;1,1;20170106
+260100012;0;PSY;4;21;4a;1,32;20170106
+260100012;0;PSY;4;21;6a;1,11;20170106
+260100012;0;PSY;4;21;7;0,13;20170106
+260100012;0;PSY;4;21;8;10,7;20170106
+260100012;0;PSY;4;21;1;23,11;20170107
+260100012;0;PSY;4;21;2;63,06;20170107
+260100012;0;PSY;4;21;3a;4,61;20170107
+260100012;0;PSY;4;21;3b;3,67;20170107
+260100012;0;PSY;4;21;3c;2,81;20170107
+260100012;0;PSY;4;21;3;1,1;20170107
+260100012;0;PSY;4;21;4a;1,32;20170107
+260100012;0;PSY;4;21;6a;1,11;20170107
+260100012;0;PSY;4;21;7;0,13;20170107
+260100012;0;PSY;4;21;8;10,7;20170107
+260100012;0;PSY;4;23;1;23,11;20170107
+260100012;0;PSY;4;23;2;63,06;20170107
+260100012;0;PSY;4;23;4a;1,32;20170107
+260100012;0;PSY;4;23;6a;1,11;20170107
+260100012;0;PSY;4;21;1;23,11;20170108
+260100012;0;PSY;4;21;2;63,06;20170108
+260100012;0;PSY;4;21;3a;4,61;20170108
+260100012;0;PSY;4;21;3b;3,67;20170108
+260100012;0;PSY;4;21;3c;2,81;20170108
+260100012;0;PSY;4;21;3;1,1;20170108
+260100012;0;PSY;4;21;4a;1,32;20170108
+260100012;0;PSY;4;21;6a;1,11;20170108
+260100012;0;PSY;4;21;7;0,13;20170108
+260100012;0;PSY;4;21;8;10,7;20170108
+260100012;0;PSY;4;21;1;23,11;20170109
+260100012;0;PSY;4;21;2;63,06;20170109
+260100012;0;PSY;4;21;3a;4,61;20170109
+260100012;0;PSY;4;21;3b;3,67;20170109
+260100012;0;PSY;4;21;3c;2,81;20170109
+260100012;0;PSY;4;21;3;1,1;20170109
+260100012;0;PSY;4;21;4a;1,32;20170109
+260100012;0;PSY;4;21;6a;1,11;20170109
+260100012;0;PSY;4;21;7;0,13;20170109
+260100012;0;PSY;4;21;8;10,7;20170109
+260100012;0;PSY;4;21;1;23,11;20170110
+260100012;0;PSY;4;21;2;63,06;20170110
+260100012;0;PSY;4;21;3a;4,61;20170110
+260100012;0;PSY;4;21;3b;3,67;20170110
+260100012;0;PSY;4;21;3c;2,81;20170110
+260100012;0;PSY;4;21;3;1,1;20170110
+260100012;0;PSY;4;21;4a;1,32;20170110
+260100012;0;PSY;4;21;6a;1,11;20170110
+260100012;0;PSY;4;21;7;0,13;20170110
+260100012;0;PSY;4;21;8;10,7;20170110
+260100012;0;PSY;4;21;1;23,11;20170111
+260100012;0;PSY;4;21;2;63,06;20170111
+260100012;0;PSY;4;21;3a;4,61;20170111
+260100012;0;PSY;4;21;3b;3,67;20170111
+260100012;0;PSY;4;21;3c;2,81;20170111
+260100012;0;PSY;4;21;3;1,1;20170111
+260100012;0;PSY;4;21;4a;1,32;20170111
+260100012;0;PSY;4;21;6a;1,11;20170111
+260100012;0;PSY;4;21;7;0,13;20170111
+260100012;0;PSY;4;21;8;10,7;20170111
+260100012;0;PSY;4;21;1;23,11;20170112
+260100012;0;PSY;4;21;2;63,06;20170112
+260100012;0;PSY;4;21;3a;4,61;20170112
+260100012;0;PSY;4;21;3b;3,67;20170112
+260100012;0;PSY;4;21;3c;2,81;20170112
+260100012;0;PSY;4;21;3;1,1;20170112
+260100012;0;PSY;4;21;4a;1,32;20170112
+260100012;0;PSY;4;21;6a;1,11;20170112
+260100012;0;PSY;4;21;7;0,13;20170112
+260100012;0;PSY;4;21;8;10,7;20170112
+260100012;0;PSY;4;21;1;23,11;20170113
+260100012;0;PSY;4;21;2;63,06;20170113
+260100012;0;PSY;4;21;3a;4,61;20170113
+260100012;0;PSY;4;21;3b;3,67;20170113
+260100012;0;PSY;4;21;3c;2,81;20170113
+260100012;0;PSY;4;21;3;1,1;20170113
+260100012;0;PSY;4;21;4a;1,32;20170113
+260100012;0;PSY;4;21;6a;1,11;20170113
+260100012;0;PSY;4;21;7;0,13;20170113
+260100012;0;PSY;4;21;8;10,7;20170113
+260100012;0;PSY;4;21;1;23,11;20170114
+260100012;0;PSY;4;21;2;63,06;20170114