histream-core/examples/dwh-eav.xml

@@ -37,13 +37,14 @@
 		<birthdate>2001-01-01</birthdate>
 		<deathdate>2020</deathdate>
 		<sex>F</sex>
-		<encounter start="2014-01-01T10:30:00" end="2014-01-05T10:30:00">XXE12345</encounter>
+		<encounter start="2014-01-01T10:30:00" end="2014-01-05T10:30:00.123">XXE12345</encounter>
 		<location>Zuhause</location>
 		<!-- TODO inpatient/outpatient -->
 		<provider>xxxa</provider>
 		<facts>
 			<!--  test parsing of partial time stamps -->
 			<eav-item concept="T:date:secs" start="2014-09-07T10:40:03"/>
+			<eav-item concept="T:date:msec" start="2014-09-07T10:40:03.123"/>
 			<eav-item concept="T:date:mins" start="2014-09-07T10:40"/>
 			<eav-item concept="T:date:hours" start="2014-09-07T10"/>
 			<eav-item concept="T:date:day" start="2014-09-07"/>

histream-core/examples/dwh-flat.txt

@@ -23,9 +23,10 @@ XX12345	XXE12345	gest	dat	2020
 XX12345	XXE12345	sex	str	F
 XX12345	XXE12345	psurname	str	Dampf
 XX12345	XXE12345	pnames	str	A B
-XX12345	XXE12345	visit	@	@	@	2014-01-01T10:30:00	2014-01-05T10:30:00	xxxa	Zuhause
+XX12345	XXE12345	visit	@	@	@	2014-01-01T10:30:00	2014-01-05T10:30:00.123	xxxa	Zuhause
 # normale werte
 XX12345	XXE12345	T:date:secs	@	@	@	2014-09-07T10:40:03
+XX12345	XXE12345	T:date:msec	@	@	@	2014-09-07T10:40:03.123
 XX12345	XXE12345	T:date:mins	@	@	@	2014-09-07T10:40
 XX12345	XXE12345	T:date:hours	@	@	@	2014-09-07T10
 XX12345	XXE12345	T:date:day	@	@	@	2014-09-07

histream-core/src/main/java/de/sekmi/histream/DateTimeAccuracy.java

@@ -51,7 +51,7 @@ import de.sekmi.histream.xml.DateTimeAccuracyAdapter;
  */
 @XmlJavaTypeAdapter(DateTimeAccuracyAdapter.class)
 public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
-	static final String PARTIAL_FORMATTER_PATTERN = "u[-M[-d['T'H[:m[:s[.S]]][X]]]]";
+	static final String PARTIAL_FORMATTER_PATTERN = "u[-M[-d['T'H[:m[:s[.SSS]]][X]]]]";
 	static final DateTimeFormatter PARTIAL_FORMATTER  = DateTimeFormatter.ofPattern(PARTIAL_FORMATTER_PATTERN);
 
 	// TODO why not use instant, since we always calculate UTC? or Offset/ZonedDateTime?
@@ -67,33 +67,33 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 		this.accuracy = ChronoUnit.SECONDS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year) {
-		instant = LocalDateTime.of(year, 1, 1, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year) {
+		instant = LocalDateTime.of(year, 1, 1, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.YEARS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month) {
-		instant = LocalDateTime.of(year, month, 1, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month) {
+		instant = LocalDateTime.of(year, month, 1, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.MONTHS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day) {
-		instant = LocalDateTime.of(year, month, day, 0, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day) {
+		instant = LocalDateTime.of(year, month, day, 0, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.DAYS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours) {
-		instant = LocalDateTime.of(year, month, day, hours, 0).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours) {
+		instant = LocalDateTime.of(year, month, day, hours, 0).atZone(zone).toInstant();
 		accuracy = ChronoUnit.HOURS;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours, int mins) {
-		instant = LocalDateTime.of(year, month, day, hours, mins).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours, int mins) {
+		instant = LocalDateTime.of(year, month, day, hours, mins).atZone(zone).toInstant();
 		accuracy = ChronoUnit.MINUTES;
 	}
 	@Deprecated
-	public DateTimeAccuracy(int year, int month, int day, int hours, int mins, int secs) {
-		instant = LocalDateTime.of(year, month, day, hours, mins, secs).toInstant(ZoneOffset.UTC);
+	public DateTimeAccuracy(ZoneId zone, int year, int month, int day, int hours, int mins, int secs) {
+		instant = LocalDateTime.of(year, month, day, hours, mins, secs).atZone(zone).toInstant();
 		accuracy = ChronoUnit.SECONDS;
 	}
 	
@@ -201,9 +201,9 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 			dt = instant.atOffset(ZoneOffset.UTC).toLocalDateTime();
 		}
 		
-		char[] prefixes = {0,'-','-','T',':',':'};
-		ChronoField[] fields = {ChronoField.YEAR, ChronoField.MONTH_OF_YEAR, ChronoField.DAY_OF_MONTH, ChronoField.HOUR_OF_DAY, ChronoField.MINUTE_OF_HOUR, ChronoField.SECOND_OF_MINUTE};
-		int[] digits = {4,2,2,2,2,2};
+		char[] prefixes = {0,'-','-','T',':',':','.'};
+		ChronoField[] fields = {ChronoField.YEAR, ChronoField.MONTH_OF_YEAR, ChronoField.DAY_OF_MONTH, ChronoField.HOUR_OF_DAY, ChronoField.MINUTE_OF_HOUR, ChronoField.SECOND_OF_MINUTE,ChronoField.MILLI_OF_SECOND};
+		int[] digits = {4,2,2,2,2,2,3};
 		int i;
 		for( i=0; i<fields.length; i++ ){
 			if( prefixes[i] != 0 )b.append(prefixes[i]);
@@ -285,8 +285,9 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 		if( a.isSupported(ChronoField.NANO_OF_SECOND) ){
 			// maximum accuracy of nanoseconds
 			// not supported yet, truncate to seconds
-			accuracy = ChronoUnit.NANOS;
-			dateTime = LocalDateTime.from(a);
+			accuracy = ChronoUnit.MILLIS;
+//			dateTime = LocalDateTime.from(a);
+			dateTime = LocalDateTime.of(a.get(ChronoField.YEAR), a.get(ChronoField.MONTH_OF_YEAR), a.get(ChronoField.DAY_OF_MONTH), a.get(ChronoField.HOUR_OF_DAY), a.get(ChronoField.MINUTE_OF_HOUR), a.get(ChronoField.SECOND_OF_MINUTE), a.get(ChronoField.NANO_OF_SECOND));
 		}else if( a.isSupported(ChronoField.SECOND_OF_MINUTE) ){
 			accuracy = ChronoUnit.SECONDS;
 			dateTime = LocalDateTime.of(a.get(ChronoField.YEAR), a.get(ChronoField.MONTH_OF_YEAR), a.get(ChronoField.DAY_OF_MONTH), a.get(ChronoField.HOUR_OF_DAY), a.get(ChronoField.MINUTE_OF_HOUR), a.get(ChronoField.SECOND_OF_MINUTE));
@@ -334,54 +335,64 @@ public class DateTimeAccuracy implements Comparable<DateTimeAccuracy> {
 	 * 
 	 * @param formatter formatter
 	 * @param text input text
+	 * @param zoneId time zone to use, if the parser doesn't supply a time zone or offset
 	 * @return date time with accuracy
 	 */
-	public static DateTimeAccuracy parse(DateTimeFormatter formatter, CharSequence text){
+	public static DateTimeAccuracy parse(DateTimeFormatter formatter, CharSequence text, ZoneId zoneId){
 		ParsePosition pos = new ParsePosition(0);
 		TemporalAccessor a = formatter.parseUnresolved(text, pos);
 		if( pos.getErrorIndex() != -1 ){
-			throw new DateTimeParseException("Text '"+String.valueOf(text.charAt(pos.getErrorIndex()))+"' could not be parsed at index "+pos.getErrorIndex(), text, pos.getErrorIndex());
+			throw new DateTimeParseException("Text '"+text+"' could not be parsed at index "+pos.getErrorIndex(), text, pos.getErrorIndex());
 		}else if( pos.getIndex() != text.length() ){
 			throw new DateTimeParseException("Unparsed text found at index "+pos.getIndex(), text, pos.getIndex());
 		}
+		
+		try{
+			int offset = a.get(ChronoField.OFFSET_SECONDS);
+			// explicit offset specified, use that information
+			zoneId = ZoneOffset.ofTotalSeconds(offset);
+		}catch( DateTimeException e ){
+			// no offset available
+			// use default specified in zoneId param
+		}
 		int year = a.get(ChronoField.YEAR);
 		// month
 		int month;
 		try{
 			month = a.get(ChronoField.MONTH_OF_YEAR);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year);
+			return new DateTimeAccuracy(zoneId, year);
 		}
 		
 		int day;
 		try{
 			day = a.get(ChronoField.DAY_OF_MONTH);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month);
+			return new DateTimeAccuracy(zoneId, year,month);
 		}
 
 		int hour;
 		try{
 			hour = a.get(ChronoField.HOUR_OF_DAY);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day);
+			return new DateTimeAccuracy(zoneId, year,month,day);
 		}
 
 		int minute;
 		try{
 			minute = a.get(ChronoField.MINUTE_OF_HOUR);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day, hour);
+			return new DateTimeAccuracy(zoneId, year,month,day, hour);
 		}
 		
 		int seconds;
 		try{
 			seconds = a.get(ChronoField.SECOND_OF_MINUTE);
 		}catch( DateTimeException e ){
-			return new DateTimeAccuracy(year,month,day, hour, minute);
+			return new DateTimeAccuracy(zoneId, year,month,day, hour, minute);
 		}
 
-		return new DateTimeAccuracy(year,month,day, hour, minute, seconds);
+		return new DateTimeAccuracy(zoneId, year,month,day, hour, minute, seconds);
 		// milliseconds not supported for now
 	}
 	@Override

histream-core/src/main/java/de/sekmi/histream/impl/SimpleVisitExtension.java

@@ -47,7 +47,7 @@ public class SimpleVisitExtension implements Extension<VisitImpl>{
 		}
 		VisitImpl visit = new VisitImpl();
 		visit.setId((String)args[0]);
-		visit.setPatientId(((Patient)args[1]).getId());
+		visit.setPatient(((Patient)args[1]));
 		ExternalSourceType source = (ExternalSourceType)args[2];
 		visit.setSourceId(source.getSourceId());
 		visit.setSourceTimestamp(source.getSourceTimestamp());

histream-core/src/main/java/de/sekmi/histream/impl/VisitImpl.java

 package de.sekmi.histream.impl;
 
 
+import java.util.Objects;
+
 /*
  * #%L
  * histream
@@ -24,6 +26,7 @@ package de.sekmi.histream.impl;
 
 
 import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.ext.Patient;
 import de.sekmi.histream.ext.StoredExtensionType;
 import de.sekmi.histream.ext.Visit;
 
@@ -34,28 +37,36 @@ public class VisitImpl extends StoredExtensionType implements Visit {
 	private String patientId;
 	private String locationId;
 	
-	public VisitImpl(){
+	/**
+	 * Empty constructor protected, only
+	 * available to overriding classes.
+	 */
+	protected VisitImpl() {
 		
 	}
-	
-	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime, DateTimeAccuracy endTime, Status status){
+	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime){
 		setId(id);
 		this.patientId = patientId;
+		this.startTime = startTime;		
+		markDirty(true);
+	}
+	
+	public VisitImpl(String id, String patientId, DateTimeAccuracy startTime, DateTimeAccuracy endTime, Status status){
+		this(id, patientId, startTime);
 		this.status = status;
-		this.startTime = startTime;
 		this.endTime = endTime;
-		markDirty(true);
 	}
 
 	public String getPatientId(){return patientId;}
 	
-	public void setPatientId(String patientId){
+	public void setPatient(Patient patient){
+		Objects.requireNonNull(patient);
 		// patient id should not be changed normally.
-		this.patientId = patientId;
+		this.patientId = patient.getId();
 		// TODO need to update dirty flag?
 		markDirty(true);
 	}
-	
+
 	@Override
 	public DateTimeAccuracy getStartTime() {
 		return startTime;

histream-core/src/main/java/de/sekmi/histream/io/GroupedXMLWriter.java

@@ -13,6 +13,8 @@ import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamWriter;
 import javax.xml.transform.Result;
 
+import org.w3c.dom.DOMException;
+
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationException;
 import de.sekmi.histream.ext.ExternalSourceType;
@@ -394,7 +396,7 @@ public class GroupedXMLWriter extends GroupedObservationHandler{
 			formatIndent();
 			marshalFactWithContext(observation, observation.getExtension(Visit.class), meta.source);
 			formatNewline();
-		} catch (JAXBException | XMLStreamException e) {
+		} catch (JAXBException | XMLStreamException | DOMException e) {
 			throw new ObservationException(e);
 		}
 		this.observationCount ++;

histream-core/src/main/java/de/sekmi/histream/io/XMLObservationSupplier.java

@@ -36,6 +36,7 @@ import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
 
+// TODO remove this class
 @Deprecated
 public class XMLObservationSupplier extends XMLObservationParser implements ObservationSupplier{
 	//private static final String namespaceURI = "http://sekmi.de/histream/dwh-eav";

histream-core/src/test/java/de/sekmi/histream/TestDateTimeAccuracy.java

@@ -23,7 +23,7 @@ public class TestDateTimeAccuracy {
 	public void testParseYYYYDD(){
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("M.u");
 		String text = "02.2003";
-		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, text);
+		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, text, ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.MONTHS,a.getAccuracy());
 		TemporalAccessor ac = a.toInstantMin().atOffset(ZoneOffset.UTC);
 		Assert.assertEquals(2, ac.get(ChronoField.MONTH_OF_YEAR));
@@ -35,10 +35,10 @@ public class TestDateTimeAccuracy {
 		DateTimeFormatter formatter = DateTimeFormatter.ofPattern("d.M.u[ H[:m[:s]]]");
 		DateTimeAccuracy a;
 		ZoneId tz = ZoneId.of("Asia/Shanghai"); // China standard time
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003", ZoneOffset.UTC);
 		Assert.assertEquals("2003-02-01", a.toPartialIso8601(null));
 		Assert.assertEquals("2003-02-01", a.toPartialIso8601(tz));
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13", ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.HOURS, a.getAccuracy());
 		Assert.assertEquals("2003-02-01T13", a.toPartialIso8601(null));
 		Assert.assertEquals("2003-02-01T21+0800", a.toPartialIso8601(tz));
@@ -82,6 +82,8 @@ public class TestDateTimeAccuracy {
 		assertEquals(ChronoUnit.MINUTES, a.getAccuracy());
 		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06");
 		assertEquals(ChronoUnit.SECONDS, a.getAccuracy());
+		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06.789");
+		assertEquals(ChronoUnit.MILLIS, a.getAccuracy());
 		// verify zone offset
 		// for second accuracy
 		a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06+0800");
@@ -103,13 +105,13 @@ public class TestDateTimeAccuracy {
 		formatter.withResolverStyle(ResolverStyle.STRICT);
 
 		DateTimeAccuracy a;
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003",  ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.DAYS,a.getAccuracy());
 		
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13",  ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.HOURS,a.getAccuracy());
 
-		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13:14");
+		a = DateTimeAccuracy.parse(formatter, "01.02.2003 13:14", ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.MINUTES,a.getAccuracy());
 	}
 
@@ -144,16 +146,16 @@ public class TestDateTimeAccuracy {
 		}
 		// verify same behavior for DateTimeAccurecy
 		// should not fail below
-		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, "2003");
+		DateTimeAccuracy a = DateTimeAccuracy.parse(formatter, "2003" ,ZoneOffset.UTC);
 		Assert.assertEquals(ChronoUnit.YEARS, a.getAccuracy());
 		// next two calls should throw exceptions
 		try{
-			DateTimeAccuracy.parse(formatter, "+");
+			DateTimeAccuracy.parse(formatter, "+", ZoneOffset.UTC);
 			Assert.fail("Exception unexpected input");
 		}catch( DateTimeParseException e ){
 		}
 		try{
-			DateTimeAccuracy.parse(formatter, "2003+");
+			DateTimeAccuracy.parse(formatter, "2003+", ZoneOffset.UTC);
 			Assert.fail("Exception expected for unparsed text at end of input");
 		}catch( DateTimeParseException e ){
 		}
@@ -165,6 +167,13 @@ public class TestDateTimeAccuracy {
 		assertEquals("2001-02-02T20Z", a.toPartialIso8601(ZoneId.of("UTC")));
 	}
 
+	@Test
+	public void verifyMillisecondParseAndToString() throws ParseException{
+		DateTimeAccuracy a = DateTimeAccuracy.parsePartialIso8601("2001-02-03T04:05:06.789", ZoneId.of("UTC"));
+		// date should be treated as if it had a +08:00 offset
+		assertEquals("2001-02-03T04:05:06.789Z", a.toPartialIso8601(ZoneId.of("UTC")));
+	}
+
 	@Test
 	public void verifyComparison() throws ParseException{
 		ZoneId zone = ZoneOffset.UTC.normalized();

histream-core/src/test/java/de/sekmi/histream/impl/ECMAEvaluatorTest.java

@@ -35,9 +35,10 @@ public class ECMAEvaluatorTest {
 		}
 
 		// skip to string value
-		for( int i=0; i<6; i++ ){
+		for( int i=0; i<7; i++ ){
 			o = s.get();
 		}
+		Assert.assertEquals("T:type:str", o.getConceptId());
 
 		// compare string value
 		Assert.assertEquals(true, eval.test("fact.value != null", o));

histream-core/src/test/java/de/sekmi/histream/impl/ObservationImplJAXBTest.java

@@ -8,6 +8,7 @@ import java.io.StringWriter;
 import java.math.BigDecimal;
 import java.net.URL;
 import java.time.Instant;
+import java.time.ZoneOffset;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.JAXB;
@@ -61,7 +62,7 @@ public class ObservationImplJAXBTest {
 		o.conceptId = "C"+index;
 		o.patientId = "P"+index;
 		o.encounterId = "E"+index;
-		o.startTime = new DateTimeAccuracy(2015,1,1,index);
+		o.startTime = new DateTimeAccuracy(ZoneOffset.UTC, 2015,1,1,index);
 		switch( index ){
 		case 0:
 			// string value

histream-core/src/test/java/de/sekmi/histream/impl/XPathEvaluatorTest.java

@@ -38,9 +38,10 @@ public class XPathEvaluatorTest {
 			Assert.assertEquals(expr, true, eval.test(expr, o));
 		}
 		// skip to string value
-		for( int i=0; i<6; i++ ){
+		for( int i=0; i<7; i++ ){
 			o = s.get();
 		}
+		Assert.assertEquals("T:type:str", o.getConceptId());
 
 		// compare string value
 		Assert.assertEquals(true, eval.test("f:fact/f:value='abc123'", o));

histream-core/src/test/java/de/sekmi/histream/io/FileObservationProviderTest.java

@@ -101,7 +101,7 @@ public class FileObservationProviderTest {
 				Assert.assertEquals("Zuhause", v.getLocationId());
 				try{
 					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-01T10:30:00"), v.getStartTime());
-					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-05T10:30:00"), v.getEndTime());
+					Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2014-01-05T10:30:00.123"), v.getEndTime());
 				} catch (ParseException e) {
 					throw new RuntimeException(e);
 				}
@@ -111,6 +111,12 @@ public class FileObservationProviderTest {
 				ExternalSourceType s = o.getSource();
 				Assert.assertNotNull(s);
 			},
+			(Observation o) ->  {
+				Assert.assertEquals("T:date:msec", o.getConceptId());
+				// TODO store and calculate time in nanos
+				Assert.assertEquals(ChronoUnit.MILLIS, o.getStartTime().getAccuracy());
+				Assert.assertEquals(123, o.getStartTime().toInstantMin().atOffset(ZoneOffset.UTC).getLong(ChronoField.MILLI_OF_SECOND));
+			},
 			(Observation o) ->  {
 				Assert.assertEquals("T:date:mins", o.getConceptId());
 				Assert.assertEquals(ChronoUnit.MINUTES, o.getStartTime().getAccuracy());
@@ -232,6 +238,7 @@ public class FileObservationProviderTest {
 	
 	@Test
 	public void testStAXReader() throws FileNotFoundException, XMLStreamException, FactoryConfigurationError  {
+		// TODO delete XMLObservationSupplier
 		XMLObservationSupplier xos = new XMLObservationSupplier(factory, new FileInputStream("examples/dwh-eav.xml"));
 		validateExample(xos);
 		xos.close();

histream-core/src/test/java/de/sekmi/histream/io/TestXMLWriter.java

@@ -7,11 +7,17 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.text.ParseException;
 import java.time.Instant;
 import java.time.ZoneId;
+import java.time.ZoneOffset;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.JAXB;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlValue;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
@@ -33,10 +39,19 @@ import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
 import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationException;
+import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
+import de.sekmi.histream.ext.ExternalSourceType;
+import de.sekmi.histream.ext.Patient;
+import de.sekmi.histream.ext.Visit;
 import de.sekmi.histream.impl.ExternalSourceImpl;
 import de.sekmi.histream.impl.Meta;
+import de.sekmi.histream.impl.ObservationFactoryImpl;
+import de.sekmi.histream.impl.SimplePatientExtension;
+import de.sekmi.histream.impl.SimpleVisitExtension;
+import de.sekmi.histream.impl.StringValue;
 import de.sekmi.histream.xml.XMLUtils;
 
 public class TestXMLWriter {
@@ -66,6 +81,7 @@ public class TestXMLWriter {
 		Document doc = builder.newDocument();
 		doc.getDomConfig().setParameter("namespaces", true);
 		doc.getDomConfig().setParameter("namespace-declarations", true);
+		// TODO check if DOM can be configured to allow newline characters in values
 
 // not suppoted by default implementation
 //		doc.getDomConfig().setParameter("canonical-form", true);
@@ -102,6 +118,19 @@ public class TestXMLWriter {
 
 		w.writeEndDocument();		
 	}
+
+	@Test
+	public void testMarshallNewlineInValues() throws ParserConfigurationException, ParseException {
+		DOMResult result = new DOMResult(createDocument());
+		ObservationFactory factory = new ObservationFactoryImpl();
+		factory.registerExtension(new SimplePatientExtension());
+		factory.registerExtension(new SimpleVisitExtension());
+		
+		Observation o = factory.createObservation("A", "B", DateTimeAccuracy.parsePartialIso8601("2018", ZoneOffset.UTC.normalized()));
+		o.setValue(new StringValue("1\n2"));
+		JAXB.marshal(o, result);
+		
+	}
 	@Test
 	public void testStreamWriterNamespaces() throws XMLStreamException, ParserConfigurationException{
 		XMLOutputFactory factory = XMLOutputFactory.newInstance();
@@ -164,6 +193,42 @@ public class TestXMLWriter {
 			reader.close();
 		}
 	}
+	@XmlRootElement
+	private static class ValType{
+		public ValType() {}
+		public ValType(String s) { this.value = s;}
+		@XmlValue 
+		String value;
+	}
+	@Test
+	public void testWriteDOM2() throws Exception{
+
+		// create document
+		DocumentBuilderFactory f = DocumentBuilderFactory.newInstance();
+//		f.setNamespaceAware(true);
+//		f.setCoalescing(true);
+//		f.setIgnoringComments(true);
+		DocumentBuilder builder = f.newDocumentBuilder();
+		Document doc = builder.newDocument();
+//		doc.getDomConfig().setParameter("namespaces", true);
+//		doc.getDomConfig().setParameter("namespace-declarations", true);
+
+		DOMResult dr = new DOMResult(doc);
+
+		XMLOutputFactory factory = XMLOutputFactory.newInstance();
+		// enable repairing namespaces to remove duplicate namespace declarations by JAXB marshal
+		// this does not work with the DOM stream writer
+
+//		factory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, Boolean.TRUE);
+		XMLStreamWriter writer = factory.createXMLStreamWriter(dr);
+		Marshaller marshaller = JAXBContext.newInstance(ValType.class).createMarshaller();
+//		marshaller.setProperty(Marshaller.JAXB_FRAGMENT, Boolean.TRUE);
+		marshaller.marshal(new ValType("1\n2"), writer);
+		doc.normalizeDocument();
+		XMLUtils.printDOM(doc, debugLog);
+
+	}
+	
 	@Test
 	public void testWriteDOM() throws Exception{
 
@@ -175,6 +240,17 @@ public class TestXMLWriter {
 		w.setZoneId(ZoneId.of("Asia/Shanghai"));
 		Meta.transfer(s, w);
 		Streams.transfer(s, w);
+		
+		// manually create observation
+		Observation o = t.getFactory().createObservation("A", "B", DateTimeAccuracy.parsePartialIso8601("2018", ZoneOffset.UTC.normalized()));
+		o.setValue(new StringValue("1\n2"));
+		ExternalSourceType es = new ExternalSourceImpl("manual", Instant.now());
+		o.setSource(es);
+		Patient pat = t.getFactory().getExtension(Patient.class).createInstance("A",es);
+		o.setExtension(Patient.class, pat);
+		o.setExtension(Visit.class, t.getFactory().getExtension(Visit.class).createInstance("V",pat,es));
+		w.accept(o);
+
 		w.close();
 		s.close();
 

histream-core/src/test/resources/dwh.xml

@@ -40,13 +40,14 @@
 		<source timestamp="2015-09-28T09:41:10Z"/>
 		<encounter id="XXE12345">
 			<start>2014-01-01T10:30:00</start>
-			<end>2014-01-05T10:30:00</end>
+			<end>2014-01-05T10:30:00.123</end>
 			<location>Zuhause</location>
 			<!-- TODO inpatient/outpatient 
 			<provider>xxxa</provider>-->
 			<source timestamp="2015-09-28T08:41:10Z"/>
 <!-- no more <facts> group -->
 			<fact concept="T:date:secs" start="2014-09-07T10:40:03"/>
+			<fact concept="T:date:msec" start="2014-09-07T10:40:03.123"/>
 			<fact concept="T:date:mins" start="2014-09-07T10:40"/>
 			<fact concept="T:date:hours" start="2014-09-07T10"/>
 			<fact concept="T:date:day" start="2014-09-07"/>

histream-export/src/main/java/de/sekmi/histream/export/VisitFragmentParser.java

@@ -140,7 +140,7 @@ abstract class VisitFragmentParser extends GroupedXMLWriter {
 	/**
 	 * Called after each patient fragment was parsed.
 	 * The patient fragment does not contain any encounters,
-	 * these are provided to {@link #visitFragment(Node)}.
+	 * these are provided to {@link #visitFragment(Element)}.
 	 * @param patient patient node
 	 * @throws ObservationException error
 	 */

histream-export/src/main/java/de/sekmi/histream/export/csv/Table.java

@@ -14,8 +14,8 @@ class Table implements TableWriter{
 	private final CSVWriter export;
 	private PrintWriter out;
 	/**
-	 * @param csvWriter
-	 * @throws IOException 
+	 * @param csvWriter CSV writer
+	 * @throws IOException IO error
 	 */
 	Table(CSVWriter csvWriter, String filename) throws IOException {
 		export = csvWriter;

histream-export/src/test/java/de/sekmi/histream/export/TestExport.java

@@ -44,8 +44,8 @@ public class TestExport {
 		Assert.assertEquals("T:type:str", m.get(MemoryExportWriter.VISIT_TABLE, "byclass", 0));
 		m.dump();
 		// verify eav table
-		Assert.assertEquals(6, m.rowCount("eavtabletest"));
-		Assert.assertEquals("T:date:month", m.get("eavtabletest", "code", 4));
+		Assert.assertEquals(7, m.rowCount("eavtabletest"));
+		Assert.assertEquals("T:date:month", m.get("eavtabletest", "code", 5));
 		
 		// TODO something wrong with namespaces in xpath/dom
 	}

histream-export/src/test/java/de/sekmi/histream/export/config/TestVisitFragmentParser.java

@@ -46,7 +46,7 @@ public class TestVisitFragmentParser {
 //		System.out.println("XPath="+ret);
 		ret = (String)xp.evaluate("count(eav:fact)", visit, XPathConstants.STRING);
 //		System.out.println("Facts:"+ret);
-		Assert.assertEquals("12", ret);
+		Assert.assertEquals("13", ret);
 	}
 	
 }

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2Extractor.java

@@ -58,8 +58,8 @@ public abstract class I2b2Extractor implements ObservationSupplier {
 	 * <p>
 	 * 	
 	 * </p>
-	 * @param factory
-	 * @param dbc
+	 * @param factory extractor factory
+	 * @param dbc database connection
 	 * @throws SQLException error
 	 */
 	I2b2Extractor(I2b2ExtractorFactory factory, Connection dbc) throws SQLException {

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2Visit.java

@@ -2,6 +2,7 @@ package de.sekmi.histream.i2b2;
 
 import java.time.temporal.ChronoUnit;
 
+import de.sekmi.histream.ext.Patient;
 import de.sekmi.histream.ext.Visit;
 
 /*
@@ -64,7 +65,17 @@ public class I2b2Visit extends VisitImpl {
 
 	public int getNum(){return encounter_num;}
 	public int getPatientNum(){return patient_num;}
-	
+
+	@Override
+	public void setPatient(Patient patient) {
+		super.setPatient(patient);
+		if( patient instanceof I2b2Patient ) {
+			// also set the patient_num 
+			int patient_num = ((I2b2Patient)patient).getNum();
+			this.patient_num = patient_num;
+		}
+	}
+
 	@Override
 	public String toString(){
 		return "I2b2Visit(encounter_um="+encounter_num+")";