DataDialect: timezone for database timestamp with conversions (33910d0a) · Commits · Raphael Majeed / histream

histream-core/src/main/java/de/sekmi/histream/ObservationExtractor.java

+3 −1

Original line number	Diff line number	Diff line
		package de.sekmi.histream;

		import java.io.IOException;
		import java.time.Instant;

		/**
		@@ -14,6 +15,7 @@ public interface ObservationExtractor {
		/**
		* Extract observations with a start time stamp between the specified limits.
		* Only observations with the specified notations are extracted.
		* TODO evaluate change from ObservationException to IOException
		*
		* @param start_min minimum time for observation start (inclusive)
		* @param start_max maximum time for observation start (inclusive)
		@@ -21,5 +23,5 @@ public interface ObservationExtractor {
		* @return supplier for the extracted observations. Must be closed after use.
		* @throws ObservationException error (e.g. database failure)
		*/
		ObservationSupplier extract(Instant start_min, Instant start_max, Iterable<String> notations) throws ObservationException;
		ObservationSupplier extract(Instant start_min, Instant start_max, Iterable<String> notations) throws IOException;
		}

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/DataDialect.java

+48 −1

Original line number	Diff line number	Diff line
		package de.sekmi.histream.i2b2;

		import java.sql.Timestamp;
		import java.time.Instant;
		import java.time.ZoneId;
		import java.time.ZoneOffset;

		import de.sekmi.histream.AbnormalFlag;
		import de.sekmi.histream.DateTimeAccuracy;
		import de.sekmi.histream.Value;

		/**
		* Configuration of exact meaning of values in
		* {@code observation_fact} table. E.g. how to
		* store/interpret null values.
		* <p>
		* The function calls beginning with {@code encode} produce
		* values which are stored in the database. The {@code decode} functions
		* are used to decode the database values and produce a usable value.
		*
		* @author R.W.Majeed
		*
		@@ -18,6 +28,9 @@ public class DataDialect {
		private String nullModifierCd;
		private String nullValueFlagCd;
		private String nullValueTypeCd;
		/** Timezone for timestamp / date time columns */
		private ZoneId zoneId;
		// TODO nullSexCd, nullInOutCd

		public DataDialect(){
		this.nullUnitCd = "@"; // technically, null is allowed, but the demodata uses both '@' and ''
		@@ -28,10 +41,14 @@ public class DataDialect {
		this.nullValueTypeCd = "@"; // TODO check database
		// null not allowed, use default
		this.nullProviderId = "@";
		this.zoneId = ZoneId.systemDefault();
		}
		void setDefaultProviderId(String providerId){
		this.nullProviderId = providerId;
		}
		public void setTimeZone(ZoneId zone){
		this.zoneId = zone;
		}
		public String getDefaultProviderId(){
		return nullProviderId;
		}
		@@ -43,7 +60,9 @@ public class DataDialect {
		public String getNullLocationCd(){
		return nullLocationCd;
		}

		public ZoneId getTimeZone(){
		return zoneId;
		}
		public String getNullModifierCd(){
		return nullModifierCd;
		}
		@@ -53,6 +72,34 @@ public class DataDialect {
		public String getNullValueTypeCd(){
		return nullValueTypeCd;
		}
		public Timestamp encodeInstant(Instant instant){
		if( instant == null ){
		return null;
		}else{
		return Timestamp.from(instant.atZone(zoneId).toLocalDateTime().atOffset(ZoneOffset.UTC).toInstant());
		}
		}
		public Timestamp encodeInstantPartial(DateTimeAccuracy instant){
		if( instant == null ){
		return null;
		}else{
		return encodeInstant(instant.toInstantMin());
		}
		}
		public Instant decodeInstant(Timestamp timestamp){
		if( timestamp == null ){
		return null;
		}else{
		return timestamp.toInstant().atOffset(ZoneOffset.UTC).toLocalDateTime().atZone(zoneId).toInstant();
		}
		}
		public DateTimeAccuracy decodeInstantPartial(Timestamp timestamp){
		if( timestamp == null ){
		return null;
		}else{
		return new DateTimeAccuracy(decodeInstant(timestamp));
		}
		}
		private boolean isNullComparison(String value, String nullValue){
		if( value == null ){
		return true;

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2ExtractorFactory.java

+6 −6

Original line number	Diff line number	Diff line
		package de.sekmi.histream.i2b2;

		import java.io.IOException;
		import java.sql.Connection;
		import java.sql.PreparedStatement;
		import java.sql.ResultSet;
		@@ -14,10 +15,8 @@ import java.util.logging.Logger;

		import javax.sql.DataSource;

		import de.sekmi.histream.ObservationException;
		import de.sekmi.histream.ObservationExtractor;
		import de.sekmi.histream.ObservationFactory;
		import de.sekmi.histream.ObservationSupplier;
		import de.sekmi.histream.ext.Patient;
		import de.sekmi.histream.ext.Visit;

		@@ -64,6 +63,7 @@ public class I2b2ExtractorFactory implements AutoCloseable, ObservationExtractor
		ds = crc_ds;
		dialect = new DataDialect();
		}

		public ObservationFactory getObservationFactory(){
		return observationFactory;
		}
		@@ -140,7 +140,7 @@ public class I2b2ExtractorFactory implements AutoCloseable, ObservationExtractor
		* @throws SQLException error
		*/
		//@SuppressWarnings("resource")
		public I2b2Extractor extract(Timestamp start_min, Timestamp start_max, Iterable<String> notations) throws SQLException{
		I2b2Extractor extract(Timestamp start_min, Timestamp start_max, Iterable<String> notations) throws SQLException{
		// TODO move connection and prepared statement to I2b2Extractor
		PreparedStatement ps = null;
		ResultSet rs = null;
		@@ -204,11 +204,11 @@ public class I2b2ExtractorFactory implements AutoCloseable, ObservationExtractor

		}
		@Override
		public ObservationSupplier extract(Instant start_min, Instant start_max, Iterable<String> notations) throws ObservationException{
		public I2b2Extractor extract(Instant start_min, Instant start_max, Iterable<String> notations) throws IOException{
		try {
		return extract(Timestamp.from(start_min), Timestamp.from(start_max), notations);
		return extract(dialect.encodeInstant(start_min),dialect.encodeInstant(start_max), notations);
		} catch (SQLException e) {
		throw new ObservationException(e);
		throw new IOException(e);
		}
		}
		}

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/I2b2Inserter.java

+8 −7

Original line number	Diff line number	Diff line
		@@ -26,7 +26,6 @@ import java.io.IOException;
		import java.sql.Connection;
		import java.sql.PreparedStatement;
		import java.sql.SQLException;
		import java.sql.Timestamp;
		import java.util.HashSet;
		import java.util.Iterator;
		import java.util.Objects;
		@@ -55,6 +54,10 @@ import de.sekmi.histream.impl.AbstractObservationHandler;
		* valtype_cd: N numeric, B stored in observation_blob, T text, '@' no value, 'NLP' NLP result xml objects.
		* Undocumented but used in demodata: D: datetime "YYYY-MM-DD HH:mm" stored in tval_char, "YYYYMMDD.HHmm0" stored in nval_num.
		* <p>
		* Timestamp and datetime values are stored without timezone information. The timezone
		* which should be used when reading/writing to database can be specified via
		* the {@link DataDialect} param in {@link #open(Connection, DataDialect)}.
		* <p>
		* The most difficult part is handling the instance_num field.
		* By default, i2b2 uses a four byte signed integer for instance_num. Incrementing
		* instance_num for every record would lead eventually to a number overflow.
		@@ -88,9 +91,7 @@ public class I2b2Inserter extends AbstractObservationHandler implements Observat
		// initialize(config);
		// }
		public I2b2Inserter(){

		}

		private interface Preprocessor{
		void preprocess(Observation fact)throws SQLException;
		}
		@@ -273,7 +274,7 @@ public class I2b2Inserter extends AbstractObservationHandler implements Observat
		insertFact.setString(4, dialect.encodeProviderId(o.getProviderId()));
		// start_date
		Objects.requireNonNull(o.getStartTime());
		insertFact.setTimestamp(5, Timestamp.from(o.getStartTime().toInstantMin()));
		insertFact.setTimestamp(5, dialect.encodeInstant(o.getStartTime().toInstantMin()));

		insertFact.setString(6, (m==null)?dialect.getNullModifierCd():m.getConceptId());
		insertFact.setInt(7, instanceNum);
		@@ -324,12 +325,12 @@ public class I2b2Inserter extends AbstractObservationHandler implements Observat
		if( o.getEndTime() == null ){
		insertFact.setTimestamp(13, null);
		}else{
		insertFact.setTimestamp(13, Timestamp.from(o.getEndTime().toInstantMin()));
		insertFact.setTimestamp(13, dialect.encodeInstant(o.getEndTime().toInstantMin()));
		}
		// location_cd
		insertFact.setString(14, dialect.encodeLocationCd(o.getLocationId()));
		// download_date
		insertFact.setTimestamp(15, Timestamp.from(o.getSource().getSourceTimestamp()));
		insertFact.setTimestamp(15, dialect.encodeInstant(o.getSource().getSourceTimestamp()));
		insertFact.setString(16, o.getSource().getSourceId());

		insertFact.executeUpdate();

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/PostgresExtension.java

+1 −10

Original line number	Diff line number	Diff line
		@@ -22,9 +22,6 @@ package de.sekmi.histream.i2b2;


		import java.sql.SQLException;
		import java.sql.Timestamp;

		import de.sekmi.histream.DateTimeAccuracy;
		import de.sekmi.histream.Extension;
		/**
		* Extension with database connectivity.
		@@ -92,12 +89,6 @@ public abstract class PostgresExtension<T> implements Extension<T> {
		// }
		// }

		public static Timestamp inaccurateSqlTimestamp(DateTimeAccuracy dateTime){
		if( dateTime == null )return null;
		else return Timestamp.from(dateTime.toInstantMin());
		}


		/**
		* Write updates to disk.
		*/