use source timestamps from import tables/files (56150288) · Commits · Raphael Majeed / histream

histream-core/src/main/java/de/sekmi/histream/impl/ExternalSourceImpl.java

+19 −3

Original line number	Diff line number	Diff line
		@@ -15,18 +15,34 @@ public class ExternalSourceImpl implements ExternalSourceType {
		private Instant timestamp;
		private String id;

		public static class Adapter extends XmlAdapter<String, Instant>{
		/**
		* Empty constructor for JAXB
		*/
		public ExternalSourceImpl(){
		}
		/**
		* Create external source
		* @param id id string
		* @param timestamp timestamp
		*/
		public ExternalSourceImpl(String id, Instant timestamp){
		this();
		this.id = id;
		this.timestamp = timestamp;
		}

		private static class Adapter extends XmlAdapter<String, Instant>{
		@Override
		public Instant unmarshal(String v) throws Exception {
		if( v == null )return null;
		return javax.xml.bind.DatatypeConverter.parseDateTime(v).toInstant();
		}

		@Override
		public String marshal(Instant v) throws Exception {
		if( v == null )return null;
		return v.toString();
		}

		}

		@XmlAttribute(name="timestamp")

histream-core/src/test/java/de/sekmi/histream/io/TestXMLWriter.java

+7 −3

Original line number	Diff line number	Diff line
		@@ -37,12 +37,16 @@ public class TestXMLWriter {
		public void testWriteMeta(){
		Meta meta = new Meta();
		meta.etlStrategy = "lala";
		meta.source = new ExternalSourceImpl();
		meta.source.setSourceId("sid");
		meta.source.setSourceTimestamp(Instant.now());
		meta.source = new ExternalSourceImpl("sid", Instant.now());
		meta.order = new Meta.Order(true,false);

		JAXB.marshal(meta, System.out);

		meta = new Meta();
		meta.etlStrategy = "lala";
		meta.source = new ExternalSourceImpl("sid", null);
		meta.order = null;
		JAXB.marshal(meta, System.out);
		}

		@Test

histream-import/src/main/java/de/sekmi/histream/etl/ETLObservationSupplier.java

+30 −15

Original line number	Diff line number	Diff line
		@@ -62,25 +62,29 @@ public class ETLObservationSupplier implements ObservationSupplier{

		private FactGroupingQueue queue;

		private DataSource ds;

		public ETLObservationSupplier(DataSource ds, ObservationFactory factory) throws IOException, ParseException {
		this.ds = ds;

		pt = ds.getPatientTable();
		vt = ds.getVisitTable();
		wt = ds.getWideTables();
		// TODO long tables

		String sourceId = ds.getMeta().getSourceId();
		// in case of exception, make sure already opened suppliers are closed
		try{
		pr = pt.open(factory);
		vr = vt.open(factory);
		pr = pt.open(factory, sourceId);
		vr = vt.open(factory, sourceId);
		queue = new FactGroupingQueue(pr, vr,
		factory.getExtensionAccessor(Patient.class),
		factory.getExtensionAccessor(Visit.class),
		ds.getMeta().getSource());
		factory.getExtensionAccessor(Visit.class));

		// open all tables
		wr = new ArrayList<>(wt.size());
		for( WideTable t : wt ){
		RecordSupplier<WideRow> s = t.open(factory);
		RecordSupplier<WideRow> s = t.open(factory, sourceId);
		queue.addFactTable(s);
		wr.add(s);
		}
		@@ -117,6 +121,7 @@ public class ETLObservationSupplier implements ObservationSupplier{
		}
		vr=null;
		}
		if( wr != null ){
		Iterator<RecordSupplier<WideRow>> i = wr.iterator();
		while( i.hasNext() ){
		try{ i.next().close(); }
		@@ -126,13 +131,23 @@ public class ETLObservationSupplier implements ObservationSupplier{
		}
		i.remove();
		}
		}

		if( error != null )throw error;
		}

		@Override
		public String getMeta(String arg0) {
		// TODO Auto-generated method stub
		public String getMeta(String key) {
		switch( key ){
		case ObservationSupplier.META_ETL_STRATEGY:
		return ds.getMeta().getETLStrategy();
		case ObservationSupplier.META_SOURCE_ID:
		return ds.getMeta().getSourceId();
		case ObservationSupplier.META_ORDER_GROUPED:
		return "true";
		default:
		return null;
		}
		}

		}

histream-import/src/main/java/de/sekmi/histream/etl/FactGroupingQueue.java

+3 −6

Original line number	Diff line number	Diff line
		@@ -8,7 +8,6 @@ import java.util.Queue;

		import de.sekmi.histream.ExtensionAccessor;
		import de.sekmi.histream.Observation;
		import de.sekmi.histream.ext.ExternalSourceType;
		import de.sekmi.histream.ext.Patient;
		import de.sekmi.histream.ext.Visit;

		@@ -29,7 +28,6 @@ public class FactGroupingQueue{
		private ExtensionAccessor<Patient> patientAccessor;
		private ExtensionAccessor<Visit> visitAccessor;

		private ExternalSourceType metaSource;
		private List<RecordSupplier<? extends FactRow>> factTables;


		@@ -64,7 +62,7 @@ public class FactGroupingQueue{
		}


		public FactGroupingQueue(RecordSupplier<PatientRow> patientTable, RecordSupplier<VisitRow>visitTable, ExtensionAccessor<Patient> patientAccessor, ExtensionAccessor<Visit> visitAccessor, ExternalSourceType metaSource){
		public FactGroupingQueue(RecordSupplier<PatientRow> patientTable, RecordSupplier<VisitRow>visitTable, ExtensionAccessor<Patient> patientAccessor, ExtensionAccessor<Visit> visitAccessor){
		this.patientTable = patientTable;
		Objects.requireNonNull(patientAccessor);
		Objects.requireNonNull(visitAccessor);
		@@ -73,7 +71,6 @@ public class FactGroupingQueue{
		this.visitTable = visitTable;
		this.factTables = new ArrayList<>();
		this.workQueue = new ArrayDeque<>();
		this.metaSource = metaSource;
		}
		public void addFactTable(RecordSupplier<? extends FactRow> supplier){
		if( supplier == patientTable \|\| supplier == visitTable )throw new IllegalArgumentException("Cannot add patient or visit table as fact table");
		@@ -85,7 +82,7 @@ public class FactGroupingQueue{
		* Current patient changed: {@link #currentPatient}
		*/
		private void patientChanged(){
		currentPatientInstance = patientAccessor.accessStatic(currentPatient.getPatientId(), metaSource);
		currentPatientInstance = patientAccessor.accessStatic(currentPatient.getPatientId(), patientTable.getSource());
		currentPatientInstance.setBirthDate(currentPatient.getBirthDate());
		currentPatientInstance.setDeathDate(currentPatient.getDeathDate());
		currentPatientInstance.setSex(currentPatient.getSex());
		@@ -106,7 +103,7 @@ public class FactGroupingQueue{
		// TODO later support facts without encounter
		}else{
		// sync visit with extension factory
		currentVisitInstance = visitAccessor.accessStatic(currentVisitId, currentPatientInstance, metaSource);
		currentVisitInstance = visitAccessor.accessStatic(currentVisitId, currentPatientInstance, visitTable.getSource());
		currentVisitInstance.setStartTime(nextVisit.getStartTime());
		currentVisitInstance.setEndTime(nextVisit.getEndTime());
		currentVisitInstance.setLocationId(nextVisit.getLocationId());

histream-import/src/main/java/de/sekmi/histream/etl/FileRowSupplier.java

+37 −3

Original line number	Diff line number	Diff line
		package de.sekmi.histream.etl;

		import java.io.BufferedReader;
		import java.io.File;
		import java.io.IOException;
		import java.io.InputStreamReader;
		import java.io.UncheckedIOException;
		import java.net.URL;
		import java.time.Instant;
		import java.util.regex.Pattern;

		public class FileRowSupplier extends RowSupplier {
		@@ -12,6 +14,8 @@ public class FileRowSupplier extends RowSupplier {
		private BufferedReader in;
		private String[] headers;

		private Instant timestamp;

		public FileRowSupplier(URL location, String fieldSeparator) throws IOException{
		this(location, Pattern.compile(Pattern.quote(fieldSeparator)));
		}
		@@ -24,8 +28,33 @@ public class FileRowSupplier extends RowSupplier {
		// load headers
		String line = in.readLine();
		this.headers = fieldSeparatorPattern.split(line);

		determineFileTimestamp(location);
		}

		private void determineFileTimestamp(URL url) throws IOException{
		if( url.getProtocol().equals("file") ){
		// get file timestamp

		/*
		Path path;
		try {
		path = new File(url.toURI()).toPath();
		//Paths.get(url.getPath()); does not work with URLs like file:/C:/lala
		} catch (URISyntaxException e) {
		throw new IOException(e);
		}
		BasicFileAttributes atts = Files.readAttributes(path, BasicFileAttributes.class);
		this.timestamp = atts.creationTime().toInstant();
		*/
		this.timestamp = Instant.ofEpochMilli(new File(url.getPath()).lastModified());

		}else{
		throw new IOException("Unable to determine timestamp for URL: "+url);
		// TODO e.g. use URLConnection to get timestamp
		}

		}
		@Override
		public String[] getHeaders() {
		return headers;
		@@ -53,4 +82,9 @@ public class FileRowSupplier extends RowSupplier {
		in.close();
		}

		@Override
		public Instant getTimestamp() {
		return timestamp;
		}

		}