javascript integration for histream-import, still bugs in VisitPostProcessorQueue

histream-import/pom.xml

@@ -52,6 +52,12 @@
 			<artifactId>histream-core</artifactId>
 			<version>0.8-SNAPSHOT</version>
 		</dependency>
+		<!-- script support -->
+		<dependency>
+			<groupId>de.sekmi.histream</groupId>
+			<artifactId>histream-js</artifactId>
+			<version>0.8-SNAPSHOT</version>
+		</dependency>
 		<!-- add later for sorting data tables
 		<dependency>
 			<groupId>com.google.protobuf</groupId>

histream-import/src/main/java/de/sekmi/histream/etl/ETLObservationSupplier.java

@@ -7,16 +7,12 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import javax.xml.bind.JAXB;
-
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.ObservationSupplier;
 import de.sekmi.histream.etl.config.DataSource;
 import de.sekmi.histream.etl.config.EavTable;
 import de.sekmi.histream.etl.config.Meta;
-import de.sekmi.histream.etl.config.PatientTable;
-import de.sekmi.histream.etl.config.VisitTable;
 import de.sekmi.histream.etl.config.WideTable;
 import de.sekmi.histream.impl.ObservationFactoryImpl;
 import de.sekmi.histream.impl.SimplePatientExtension;
@@ -112,7 +108,7 @@ public class ETLObservationSupplier implements ObservationSupplier{
 	 * @throws ParseException configuration error
 	 */
 	public ETLObservationSupplier(DataSource ds, ObservationFactory factory) throws IOException, ParseException {
-		this(ds,factory,new FactGroupingQueue());
+		this(ds,factory,ds.createFactQueue(factory));
 	}
 	/**
 	 * Construct a new observation supplier directly from a {@link DataSource}.
@@ -123,7 +119,7 @@ public class ETLObservationSupplier implements ObservationSupplier{
 	 * @throws IOException error reading configuration or table data
 	 * @throws ParseException configuration error
 	 */
-	public ETLObservationSupplier(DataSource ds, ObservationFactory factory, FactGroupingQueue queue) throws IOException, ParseException {
+	protected ETLObservationSupplier(DataSource ds, ObservationFactory factory, FactGroupingQueue queue) throws IOException, ParseException {
 		this.ds = ds;
 		this.sync = new ExtensionSync(factory);
 		this.queue = queue;

histream-import/src/main/java/de/sekmi/histream/etl/ScriptProcessingQueue.java

+package de.sekmi.histream.etl;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import java.net.URL;
+
+import javax.script.ScriptException;
+
+import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.etl.config.Script;
+import de.sekmi.histream.scripting.EncounterScriptEngine;
+
+public class ScriptProcessingQueue extends VisitPostProcessorQueue {
+	private EncounterScriptEngine engine;
+
+	public ScriptProcessingQueue(Script[] scripts, URL baseURL, ObservationFactory factory) throws IOException {
+		try {
+			engine = new EncounterScriptEngine();
+		} catch (ScriptException e) {
+			throw new IOException("Unable to create script engine", e);
+		}
+		engine.setObservationFactory(factory);
+
+		// load scripts
+		for( int i=0; i<scripts.length; i++ ){
+			try( Reader r = scripts[i].openReader(baseURL) ){
+				engine.addScript(r);
+			} catch (ScriptException e) {
+				throw new IOException("Script error in script "+i, e);
+			}
+		}
+	}
+	@Override
+	protected void postProcessVisit() {
+		if( getVisit() == null ){
+			return; // don't want null visits
+		}
+		try {
+			engine.processEncounter(getPatient().getId(), getVisit().getId(), getVisit().getStartTime(), getVisitFacts());
+		} catch (ScriptException e) {
+			IOException io = new IOException("Error during script execution for patient="+getPatient().getId()+", visit="+getVisit().getId(), e);
+			throw new UncheckedIOException(io);
+		}
+	}
+
+}

histream-import/src/main/java/de/sekmi/histream/etl/VisitPostProcessorQueue.java

@@ -75,7 +75,8 @@ public abstract class VisitPostProcessorQueue extends FactGroupingQueue {
 			// next fact
 			nextFact = super.get();
 		}
-		
+		// TODO visit can be processed, without having any facts left to return (either there were no facts in the visit or or postProcessing removed all facts)
+		// TODO maybe rename visitProcessed to visitProcessedNonempty
 		if( visitProcessed ){
 			if( !visitFacts.isEmpty() ){
 				// remove first

histream-import/src/main/java/de/sekmi/histream/etl/config/DataSource.java

 package de.sekmi.histream.etl.config;
 
+import java.io.IOException;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.List;
 
+import javax.script.ScriptException;
 import javax.xml.bind.JAXB;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -12,6 +14,11 @@ import javax.xml.bind.annotation.XmlElementWrapper;
 import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 
+import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.etl.FactGroupingQueue;
+import de.sekmi.histream.etl.ScriptProcessingQueue;
+import de.sekmi.histream.etl.VisitPostProcessorQueue;
+
 /**
  * Data source configuration.
  * This is the XML root element which can be loaded 
@@ -88,4 +95,27 @@ public class DataSource {
 		ds.getMeta().setLocation(configuration);
 		return ds;
 	}
+
+	/**
+	 * If scripts are present, an instance of {@link ScriptProcessingQueue}
+	 * is returned. Otherwise an instance of {@link FactGroupingQueue}.
+	 * @return fact queue
+	 * @throws IOException 
+	 * @throws  
+	 */
+	public FactGroupingQueue createFactQueue(ObservationFactory factory) throws IOException{
+//		if( true ){
+//			// TODO debug problems with visitpostprocessorqueue
+//			return new VisitPostProcessorQueue() {
+//				@Override
+//				protected void postProcessVisit() {
+//				}
+//			};
+//		}
+		if( true || scripts == null || scripts.length == 0 ){
+			return new FactGroupingQueue();
+		}else{
+			return new ScriptProcessingQueue(scripts, meta.getLocation(), factory);
+		}
+	}
 }

histream-import/src/main/java/de/sekmi/histream/etl/config/Script.java

 package de.sekmi.histream.etl.config;
 
 
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.net.URL;
+
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlValue;
 
@@ -27,4 +34,15 @@ public class Script {
 	
 	@XmlValue
 	public String content;
+	
+	public Reader openReader(URL baseURL) throws IOException{
+		if( content != null ){
+			return new StringReader(content);
+		}else if( src != null ){
+			URL url = new URL(baseURL, src);
+			return new InputStreamReader(url.openStream(), this.charset);
+		}else{
+			return null;
+		}
+	}
 }

histream-import/src/test/resources/data/test-1-datasource.xml

@@ -134,9 +134,11 @@
 			<!-- ... more value elements -->
 		</virtual>
 	</eav-table>
-
-	<script type="text/javascript"><![CDATA[ 
-	
+	<!-- scripts are run for each complete encounter in the order of occurrence -->
+	<script type="text/javascript"><![CDATA[
+		if( facts.get("natrium") && facts.get("kalium") ){
+			facts.add("nakl").value(1);
+		}
 	]]>
 	</script>
 	<script type="text/javascript" charset="UTF-8" src="test-1-script.js"/>

histream-import/src/test/resources/data/test-1-script.js

+// add a fact which indicates the number of observations
+// in this encounter
+
+facts.add("COUNT").value(facts.size());
\ No newline at end of file

histream-js/src/main/java/de/sekmi/histream/scripting/EncounterScriptEngine.java

+package de.sekmi.histream.scripting;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+import java.util.LinkedList;
+import java.util.List;
+
+import javax.script.Bindings;
+import javax.script.Compilable;
+import javax.script.CompiledScript;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+
+import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.Observation;
+import de.sekmi.histream.ObservationFactory;
+
+
+public class EncounterScriptEngine {
+	private ScriptEngine engine;
+	private List<CompiledScript> scripts;
+	private ObservationFactory factory;
+	
+	public EncounterScriptEngine() throws javax.script.ScriptException {
+		this.engine = new ScriptEngineManager().getEngineByName("nashorn");
+		if( engine == null ){
+			throw new ScriptException("Script engine not available");
+		}else if( !(engine instanceof Compilable) ){
+			throw new ScriptException("Script engine does not support compilation");
+		}
+		// TODO use custom context for error writer and global bindings
+		
+		// use strict mode
+		engine.eval("'use strict';");
+		scripts = new LinkedList<>();
+	}
+
+	public void addScript(String script) throws ScriptException{
+		scripts.add(((Compilable)engine).compile(script));
+	}
+	public void addScript(URL location, String charset) throws ScriptException, IOException{
+		try(
+				InputStream in = location.openStream();
+				Reader reader = new InputStreamReader(in, charset)
+		){
+			addScript(reader);
+		}
+	}
+	public void addScript(Reader reader) throws ScriptException{
+		scripts.add(((Compilable)engine).compile(reader));	
+	}
+	
+	public void setObservationFactory(ObservationFactory factory){
+		this.factory = factory;
+	}
+	public void processEncounter(String patientId, String encounterId, DateTimeAccuracy defaultStartTime, List<Observation> facts) throws ScriptException{
+		Facts f = new Facts(factory, patientId, encounterId, defaultStartTime);
+		f.setObservations(facts);
+		Bindings b = engine.createBindings();
+		b.put("facts", f);
+		for( CompiledScript script : scripts ){
+			script.eval(b);
+		}
+		// TODO is there a way to add information which script threw an exception?
+	}
+}

histream-js/src/main/java/de/sekmi/histream/scripting/Fact.java

 package de.sekmi.histream.scripting;
 
+import java.math.BigDecimal;
+
 import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.Value;
+import de.sekmi.histream.impl.NumericValue;
 import de.sekmi.histream.impl.StringValue;
 
 public class Fact {
@@ -27,11 +30,31 @@ public class Fact {
 	}
 	
 	public Value value(String value){
+		if( value == null ){
+			observation.setValue(null);
+			return null;
+		}
 		Value v = new StringValue(value);
 		observation.setValue(v);
 		return v;
 	}
 	
+	/**
+	 * Javascript treats all values as double, hence this method
+	 * will be used by javascript calls using numbers.
+	 * @param value value
+	 * @return value object
+	 */
+	public Value value(Double value){
+		if( value == null ){
+			observation.setValue(null);
+			return null;
+		}
+		Value v = new NumericValue(new BigDecimal(value));
+		observation.setValue(v);
+		return v;
+	}
+	
 	public Observation getObservation(){
 		return observation;
 	}

histream-js/src/test/java/de/sekmi/histream/scripting/TestEncounterScriptEngine.java

+package de.sekmi.histream.scripting;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import javax.script.ScriptException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import de.sekmi.histream.DateTimeAccuracy;
+import de.sekmi.histream.Observation;
+import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.Value;
+import de.sekmi.histream.impl.ObservationFactoryImpl;
+
+public class TestEncounterScriptEngine {
+
+	@Test
+	public void multipleCalls() throws Exception{
+		// prepare facts
+		ObservationFactory factory = new ObservationFactoryImpl();
+		Observation[] facts = new Observation[]{
+				factory.createObservation("P1", "C1", DateTimeAccuracy.parsePartialIso8601("2011")),
+				factory.createObservation("P1", "C2", DateTimeAccuracy.parsePartialIso8601("2011-02-01")),
+				factory.createObservation("P1", "C3", DateTimeAccuracy.parsePartialIso8601("2011-02-01"))				
+		};
+		List<Observation> list = new ArrayList<>();
+		Collections.addAll(list, facts);
+
+		// prepare engine
+		EncounterScriptEngine e = new EncounterScriptEngine();
+		e.setObservationFactory(factory);
+		e.addScript(getClass().getResource("/postprocess-encounter-1.js"), "UTF-8");
+
+		// run
+		e.processEncounter("P1", "E1", DateTimeAccuracy.parsePartialIso8601("2011-02-01"), list);
+
+		// verify
+		Observation last = list.get(list.size()-1);
+		Assert.assertEquals("test1", last.getConceptId());
+		Assert.assertEquals(Value.Type.Numeric, last.getValue().getType());
+	}
+}