Commit 58551eac authored by R.W.Majeed's avatar R.W.Majeed

javascript integration for histream-import, still bugs in VisitPostProcessorQueue

parent 2d256f9d
......@@ -52,6 +52,12 @@
<artifactId>histream-core</artifactId>
<version>0.8-SNAPSHOT</version>
</dependency>
<!-- script support -->
<dependency>
<groupId>de.sekmi.histream</groupId>
<artifactId>histream-js</artifactId>
<version>0.8-SNAPSHOT</version>
</dependency>
<!-- add later for sorting data tables
<dependency>
<groupId>com.google.protobuf</groupId>
......
......@@ -7,16 +7,12 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import javax.xml.bind.JAXB;
import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.ObservationSupplier;
import de.sekmi.histream.etl.config.DataSource;
import de.sekmi.histream.etl.config.EavTable;
import de.sekmi.histream.etl.config.Meta;
import de.sekmi.histream.etl.config.PatientTable;
import de.sekmi.histream.etl.config.VisitTable;
import de.sekmi.histream.etl.config.WideTable;
import de.sekmi.histream.impl.ObservationFactoryImpl;
import de.sekmi.histream.impl.SimplePatientExtension;
......@@ -112,7 +108,7 @@ public class ETLObservationSupplier implements ObservationSupplier{
* @throws ParseException configuration error
*/
public ETLObservationSupplier(DataSource ds, ObservationFactory factory) throws IOException, ParseException {
this(ds,factory,new FactGroupingQueue());
this(ds,factory,ds.createFactQueue(factory));
}
/**
* Construct a new observation supplier directly from a {@link DataSource}.
......@@ -123,7 +119,7 @@ public class ETLObservationSupplier implements ObservationSupplier{
* @throws IOException error reading configuration or table data
* @throws ParseException configuration error
*/
public ETLObservationSupplier(DataSource ds, ObservationFactory factory, FactGroupingQueue queue) throws IOException, ParseException {
protected ETLObservationSupplier(DataSource ds, ObservationFactory factory, FactGroupingQueue queue) throws IOException, ParseException {
this.ds = ds;
this.sync = new ExtensionSync(factory);
this.queue = queue;
......
package de.sekmi.histream.etl;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.net.URL;
import javax.script.ScriptException;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.etl.config.Script;
import de.sekmi.histream.scripting.EncounterScriptEngine;
public class ScriptProcessingQueue extends VisitPostProcessorQueue {
private EncounterScriptEngine engine;
public ScriptProcessingQueue(Script[] scripts, URL baseURL, ObservationFactory factory) throws IOException {
try {
engine = new EncounterScriptEngine();
} catch (ScriptException e) {
throw new IOException("Unable to create script engine", e);
}
engine.setObservationFactory(factory);
// load scripts
for( int i=0; i<scripts.length; i++ ){
try( Reader r = scripts[i].openReader(baseURL) ){
engine.addScript(r);
} catch (ScriptException e) {
throw new IOException("Script error in script "+i, e);
}
}
}
@Override
protected void postProcessVisit() {
if( getVisit() == null ){
return; // don't want null visits
}
try {
engine.processEncounter(getPatient().getId(), getVisit().getId(), getVisit().getStartTime(), getVisitFacts());
} catch (ScriptException e) {
IOException io = new IOException("Error during script execution for patient="+getPatient().getId()+", visit="+getVisit().getId(), e);
throw new UncheckedIOException(io);
}
}
}
......@@ -75,7 +75,8 @@ public abstract class VisitPostProcessorQueue extends FactGroupingQueue {
// next fact
nextFact = super.get();
}
// TODO visit can be processed, without having any facts left to return (either there were no facts in the visit or or postProcessing removed all facts)
// TODO maybe rename visitProcessed to visitProcessedNonempty
if( visitProcessed ){
if( !visitFacts.isEmpty() ){
// remove first
......
package de.sekmi.histream.etl.config;
import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import javax.script.ScriptException;
import javax.xml.bind.JAXB;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
......@@ -12,6 +14,11 @@ import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.etl.FactGroupingQueue;
import de.sekmi.histream.etl.ScriptProcessingQueue;
import de.sekmi.histream.etl.VisitPostProcessorQueue;
/**
* Data source configuration.
* This is the XML root element which can be loaded
......@@ -88,4 +95,27 @@ public class DataSource {
ds.getMeta().setLocation(configuration);
return ds;
}
/**
* If scripts are present, an instance of {@link ScriptProcessingQueue}
* is returned. Otherwise an instance of {@link FactGroupingQueue}.
* @return fact queue
* @throws IOException
* @throws
*/
public FactGroupingQueue createFactQueue(ObservationFactory factory) throws IOException{
// if( true ){
// // TODO debug problems with visitpostprocessorqueue
// return new VisitPostProcessorQueue() {
// @Override
// protected void postProcessVisit() {
// }
// };
// }
if( true || scripts == null || scripts.length == 0 ){
return new FactGroupingQueue();
}else{
return new ScriptProcessingQueue(scripts, meta.getLocation(), factory);
}
}
}
package de.sekmi.histream.etl.config;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlValue;
......@@ -27,4 +34,15 @@ public class Script {
@XmlValue
public String content;
public Reader openReader(URL baseURL) throws IOException{
if( content != null ){
return new StringReader(content);
}else if( src != null ){
URL url = new URL(baseURL, src);
return new InputStreamReader(url.openStream(), this.charset);
}else{
return null;
}
}
}
......@@ -134,9 +134,11 @@
<!-- ... more value elements -->
</virtual>
</eav-table>
<script type="text/javascript"><![CDATA[
<!-- scripts are run for each complete encounter in the order of occurrence -->
<script type="text/javascript"><![CDATA[
if( facts.get("natrium") && facts.get("kalium") ){
facts.add("nakl").value(1);
}
]]>
</script>
<script type="text/javascript" charset="UTF-8" src="test-1-script.js"/>
......
// add a fact which indicates the number of observations
// in this encounter
facts.add("COUNT").value(facts.size());
\ No newline at end of file
package de.sekmi.histream.scripting;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import javax.script.Bindings;
import javax.script.Compilable;
import javax.script.CompiledScript;
import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import de.sekmi.histream.DateTimeAccuracy;
import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
public class EncounterScriptEngine {
private ScriptEngine engine;
private List<CompiledScript> scripts;
private ObservationFactory factory;
public EncounterScriptEngine() throws javax.script.ScriptException {
this.engine = new ScriptEngineManager().getEngineByName("nashorn");
if( engine == null ){
throw new ScriptException("Script engine not available");
}else if( !(engine instanceof Compilable) ){
throw new ScriptException("Script engine does not support compilation");
}
// TODO use custom context for error writer and global bindings
// use strict mode
engine.eval("'use strict';");
scripts = new LinkedList<>();
}
public void addScript(String script) throws ScriptException{
scripts.add(((Compilable)engine).compile(script));
}
public void addScript(URL location, String charset) throws ScriptException, IOException{
try(
InputStream in = location.openStream();
Reader reader = new InputStreamReader(in, charset)
){
addScript(reader);
}
}
public void addScript(Reader reader) throws ScriptException{
scripts.add(((Compilable)engine).compile(reader));
}
public void setObservationFactory(ObservationFactory factory){
this.factory = factory;
}
public void processEncounter(String patientId, String encounterId, DateTimeAccuracy defaultStartTime, List<Observation> facts) throws ScriptException{
Facts f = new Facts(factory, patientId, encounterId, defaultStartTime);
f.setObservations(facts);
Bindings b = engine.createBindings();
b.put("facts", f);
for( CompiledScript script : scripts ){
script.eval(b);
}
// TODO is there a way to add information which script threw an exception?
}
}
package de.sekmi.histream.scripting;
import java.math.BigDecimal;
import de.sekmi.histream.DateTimeAccuracy;
import de.sekmi.histream.Observation;
import de.sekmi.histream.Value;
import de.sekmi.histream.impl.NumericValue;
import de.sekmi.histream.impl.StringValue;
public class Fact {
......@@ -27,11 +30,31 @@ public class Fact {
}
public Value value(String value){
if( value == null ){
observation.setValue(null);
return null;
}
Value v = new StringValue(value);
observation.setValue(v);
return v;
}
/**
* Javascript treats all values as double, hence this method
* will be used by javascript calls using numbers.
* @param value value
* @return value object
*/
public Value value(Double value){
if( value == null ){
observation.setValue(null);
return null;
}
Value v = new NumericValue(new BigDecimal(value));
observation.setValue(v);
return v;
}
public Observation getObservation(){
return observation;
}
......
package de.sekmi.histream.scripting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import javax.script.ScriptException;
import org.junit.Assert;
import org.junit.Test;
import de.sekmi.histream.DateTimeAccuracy;
import de.sekmi.histream.Observation;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.Value;
import de.sekmi.histream.impl.ObservationFactoryImpl;
public class TestEncounterScriptEngine {
@Test
public void multipleCalls() throws Exception{
// prepare facts
ObservationFactory factory = new ObservationFactoryImpl();
Observation[] facts = new Observation[]{
factory.createObservation("P1", "C1", DateTimeAccuracy.parsePartialIso8601("2011")),
factory.createObservation("P1", "C2", DateTimeAccuracy.parsePartialIso8601("2011-02-01")),
factory.createObservation("P1", "C3", DateTimeAccuracy.parsePartialIso8601("2011-02-01"))
};
List<Observation> list = new ArrayList<>();
Collections.addAll(list, facts);
// prepare engine
EncounterScriptEngine e = new EncounterScriptEngine();
e.setObservationFactory(factory);
e.addScript(getClass().getResource("/postprocess-encounter-1.js"), "UTF-8");
// run
e.processEncounter("P1", "E1", DateTimeAccuracy.parsePartialIso8601("2011-02-01"), list);
// verify
Observation last = list.get(list.size()-1);
Assert.assertEquals("test1", last.getConceptId());
Assert.assertEquals(Value.Type.Numeric, last.getValue().getType());
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment