fixed script processing to set ExternalSourceType for generated observations

histream-import/src/main/java/de/sekmi/histream/etl/ScriptProcessingQueue.java

@@ -8,13 +8,14 @@ import java.net.URL;
 import javax.script.ScriptException;
 
 import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.etl.config.Meta;
 import de.sekmi.histream.etl.config.Script;
 import de.sekmi.histream.scripting.EncounterScriptEngine;
 
 public class ScriptProcessingQueue extends VisitPostProcessorQueue {
 	private EncounterScriptEngine engine;
 
-	public ScriptProcessingQueue(Script[] scripts, URL baseURL, ObservationFactory factory) throws IOException {
+	public ScriptProcessingQueue(Script[] scripts, Meta meta, ObservationFactory factory) throws IOException {
 		try {
 			engine = new EncounterScriptEngine();
 		} catch (ScriptException e) {
@@ -24,8 +25,8 @@ public class ScriptProcessingQueue extends VisitPostProcessorQueue {
 
 		// load scripts
 		for( int i=0; i<scripts.length; i++ ){
-			try( Reader r = scripts[i].openReader(baseURL) ){
-				engine.addScript(r);
+			try( Reader r = scripts[i].openReader(meta) ){
+				engine.addScript(r, meta.getSourceId(), scripts[i].getTimestamp(meta));
 			} catch (ScriptException e) {
 				throw new IOException("Script error in script "+i, e);
 			}

histream-import/src/main/java/de/sekmi/histream/etl/config/DataSource.java

 package de.sekmi.histream.etl.config;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URL;
+import java.net.URLConnection;
 import java.util.Arrays;
 import java.util.List;
 
@@ -90,8 +92,14 @@ public class DataSource {
 		}
 	}
 	
-	public static DataSource load(URL configuration){
-		DataSource ds = JAXB.unmarshal(configuration, DataSource.class);
+	public static DataSource load(URL configuration) throws IOException{
+		URLConnection conn = configuration.openConnection();
+		conn.connect();
+		DataSource ds;
+		try( InputStream in = conn.getInputStream() ){
+			ds = JAXB.unmarshal(configuration, DataSource.class);			
+		}
+		ds.getMeta().setLastModified(conn.getLastModified());
 		ds.getMeta().setLocation(configuration);
 		return ds;
 	}
@@ -112,10 +120,10 @@ public class DataSource {
 //				}
 //			};
 //		}
-		if( true || scripts == null || scripts.length == 0 ){
+		if( scripts == null || scripts.length == 0 ){
 			return new FactGroupingQueue();
 		}else{
-			return new ScriptProcessingQueue(scripts, meta.getLocation(), factory);
+			return new ScriptProcessingQueue(scripts, meta, factory);
 		}
 	}
 }

histream-import/src/main/java/de/sekmi/histream/etl/config/Meta.java

@@ -21,6 +21,9 @@ public class Meta {
 	@XmlTransient
 	private URL location;
 	
+	@XmlTransient
+	private Long lastModified;
+	
 	protected Meta(){
 	}
 
@@ -43,4 +46,11 @@ public class Meta {
 		this.location = location;
 	}
 
+	public void setLastModified(long lastModified) {
+		this.lastModified = lastModified;
+	}
+	public Long getLastModified(){
+		return this.lastModified;
+	}
+
 }

histream-import/src/main/java/de/sekmi/histream/etl/config/Script.java

@@ -5,8 +5,8 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
-import java.io.UnsupportedEncodingException;
 import java.net.URL;
+import java.time.Instant;
 
 import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlValue;
@@ -35,14 +35,19 @@ public class Script {
 	@XmlValue
 	public String content;
 	
-	public Reader openReader(URL baseURL) throws IOException{
+	public Reader openReader(Meta meta) throws IOException{
 		if( content != null ){
 			return new StringReader(content);
 		}else if( src != null ){
-			URL url = new URL(baseURL, src);
+			URL url = new URL(meta.getLocation(), src);
 			return new InputStreamReader(url.openStream(), this.charset);
 		}else{
 			return null;
 		}
 	}
+	
+	public Instant getTimestamp(Meta meta) throws IOException{
+		// TODO use last modified of external script files
+		return Instant.ofEpochMilli(meta.getLastModified());
+	}
 }

histream-js/src/main/java/de/sekmi/histream/scripting/EncounterScriptEngine.java

@@ -5,6 +5,8 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.URL;
+import java.net.URLConnection;
+import java.time.Instant;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -18,13 +20,24 @@ import javax.script.ScriptException;
 import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.ext.ExternalSourceType;
+import de.sekmi.histream.impl.ExternalSourceImpl;
 
 
 public class EncounterScriptEngine {
 	private ScriptEngine engine;
-	private List<CompiledScript> scripts;
+	private List<Script> scripts;
 	private ObservationFactory factory;
 	
+	private static class Script{
+		CompiledScript script;
+		ExternalSourceType source;
+		
+		public Script(CompiledScript script, String sourceId, Instant timestamp){
+			this.script = script;
+			source = new ExternalSourceImpl(sourceId, timestamp);
+		}
+	}
 	public EncounterScriptEngine() throws javax.script.ScriptException {
 		this.engine = new ScriptEngineManager().getEngineByName("nashorn");
 		if( engine == null ){
@@ -39,19 +52,21 @@ public class EncounterScriptEngine {
 		scripts = new LinkedList<>();
 	}
 
-	public void addScript(String script) throws ScriptException{
-		scripts.add(((Compilable)engine).compile(script));
+	public void addScript(String script, String sourceId, Instant timestamp) throws ScriptException{
+		scripts.add(new Script(((Compilable)engine).compile(script), sourceId, timestamp));
 	}
-	public void addScript(URL location, String charset) throws ScriptException, IOException{
+	public void addScript(URL location, String charset, String sourceId) throws ScriptException, IOException{
+		URLConnection conn = location.openConnection();
+		Instant timestamp = Instant.ofEpochMilli(conn.getLastModified());
 		try(
-				InputStream in = location.openStream();
+				InputStream in = conn.getInputStream();
 				Reader reader = new InputStreamReader(in, charset)
 		){
-			addScript(reader);
+			addScript(reader, sourceId, timestamp);
 		}
 	}
-	public void addScript(Reader reader) throws ScriptException{
-		scripts.add(((Compilable)engine).compile(reader));	
+	public void addScript(Reader reader, String sourceId, Instant timestamp) throws ScriptException{
+		scripts.add(new Script(((Compilable)engine).compile(reader), sourceId, timestamp));	
 	}
 	
 	public void setObservationFactory(ObservationFactory factory){
@@ -62,8 +77,9 @@ public class EncounterScriptEngine {
 		f.setObservations(facts);
 		Bindings b = engine.createBindings();
 		b.put("facts", f);
-		for( CompiledScript script : scripts ){
-			script.eval(b);
+		for( Script script : scripts ){
+			f.setSource(script.source);
+			script.script.eval(b);
 		}
 		// TODO is there a way to add information which script threw an exception?
 	}

histream-js/src/main/java/de/sekmi/histream/scripting/Facts.java

@@ -6,6 +6,7 @@ import java.util.List;
 import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
+import de.sekmi.histream.ext.ExternalSourceType;
 
 /**
  * Javascript compatible interface for manipulating a list of
@@ -21,6 +22,7 @@ public class Facts {
 	private List<Fact> facts;
 	private List<Observation> sourceList;
 	private ObservationFactory factory;
+	private ExternalSourceType source;
 
 	private String patientId;
 	private String encounterId;
@@ -38,7 +40,9 @@ public class Facts {
 		facts.clear();
 		observations.stream().map(o -> new Fact(o)).forEach(facts::add);
 	}
-	
+	public void setSource(ExternalSourceType source){
+		this.source = source;
+	}
 	public int size(){return facts.size();}
 	
 	public List<Fact> facts(){return facts;}
@@ -46,7 +50,12 @@ public class Facts {
 	
 	public Fact add(String conceptId){
 		Observation o = factory.createObservation(patientId, conceptId, defaultStartTime);
-		o.setEncounterId(encounterId);
+		if( encounterId != null ){
+			o.setEncounterId(encounterId);
+		}
+		if( source != null ){
+			o.setSource(source);
+		}
 		Fact f = new Fact(o);
 		sourceList.add(o);
 		facts.add(f);

histream-js/src/test/java/de/sekmi/histream/scripting/TestEncounterScriptEngine.java

 package de.sekmi.histream.scripting;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -14,6 +15,8 @@ import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.Value;
+import de.sekmi.histream.ext.ExternalSourceType;
+import de.sekmi.histream.impl.ExternalSourceImpl;
 import de.sekmi.histream.impl.ObservationFactoryImpl;
 
 public class TestEncounterScriptEngine {
@@ -33,9 +36,10 @@ public class TestEncounterScriptEngine {
 		// prepare engine
 		EncounterScriptEngine e = new EncounterScriptEngine();
 		e.setObservationFactory(factory);
-		e.addScript(getClass().getResource("/postprocess-encounter-1.js"), "UTF-8");
+		e.addScript(getClass().getResource("/postprocess-encounter-1.js"), "UTF-8", "script1");
 
 		// run
+		ExternalSourceType source = new ExternalSourceImpl("script", Instant.now());
 		e.processEncounter("P1", "E1", DateTimeAccuracy.parsePartialIso8601("2011-02-01"), list);
 
 		// verify