added location information for ParseException

histream-import/src/main/java/de/sekmi/histream/etl/FileRowSupplier.java

@@ -6,6 +6,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
 import java.net.URL;
+import java.net.URLConnection;
 import java.time.Instant;
 import java.util.regex.Pattern;
 
@@ -13,6 +14,8 @@ public class FileRowSupplier extends RowSupplier {
 	private Pattern fieldSeparatorPattern;
 	private BufferedReader in;
 	private String[] headers;
+	private URL url;
+	private int lineNo;
 
 	private Instant timestamp;
 
@@ -22,11 +25,15 @@ public class FileRowSupplier extends RowSupplier {
 
 	public FileRowSupplier(URL location, Pattern pattern) throws IOException{
 		this.fieldSeparatorPattern = pattern;
+		this.url = location;
 		this.in = new BufferedReader(new InputStreamReader(location.openStream()));
 		// TODO: check whether needed to close underlying InputStream
 		
 		// load headers
+		lineNo = 0; 
 		String line = in.readLine();
+		lineNo ++;
+		
 		this.headers = fieldSeparatorPattern.split(line);
 
 		determineFileTimestamp(location);
@@ -48,7 +55,11 @@ public class FileRowSupplier extends RowSupplier {
 			this.timestamp = atts.creationTime().toInstant();
 			*/
 			this.timestamp = Instant.ofEpochMilli(new File(url.getPath()).lastModified());
-			
+		}else if( url.getProtocol().equals("jar") ){
+			// XXX this will return the jar's timestamp
+			// TODO find timestamp of actual file within the jar
+			URLConnection c = url.openConnection();
+			this.timestamp = Instant.ofEpochMilli(c.getLastModified());
 		}else{
 			throw new IOException("Unable to determine timestamp for URL: "+url);
 			// TODO e.g. use URLConnection to get timestamp
@@ -65,6 +76,7 @@ public class FileRowSupplier extends RowSupplier {
 		String line;
 		try {
 			line = in.readLine();
+			lineNo ++;
 		} catch (IOException e) {
 			throw new UncheckedIOException(e);
 		}
@@ -87,4 +99,9 @@ public class FileRowSupplier extends RowSupplier {
 		return timestamp;
 	}
 
+	@Override
+	public String getLocation() {
+		return url.toString()+":"+lineNo;
+	}
+
 }

histream-import/src/main/java/de/sekmi/histream/etl/ParseException.java

@@ -14,7 +14,8 @@ public class ParseException extends Exception {
 	 * 
 	 */
 	private static final long serialVersionUID = 1L;
-
+	private String location;
+	
 	public ParseException(String message){
 		super(message);
 	}
@@ -22,4 +23,27 @@ public class ParseException extends Exception {
 	public ParseException(String message, Throwable cause){
 		super(message, cause);
 	}
+	
+	/**
+	 * Set the location of the parse error.
+	 * E.g. filename and line 
+	 * @param location location string
+	 */
+	public void setLocation(String location){
+		this.location = location;
+	}
+	
+	/**
+	 * Get the parse exception's location
+	 * @return location string
+	 */
+	public String getLocation(){
+		return location;
+	}
+	
+	@Override
+	public String toString(){
+		if( location == null )return super.toString();
+		else return super.toString() + ": "+location;
+	}
 }

histream-import/src/main/java/de/sekmi/histream/etl/RecordSupplier.java

@@ -40,10 +40,15 @@ public class RecordSupplier<R extends FactRow> implements Supplier<R>, AutoClose
 			// no more rows
 			return null;
 		}
+		
 		R p;
 		try {
 			p = table.fillRecord(map, row, factory);
 		} catch (ParseException e) {
+			if( e.getLocation() == null ){
+				// add location information
+				e.setLocation(rows.getLocation());
+			}
 			throw new UncheckedParseException(e);
 		}
 		// fill source information

histream-import/src/main/java/de/sekmi/histream/etl/RowSupplier.java

@@ -17,6 +17,13 @@ public abstract class RowSupplier implements Supplier<Object[]>, AutoCloseable{
 	@Override
 	public abstract void close() throws IOException;
 	
+	/**
+	 * Get a the location of the data source and current cursor position.
+	 * For a text file, this would be file name and line number. 
+	 * A SQL table might return database and table name with row id.
+	 * @return location string
+	 */
+	public abstract String getLocation();
 	/**
 	 * Get the timestamp for the source. Multiple calls to this method should return the
 	 * same timestamp. The source is not allowed to change during reading.

histream-import/src/main/java/de/sekmi/histream/etl/config/VisitTable.java

@@ -6,6 +6,7 @@ import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlElementWrapper;
 import javax.xml.bind.annotation.XmlType;
 
+import de.sekmi.histream.DateTimeAccuracy;
 import de.sekmi.histream.Observation;
 import de.sekmi.histream.ObservationFactory;
 import de.sekmi.histream.etl.ColumnMap;
@@ -47,6 +48,8 @@ public class VisitTable extends Table<VisitRow> implements ConceptTable{
 		map.registerColumn(idat.visitId);
 		if( idat.start != null ){
 			map.registerColumn(idat.start);			
+		}else{
+			throw new ParseException("datasource/visit-table/idat/start column required");
 		}
 		if( idat.end != null ){
 			map.registerColumn(idat.end);
@@ -69,7 +72,13 @@ public class VisitTable extends Table<VisitRow> implements ConceptTable{
 		VisitRow visit = new VisitRow();
 		visit.setId(idat.visitId.valueOf(map, row));
 		visit.setPatientId(idat.patientId.valueOf(map, row));
-		visit.setStartTime(idat.start.valueOf(map, row));
+		DateTimeAccuracy start = idat.start.valueOf(map, row);
+		if( start == null ){
+			// no start time specified for visit row
+			// any other way to retrieve a timestamp??
+			throw new ParseException("No start timestamp found for visit row, but needed for observation");
+		}
+		visit.setStartTime(start);
 		visit.setEndTime(idat.end.valueOf(map, row));
 		if( idat.location != null ){
 			visit.setLocationId(idat.location.valueOf(map, row));

histream-import/src/test/java/de/sekmi/histream/etl/TestETLSupplier.java

@@ -28,6 +28,18 @@ public class TestETLSupplier {
 		if( os != null )os.close();		
 	}
 	
+	@Test
+	public void validateRequiredFactAttributes() throws Exception{
+		Observation fact = os.get();
+		while( fact != null ){
+			Assert.assertNotNull(fact.getPatientId());
+			Assert.assertNotNull(fact.getStartTime());
+			Assert.assertNotNull(fact.getConceptId());
+			
+			fact = os.get();
+		}
+	}
+	
 	@Test
 	public void testXMLConversion() throws Exception{
 		GroupedXMLWriter w = new GroupedXMLWriter(System.out);

histream-import/src/test/resources/test-1-visits.txt

 patid	fallnr	start	end	gewicht
 p1	v1	20.03.2013 09:00	21.03.2013 13:00:21	80
 p1	v2	20.03.2013	21.03.2013	90
-p2	v3			100
-p3	v4			110
\ No newline at end of file
+p2	v3	23.03.2013		100
+p3	v4	23.03.2013		110
\ No newline at end of file