removed experimental duplicate fact filter.

required sort index comparator moved to external repo for development, because it was not working.

histream-import/src/main/java/de/sekmi/histream/etl/filter/DuplicateFactFilter.java

@@ -18,21 +18,29 @@ public class DuplicateFactFilter extends PostProcessingFilter{
 	@XmlElement
 	public String[] concept;
 
-	private static class FactComparator implements Comparator<Fact>{
+	static class FactComparator implements Comparator<Fact>{
 		@Override
 		public int compare(Fact o1, Fact o2) {
+			return DuplicateFactFilter.compare(o1, o2);
+		}
+	}
+	public static int compare(Fact o1, Fact o2) {
 		int cmp = o1.getObservation().getStartTime().compareTo(
 					o2.getObservation().getStartTime()	);
+		if( true )return cmp;
 		if( cmp == 0 ){
 			// if times are equal, sort by concept
 			cmp = o1.getConcept().compareTo(o2.getConcept());
 		}
 		return cmp;
 	}
-	}
 	private void removeAllDuplicates(AbstractFacts facts){
 		// order by start and concept
-		facts.sort( new FactComparator() );
+		if( true ) {
+			throw new UnsupportedOperationException("Not yet implemented");
+		}
+		//facts.sort( new FactComparator() );
+		
 
 		ArrayList<Integer> duplicates = new ArrayList<>();
 		

histream-import/src/test/java/de/sekmi/histream/etl/TestETLSupplier.java

@@ -103,7 +103,7 @@ public class TestETLSupplier {
 		Patient p = fact.getExtension(Patient.class);
 		Assert.assertNotNull(p);
 		Assert.assertEquals("p1", p.getId());
-		ZoneId zone = ZoneOffset.UTC.normalized();
+		ZoneId zone = ZoneId.systemDefault();
 		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-01",zone), p.getBirthDate());
 		Assert.assertEquals(DateTimeAccuracy.parsePartialIso8601("2003-02-11",zone), p.getDeathDate());
 

histream-import/src/test/java/de/sekmi/histream/etl/config/TestMarshall.java

@@ -69,10 +69,11 @@ public class TestMarshall {
 			
 			// check post processing
 			Assert.assertNotNull(ds.postProcessing);
-			Assert.assertEquals(3, ds.postProcessing.filter.length);
-			Assert.assertEquals(DuplicateFactFilter.class, ds.postProcessing.filter[0].getClass());
-			DuplicateFactFilter f = (DuplicateFactFilter)ds.postProcessing.filter[0];
-			Assert.assertEquals(1, f.concept.length);
+			Assert.assertEquals(2, ds.postProcessing.filter.length);
+// duplicate fact filter removed for now
+//			Assert.assertEquals(DuplicateFactFilter.class, ds.postProcessing.filter[0].getClass());
+//			DuplicateFactFilter f = (DuplicateFactFilter)ds.postProcessing.filter[0];
+//			Assert.assertEquals(1, f.concept.length);
 			ScriptFilter sf = (ScriptFilter)ds.postProcessing.filter[1];
 			Assert.assertNotNull(sf);
 			

histream-import/src/test/java/de/sekmi/histream/etl/filter/TestDuplicateFactFilter.java

@@ -22,7 +22,8 @@ import static org.junit.Assert.*;
 
 public class TestDuplicateFactFilter {
 
-	@Test
+	// XXX implement filter first
+	//@Test
 	public void verifyComparator() throws ScriptException, ParseException{
 		ObservationFactory of = new ObservationFactoryImpl(new SimplePatientExtension(), new SimpleVisitExtension());
 		DuplicateFactFilter filter = new DuplicateFactFilter();
@@ -31,10 +32,18 @@ public class TestDuplicateFactFilter {
 		AbstractFacts facts = e.wrapEncounterFacts("P1", "E1", DateTimeAccuracy.parsePartialIso8601("2001-02-03", ZoneId.systemDefault()), new ArrayList<>());
 		Fact a = facts.add("lala");
 		Fact b = facts.add("xx");
-		assertTrue( DuplicateFactFilter.compare(a, b) < 0 );
-		facts.add("lala");
+//		assertTrue( DuplicateFactFilter.compare(a, b) < 0 );
+		b.start("2000-01-02T03:04:05Z");
+//		assertTrue( DuplicateFactFilter.compare(a, b) > 0 );
+		Fact c = facts.add("xx").start("2000-01-02T03:05Z");
+		Fact d = facts.add("lala"); // add duplicate
+//		assertTrue( DuplicateFactFilter.compare(c, b) > 0 );
+//		assertTrue( DuplicateFactFilter.compare(a, d) == 0 );
+//		assertTrue( DuplicateFactFilter.compare(a, c) > 0 );
+//		assertTrue( DuplicateFactFilter.compare(d, c) > 0 );
+		
 		filter.processVisit(facts);
-		assertEquals(2, facts.size());
+		assertEquals(3, facts.size());
 		// TODO compare with timestamps
 	}
 }

histream-import/src/test/java/de/sekmi/histream/etl/validation/TestValidator.java

@@ -69,7 +69,8 @@ public class TestValidator {
 		}
 		Assert.fail("Exception expected");
 	}
-	@Test
+	// XXX implement duplicate fact filter first
+	//@Test
 	public void validateData4WithDuplicateFilter() throws Exception{
 		// duplicate concepts
 		try( ObservationSupplier os = ETLObservationSupplier.load(getClass().getResource("/data/test-4-datasource2.xml")) ){

histream-import/src/test/resources/data/test-1-datasource.xml

@@ -139,9 +139,6 @@
 	</eav-table>
 	<!-- scripts are run for each complete encounter in the order of occurrence -->
 	<post-processing>
-		<filter xsi:type="duplicate-fact">
-			<concept>kalium</concept>
-		</filter>
 		<filter xsi:type="javascript">
 			<script><![CDATA[
 			if( facts.get("natrium") && facts.get("kalium") ){

histream-import/src/test/resources/data/test-4-datasource2.xml

@@ -76,5 +76,11 @@
 	</eav-table>
 	<post-processing>
 		<filter xsi:type="duplicate-fact"/>
+		<!--  also possible to specify concepts to filter
+			<filter xsi:type="duplicate-fact">
+				<concept>kalium</concept>
+			</filter>
+		 -->
+
 	</post-processing>
 </datasource>

histream-js/src/main/java/de/sekmi/histream/scripting/AbstractFacts.java

 package de.sekmi.histream.scripting;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 
 import de.sekmi.histream.Observation;
@@ -88,18 +86,9 @@ public abstract class AbstractFacts {
 		return f;
 	}
 
+	/*
+	 * TODO use linked sort to sort both arrays simultaneously by the order given by the comparator
 	public void sort(Comparator<Fact> comparator){
-		Integer[] indices = new Integer[facts.size()];
-		for( int i=0; i<indices.length; i++ ){
-			indices[i] = i;
-		}
-		// determine sort order
-		Arrays.sort(indices, new Comparator<Integer>() {
-			@Override
-			public int compare(Integer o1, Integer o2) {
-				return comparator.compare(facts.get(o1), facts.get(o2));
-			}
-		});
 		// reorder both arrays
 		for( int i=0; i<indices.length; i++ ){
 			while( i != indices[i] ){
@@ -121,4 +110,19 @@ public abstract class AbstractFacts {
 			}
 		}
 	}
+*/
+	
+	@Override
+	public String toString() {
+		StringBuilder builder = new StringBuilder(facts.size() * 128);
+		builder.append("[\n");
+		for( int i=0; i<facts.size(); i++ ){
+			if( i != 0 ) {
+				builder.append(",\n");
+			}
+			builder.append(facts.get(i).toString());
+		}
+		builder.append("\n]");
+		return builder.toString();
+	}
 }

linked-sort/src/main/java/SortIndexComparator.java

+package de.sekmi.histream.scripting;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+public class SortIndexComparator<T> {
+
+	private Comparator<T> comparator;
+
+	public SortIndexComparator(Comparator<T> comparator){
+		this.comparator = comparator;
+	}
+
+	/**
+	 * Determine the sort index for a list without changing the sort order
+	 * @param list list
+	 * @return sort order
+	 */
+	public Integer[] sortIndexes(List<T> list) {
+		Integer[] indices = new Integer[list.size()];
+		// fill array with sequence
+		for( int i=0; i<indices.length; i++ ){
+			indices[i] = i;
+		}
+		// determine sort order
+		Arrays.sort(indices, new Comparator<Integer>() {
+			@Override
+			public int compare(Integer o1, Integer o2) {
+				return comparator.compare(list.get(o1), list.get(o2));
+			}
+		});
+		return indices;
+		
+		// convert to basic array
+//		int[] order = new int[indices.length];
+//		for( int i=0; i<indices.length; i++ ) {
+//			order[i] = indices[i];
+//		}
+//		return order;
+	}
+
+	/**
+	 * Sort two lists. The primary list determines the sort order and the second
+	 * list is sorted in the same order as the primary.
+	 * @param primary primary list to sort
+	 * @param linked linked list to be sorted in the same order as the primary list
+	 */
+	// this implementation does not work
+//	public <U> void linkedSort(List<T> primary, List<U> linked){
+//		Integer[] indices = sortIndexes(primary);
+//		for( int i=0; i<indices.length; i++ ){
+//			while( i != indices[i] ){
+//
+//				// store old target values which will be overridden
+//				int oldI = indices[indices[i]];
+//				T oldF = primary.get(indices[i]);
+//				U oldO = linked.get(indices[i]);
+//
+//				// replace target values
+//				primary.set(indices[i], primary.get(i));
+//				linked.set(indices[i], linked.get(i));
+//				indices[indices[i]] = indices[i];
+//
+//				// move old targets to old values
+//				indices[i] = oldI;
+//				primary.set(i, oldF);
+//				linked.set(i, oldO);
+//			}
+//		}
+//	}
+
+//	public <U> void linkedSortNew(List<T> primary, List<U> linked){
+//		// calculate sort indices
+//		Integer[] indices = sortIndexes(primary);
+//		
+//		// first pass: transform index to destination
+//		/* write 0 at index position index[0],
+//		 * store old value and write 
+//		 */
+//		int j=0;
+//		int p=0;
+//		int t;
+//		t = indices[0];
+//		do {
+//			t = indices[j];
+//			indices[j] = p;
+//			j = t;
+//		}
+//		
+//		// TODO clone lists and use random access 
+//		for( int i=0; i<indices.length; i++ ){
+//			while( i != indices[i] ){
+//
+//				// store old target values which will be overridden
+//				int oldI = indices[indices[i]];
+//				T oldF = primary.get(i);
+//				U oldO = linked.get(i);
+//
+//				primary.set(i, primary.get(indices[i]));
+//				linked.set(i, linked.get(indices[i]));
+//				
+//				primary.set(indices[i], oldF);
+//				linked.set(indices[i], oldO);
+//
+//
+//				indices[i]
+//				
+//				// replace target values
+//				primary.set(indices[i], primary.get(i));
+//				linked.set(indices[i], linked.get(i));
+//				indices[indices[i]] = indices[i];
+//
+//				// move old targets to old values
+//				indices[i] = oldI;
+//				primary.set(i, oldF);
+//				linked.set(i, oldO);
+//			}
+//		}
+//	}
+
+}

linked-sort/src/test/java/TestSortIndexComparator.java

+package de.sekmi.histream.scripting;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.Test;
+
+
+public class TestSortIndexComparator {
+
+	private class IntComparator implements Comparator<Integer>{
+		@Override
+		public int compare(Integer o1, Integer o2) {
+			return o1.compareTo(o2);
+		}
+	}
+	private class CharComparator implements Comparator<Character>{
+		@Override
+		public int compare(Character o1, Character o2) {
+			return o1.compareTo(o2);
+		}
+	}
+	@Test
+	public void verifyComparator() {		
+		Character[] testData = new Character[] { 'd', 'b', 'g', 'e', 'h', 'j', 'i', 'c', 'f' , 'k', 'a'};
+		
+		SortIndexComparator<Character> c = new SortIndexComparator<>(new CharComparator());
+		Integer[] order = c.sortIndexes(Arrays.asList(testData));
+		
+		System.out.println("Original: "+Arrays.toString(testData));
+		
+		StringBuilder b = new StringBuilder();
+		int curr,prev = -1;
+		for( int i=0; i<testData.length; i++ ) {
+			if( i != 0 ) {
+				b.append(", ");
+			}
+			b.append(testData[order[i]]);
+			
+			curr = testData[order[i]];
+			assertTrue(prev <= curr);
+			prev = curr;
+		}
+		
+		System.out.println("with sortIndexes: "+b.toString());
+		System.out.println("testData: "+Arrays.toString(testData));
+		
+		Object[] other = new Object[testData.length];
+		for( int i=0; i<testData.length; i++ ) {
+			other[i] = new Object();
+		}
+		List<Character> a = Arrays.asList(testData);
+		List<Object> d = Arrays.asList(other);
+		c.linkedSort(a, d);
+		// output
+		b = new StringBuilder();
+		for( int i=0; i<a.size(); i++ ) {
+			if( i != 0 ) {
+				b.append(", ");
+			}
+			b.append(a.get(i));			
+		}
+		System.out.println("linkedSort.primary: "+b.toString());
+		
+	}
+	private static class Ref{
+		Integer ref;
+		public Ref(Integer ref) {
+			this.ref = ref;
+		}
+	}
+
+	/**
+	 * TODO implement and verify linked sort method
+	 */
+//	@Test
+	public void verifyLinkedSort() {
+		Random rand = new Random(42);
+		// generate a random integer array
+		Integer[] testData = new Integer[1000];
+		// and two copies of the array
+		Ref[] copy = new Ref[testData.length];
+		// fill with random numbers
+		// copies contain different objects but same numbers
+		for( int i=0; i<testData.length; i++ ) {
+			testData[i] = rand.nextInt();
+			// fill copies
+			copy[i] = new Ref(testData[i]);
+		}
+
+		// perform linked sort		
+		SortIndexComparator<Integer> sic = new SortIndexComparator<>(new IntComparator());
+		List<Integer> a = new ArrayList<>(Arrays.asList(testData));
+		List<Ref> b = new ArrayList<>(Arrays.asList(copy));
+		
+		sic.linkedSort(a, b);
+		
+		Integer prev = a.get(0);
+		for( int i=0; i<a.size(); i++ ){
+			Integer curr = a.get(i);
+			// make sure primary was sorted in ascending order
+			assertTrue("Sort order violated at index "+i+": "+prev+", "+curr, prev <= curr );
+			// make sure the copy was sorted linked to primary
+			assertTrue( curr == b.get(i).ref );
+			// next
+			prev = curr;
+		}
+		
+	}
+
+}