Skip to content
GitLab
Commit b2b13333 authored by R.W.Majeed's avatar R.W.Majeed
Browse files

Import module for any table based format

parent 70b98829
Hide whitespace changes
Inline Side-by-side
histream-import/src/test/java/de/sekmi/histream/etl/config/TestMarshall.java 0 → 100644
View file @ b2b13333
package de.sekmi.histream.etl.config;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import javax.xml.bind.JAXB;
import org.junit.Assert;
import org.junit.Test;
import de.sekmi.histream.etl.config.DataSource;
public class TestMarshall {
@Test
public void testUnmarshall() throws IOException{
try( InputStream in = getClass().getResourceAsStream("/test-1-datasource.txt") ){
DataSource ds = JAXB.unmarshal(in, DataSource.class);
Assert.assertNotNull(ds.meta);
Assert.assertEquals("replace-source",ds.meta.etlStrategy);
Assert.assertEquals("test-1",ds.meta.sourceId);
// patient table
Assert.assertNotNull(ds.patientTable);
Assert.assertNotNull(ds.patientTable.source);
Assert.assertNotNull(ds.patientTable.idat);
Assert.assertEquals("patid",ds.patientTable.idat.patientId.name);
Assert.assertEquals("geburtsdatum",ds.patientTable.idat.birthdate.name);
Assert.assertEquals("geschlecht",ds.patientTable.idat.gender.name);
// visit table
Assert.assertNotNull(ds.visitTable);
Assert.assertNotNull(ds.visitTable.source);
Assert.assertNotNull(ds.visitTable.idat);
Assert.assertEquals("patid",ds.visitTable.idat.patientId.name);
Assert.assertEquals("fallnr",ds.visitTable.idat.visitId.name);
// wide table
Assert.assertNotNull(ds.wideTables);
Assert.assertEquals(1, ds.wideTables.length);
WideTable t = ds.wideTables[0];
Assert.assertNotNull(t);
Assert.assertNotNull(t.idat);
Assert.assertEquals("patid",t.idat.patientId.name);
Assert.assertEquals("fallnr",t.idat.visitId.name);
// concepts
Assert.assertNotNull(t.concepts);
Assert.assertTrue(t.concepts.length > 0);
Concept c = t.concepts[0];
Assert.assertNotNull(c);
Assert.assertEquals("natrium", c.id);
Assert.assertEquals("na", c.value.name);
Assert.assertEquals("mmol/l", c.unit.constantValue);
}
}
@Test
public void testMarshal() throws MalformedURLException{
DataSource s = new DataSource();
s.meta = new Meta();
s.meta.sourceId = "SID";
s.meta.etlStrategy = "replace-source";
s.xmlSources = new XmlSource[1];
s.xmlSources[0] = new XmlSource();
s.xmlSources[0].url = new URL("http://lala");
s.xmlSources[0].transform = new XmlSource.Transform[1];
s.xmlSources[0].transform[0] = new XmlSource.Transform("file:my.xsl","c:/to/file");
s.patientTable = new PatientTable();
s.patientTable.source = new FileSource("file:patient.source","text/csv");
s.patientTable.idat = new PatientTable.IDAT();
s.patientTable.idat.patientId = new Column("patid");
s.visitTable = new VisitTable();
s.visitTable.source = new FileSource("file:lala.txt", "text/plain");
s.visitTable.idat = new VisitTable.IDAT();
s.visitTable.idat.patientId = new Column("patid");
s.visitTable.idat.visitId = new Column("visit");
s.visitTable.concepts = new Concept[1];
s.visitTable.concepts[0] = new Concept("vconcept","start");
s.wideTables = new WideTable[1];
s.wideTables[0] = new WideTable();
s.wideTables[0].source = new SQLSource("org.postgresql.Driver","jdbc:postgresql://localhost:15432/i2b2");
s.wideTables[0].idat = new DataTableIdat();
s.wideTables[0].idat.patientId = new Column("patid");
s.wideTables[0].concepts = new Concept[2];
s.wideTables[0].concepts[0] = new Concept("ACC","zeit");
s.wideTables[0].concepts[0].modifiers = new Concept.Modifier[1];
s.wideTables[0].concepts[0].modifiers[0] = new Concept.Modifier("DOSE");
s.wideTables[0].concepts[0].modifiers[0].value = new Column("dosis");
JAXB.marshal(s, System.out);
}
}
histream-import/src/test/resources/datasource-example.xml 0 → 100644
View file @ b2b13333
<!DOCTYPE configuration >
<datasource version="1.0">
<meta>
<etl-strategy></etl-strategy>
<source-id></source-id>
</meta>
<!-- erstmal weglassen -->
<transformations>
<xml-source url="xxxx.xml">
<transform with="a.xslt" to="lala.txt" />
<transform with="b.xslt" to="dddd.txt" />
</xml-source>
</transformations>
<!--
<value-mappings>
<map id="geschlecht">
<value from="M" to="M" />
<value from="W" to="F" />
</map>
</value-mappings>
-->
<patient-table>
<source>
<sql jdbc-driver="com.mysql..." db-url="jdbc:mysql://localhost/EMP" db-user="asdf" db-password="mkmlkd">
SELECT * FROM patienten p
</sql>
</source>
<idat>
<patient-id>ID</patient-id>
<firstname>Vorname</firstname>
<surname>Nachname</surname>
<birthdate></birthdate>
<deathdate></deathdate>
<gender map="geschlecht">geschlecht</gender>
<ignore>...</ignore>
</idat>
<mdat>
<concept id="gewicht">
<value>gewicht</value>
<start>eingabedatum</start>
<unit constant="kg"/>
</concept>
<concept id="lufu">
<value>lufu_fev1</value>
<start>visit_date</start>
</concept>
</mdat>
</patient-table>
<!-- optional -->
<visit-table>
<idat>
<patient-id>patid</patient-id>
<visit-id>event</visit-id>
<start format="ISO" na="@" truncate-to="year">asdf</start>
<end>asdff</end>
<ignore>...</ignore>
</idat>
<mdat>
<!-- in/out code -->
<concept id="lufu">
<value>lufu_fev1</value>
<start>visit_date</start>
</concept>
</mdat>
</visit-table>
<widetable>
<source>
<file url="file://c:/lala.txt" type="text/csv" key-column="PatientID" />
</source>
<idat>
<patient-id>asdfg</patient-id>
<visit-id>a33edd</visit-id>
</idat>
<mdat>
<concept id="blutdruck">
<value field="value" />
<start field="beginn" />
<end field="ende" />
<unit field="einheit" />
<modifier id="method">
<value field="methode" />
<unit fixed="mm" />
</modifier>
<modifier id="other">
<value field="other" />
</modifier>
</concept>
</mdat>
..
</widetable>
<eavtable>
<source>
<file url="file://c:/lala.txt" type="text/csv;delimiter=\t;encoding=UTF-8;" key-column="PatientID" />
<!-- file ODER sql -->
<sql>
SELECT konzept, beginn, ende, einheit FROM blutdruckmessungen WHERE patid=?
</sql>
</source>
<idat>
<patient-id>asdf</patient-id>
<visit-id>asdf</visit-id>
<ignore>...</ignore>
</idat>
<columns>
<concept>parameter</concept>
<value>wert</value>
<start>beginn</start>
<end></end>
<unit>end</unit>
</columns>
</eavtable>
</datasource>
histream-import/src/test/resources/test-1-datasource.txt 0 → 100644
View file @ b2b13333
<!DOCTYPE configuration >
<datasource version="1.0">
<meta>
<etl-strategy>replace-source</etl-strategy>
<source-id>test-1</source-id>
</meta>
<!-- erstmal weglassen -->
<transformations>
<xml-source url="xxxx.xml">
<transform with="a.xslt" to="lala.txt" />
<transform with="b.xslt" to="dddd.txt" />
</xml-source>
</transformations>
<!--
<value-mappings>
<map id="geschlecht">
<value from="M" to="M" />
<value from="W" to="F" />
</map>
</value-mappings>
-->
<patient-table>
<source xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="fileSource">
<url>file:test-1-patients.txt</url>
<type>text/csv</type>
</source>
<idat>
<patient-id>patid</patient-id>
<firstname>vorname</firstname>
<surname>nachname</surname>
<birthdate>geburtsdatum</birthdate>
<deathdate>verstorben</deathdate>
<gender>geschlecht</gender>
<ignore>ignoriert1</ignore>
<ignore>patfakt1</ignore>
</idat>
</patient-table>
<!-- optional -->
<visit-table>
<source xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="fileSource">
<url>file:test-1-visits.txt</url>
<type>text/csv</type>
</source>
<idat>
<patient-id>patid</patient-id>
<visit-id>fallnr</visit-id>
<start format="ISO" na="@" truncate-to="year">start</start>
<end>end</end>
</idat>
<mdat>
<!-- in/out code -->
<concept id="weight">
<value>gewicht</value>
<start>start</start>
</concept>
</mdat>
</visit-table>
<wide-table>
<source xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="fileSource">
<url>file:test-1-widetable.txt</url>
<type>text/csv</type>
</source>
<idat>
<patient-id>patid</patient-id>
<visit-id>fallnr</visit-id>
</idat>
<mdat>
<concept id="natrium">
<value>na</value>
<start>zeitpunkt</start>
<unit constant-value="mmol/l" />
<modifier id="other">
<value field="other" />
</modifier>
</concept>
</mdat>
..
</wide-table>
</datasource>
histream-import/src/test/resources/test-1-patients.txt 0 → 100644
View file @ b2b13333
patid inogriert1 name vorname geburtsdatum verstorben geschlecht patfakt1
1 a n1 v1 01.02.2003 11.02.2003 F c
2 b n2 v2 02.03.2004 M d
3 c n3 v3 03.04.2005 F e
\ No newline at end of file
histream-import/src/test/resources/test-1-visits.txt 0 → 100644
View file @ b2b13333
patid fallnr start end gewicht
1 1 20.03.2013 09:00 21.03.2013 13:00 80
1 2 20.03.2013 21.03.2013 90
2 3 100
3 4 110
\ No newline at end of file
histream-import/src/test/resources/test-1-widetable.txt 0 → 100644
View file @ b2b13333
patid fallnr zeitpunkt na k co2 cl glucose ca bun creatinine
1 1 01.01.2010 124 5.8 25 101 107 10.1 17 0.9
1 2 02.01.2010 123 5.7 28 101 106 9.8 15 1.3
2 3 03.01.2010 124 5.8 25 101 107 10.1 17 0.9
3 4 04.01.2010 124 5.8 25 101 107 10.1 17 0.9
pom.xml
View file @ b2b13333
......@@ -31,6 +31,7 @@
<module>HIStream-i2b2</module>
<!--<module>HIStream-HL7</module>-->
<module>HIStream-SKOS</module>
<module>histream-import</module>
<module>distribution</module>
</modules>
<build>
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment