Commit 583aaf8d authored by Volker Thiemann's avatar Volker Thiemann
Browse files

Merge branch 'master' of gitlab.uni-oldenburg.de:AKTIN/dwh-import

parents eeb53ca5 ce91fac1
CDAs werden transformiert mit cda-eav/*.xsl zu eav-data XML.
Es ist wichtig, dass bei mehreren CDA-Modulen (die beim gleichen Patient und Fall dokumentiert werden) NICHT die gleichen concept_codes erzeugt werden. Dies wrde unique constraints
der Tabelle observation_fact verletzen.
List of AKTIN module IDs:
- *base* Basismodul
- *monitor* berwachung
......
......@@ -47,7 +47,6 @@
</archive>
</configuration>
</plugin>
-->
<plugin>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-maven-plugin</artifactId>
......@@ -60,6 +59,7 @@
</dependency>
</dependencies>
</plugin>
-->
<!--
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
......@@ -89,6 +89,7 @@
</plugins>
</build>
<dependencyManagement>
<!--
<dependencies>
<dependency>
<groupId>org.jboss.arquillian</groupId>
......@@ -98,6 +99,7 @@
<type>pom</type>
</dependency>
</dependencies>
-->
</dependencyManagement>
<dependencies>
<dependency>
......@@ -112,10 +114,10 @@
<dependency>
<groupId>org.aktin</groupId>
<artifactId>dwh-api</artifactId>
<version>0.2</version>
<version>0.3-SNAPSHOT</version>
</dependency>
<!--
<dependency>
<groupId>org.jboss.arquillian.junit</groupId>
<artifactId>arquillian-junit-container</artifactId>
......@@ -135,7 +137,7 @@
<version>2.3.2.Final</version>
<scope>test</scope>
</dependency>
-->
<dependency>
......
......@@ -5,6 +5,7 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.ZoneId;
import java.util.Date;
import java.util.function.Consumer;
import javax.xml.bind.JAXBException;
......@@ -20,6 +21,7 @@ import org.aktin.cda.CDAStatus.Status;
import org.aktin.cda.UnsupportedTemplateException;
import org.aktin.cda.etl.transform.Transformation;
import org.aktin.cda.etl.transform.TransformationFactory;
import org.aktin.dwh.Anonymizer;
import org.w3c.dom.Document;
import de.sekmi.histream.Observation;
......@@ -37,15 +39,14 @@ public abstract class AbstractCDAImporter implements CDAProcessor{
private TransformationFactory cdaToDataWarehouse;
private XMLInputFactory inputFactory;
public AbstractCDAImporter() throws IOException {
public AbstractCDAImporter(Anonymizer anonymizer) throws IOException {
// create transformer
cdaToDataWarehouse = new TransformationFactory();
cdaToDataWarehouse.setAnonymizer(anonymizer);
// XML input factory
inputFactory = XMLInputFactory.newInstance();
}
/**
* Get the observation factory which will be used to create observations
* @return observation factory
......@@ -91,9 +92,17 @@ public abstract class AbstractCDAImporter implements CDAProcessor{
// insert facts
suppl.stream().forEach(getObservationInserter());
CDAStatus.Status status = deleted?Status.Updated:Status.Created;
CDAStatus.Status status;
Descriptor desc = new Descriptor(sourceId);
// TODO use/write timestamps and version
desc.lastModified = new Date();
if( deleted ){
status = Status.Updated;
}else{
status = Status.Created;
desc.created = desc.lastModified;
}
// TODO use/write version
return new CDAStatus(desc, status);
} catch (IOException e) {
throw new CDAException("Unable to read EAV temp file: "+file, e);
......
......@@ -23,6 +23,7 @@ import org.aktin.cda.CDAException;
import org.aktin.cda.CDAStatus;
import org.aktin.cda.CDASummary;
import org.aktin.cda.DocumentNotFoundException;
import org.aktin.dwh.Anonymizer;
import org.aktin.dwh.PreferenceKey;
import org.w3c.dom.Document;
......@@ -54,8 +55,8 @@ public class CDAImporter extends AbstractCDAImporter implements AutoCloseable{
* @throws IOException unable to load CDA to ETL transformation script
*/
@Inject // TODO change to ObservationFactory and see if this works
public CDAImporter(ObservationFactory factory, Preferences prefs) throws NamingException, SQLException, IOException {
super();
public CDAImporter(ObservationFactory factory, Preferences prefs, Anonymizer anonymizer) throws NamingException, SQLException, IOException {
super(anonymizer);
this.factory = factory;
this.localZone = ZoneId.of(prefs.get(PreferenceKey.timeZoneId));
log.info("Default timezone for CDA documents: "+localZone);
......@@ -73,10 +74,6 @@ public class CDAImporter extends AbstractCDAImporter implements AutoCloseable{
*/
// data dialect
DataDialect dd = new DataDialect();
String tz = prefs.get("i2b2.db.tz"); // TODO use PreferenceKey enum
if( tz != null ){
dd.setTimeZone(ZoneId.of(tz));
}
try{
inserter = new I2b2Inserter();
inserter.open(crcDS.getConnection(), dd);
......
......@@ -6,6 +6,8 @@ import org.aktin.cda.CDASummary;
public class Descriptor implements CDASummary {
private String docId;
Date lastModified;
Date created;
public Descriptor(String docId){
this.docId = docId;
......@@ -17,14 +19,12 @@ public class Descriptor implements CDASummary {
@Override
public Date getLastModified() {
// TODO Auto-generated method stub
return null;
return lastModified;
}
@Override
public Date getCreated() {
// TODO Auto-generated method stub
return null;
return created;
}
@Override
......
......@@ -16,6 +16,7 @@ import javax.xml.transform.stream.StreamResult;
import org.aktin.cda.etl.transform.fun.CalculateEncounterHash;
import org.aktin.cda.etl.transform.fun.CalculatePatientHash;
import org.aktin.cda.etl.transform.fun.CalculateSourceId;
import org.aktin.dwh.Anonymizer;
import org.w3c.dom.Document;
import net.sf.saxon.Configuration;
......@@ -32,6 +33,7 @@ public class Transformation {
private TransformerFactoryImpl transformerFactory;
private Templates transformerTemplates;
private Anonymizer anonymizer;
/**
* Construct a CDA template to EAV transformation
......@@ -42,10 +44,10 @@ public class Transformation {
* @throws TransformerFactoryConfigurationError if the transformer factory failed to initialize
* @throws TransformerConfigurationException transformer setup error
*/
public Transformation(String moduleId, String templateId, Document xslt)throws TransformerFactoryConfigurationError, TransformerConfigurationException{
public Transformation(String moduleId, String templateId, Document xslt, Anonymizer anonymizer)throws TransformerFactoryConfigurationError, TransformerConfigurationException{
this.moduleId = moduleId;
this.templateId = templateId;
this.anonymizer = anonymizer;
// create transformer
// ususally a transformer is created via TransformerFactory.newInstance(),
// but this may return a non-saxon parser
......@@ -70,9 +72,9 @@ public class Transformation {
// }
// Configuration config = ((TransformerFactoryImpl)factory).getConfiguration();
Configuration config = transformerFactory.getConfiguration();
config.registerExtensionFunction(new CalculatePatientHash());
config.registerExtensionFunction(new CalculateEncounterHash());
config.registerExtensionFunction(new CalculateSourceId());
config.registerExtensionFunction(new CalculatePatientHash(anonymizer));
config.registerExtensionFunction(new CalculateEncounterHash(anonymizer));
config.registerExtensionFunction(new CalculateSourceId(anonymizer));
// TODO don't need moduleId and factory?
}
......
......@@ -22,6 +22,7 @@ import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import org.aktin.cda.NamespaceContextImpl;
import org.aktin.dwh.Anonymizer;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
......@@ -38,6 +39,7 @@ public class TransformationFactory {
private Map<String, Transformation> cache;
private XPath xpath;
private DocumentBuilderFactory builderFactory;
private Anonymizer anonymizer;
public TransformationFactory(){
// inputFactory = XMLInputFactory.newInstance();
......@@ -77,6 +79,7 @@ public class TransformationFactory {
}
private Transformation loadTransformation(String templateId) throws IOException, TransformerConfigurationException, TransformerFactoryConfigurationError{
Objects.requireNonNull(this.anonymizer, "no anonymizer configured");
// need to locate the transformation
URL url = locateTransformationByTemplate(templateId);
if( url == null ){
......@@ -101,9 +104,12 @@ public class TransformationFactory {
// this should be reported to the developers
log.warning("Mismatch between template name="+templateId+" and declared template="+declaredTemplate);
}
return new Transformation(moduleId, templateId, doc);
return new Transformation(moduleId, templateId, doc, anonymizer);
}
public void setAnonymizer(Anonymizer anonymizer){
this.anonymizer = anonymizer;
}
public Transformation getTransformation(String templateId) throws IOException, TransformerConfigurationException, TransformerFactoryConfigurationError{
// look in cache
Transformation transform = cache.get(templateId);
......
package org.aktin.cda.etl.transform.fun;
import org.aktin.dwh.Anonymizer;
import net.sf.saxon.om.StructuredQName;
import net.sf.saxon.value.SequenceType;
public class CalculateEncounterHash extends OneWayHashFunction{
public CalculateEncounterHash(Anonymizer anonymizer) {
super(anonymizer);
}
public static final StructuredQName QNAME = OneWayHashFunction.buildFunctionQName("encounter-hash");
protected static final SequenceType[] TWO_STRINGS = new SequenceType[]{SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING};
......
package org.aktin.cda.etl.transform.fun;
import org.aktin.dwh.Anonymizer;
import net.sf.saxon.om.StructuredQName;
import net.sf.saxon.value.SequenceType;
public class CalculatePatientHash extends OneWayHashFunction{
public CalculatePatientHash(Anonymizer anonymizer) {
super(anonymizer);
}
public static final StructuredQName QNAME = OneWayHashFunction.buildFunctionQName("patient-hash");
protected static final SequenceType[] TWO_STRINGS = new SequenceType[]{SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING};
......
package org.aktin.cda.etl.transform.fun;
import org.aktin.dwh.Anonymizer;
import net.sf.saxon.om.StructuredQName;
import net.sf.saxon.value.SequenceType;
......@@ -28,6 +30,10 @@ import net.sf.saxon.value.SequenceType;
*
*/
public class CalculateSourceId extends OneWayHashFunction{
public CalculateSourceId(Anonymizer anonymizer) {
super(anonymizer);
}
public static final StructuredQName QNAME = OneWayHashFunction.buildFunctionQName("import-hash");
protected static final SequenceType[] FIVE_STRINGS = new SequenceType[]{SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING,SequenceType.SINGLE_STRING};
......
package org.aktin.cda.etl.transform.fun;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.security.DigestException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.logging.Logger;
import org.aktin.dwh.Anonymizer;
import net.sf.saxon.expr.XPathContext;
import net.sf.saxon.lib.ExtensionFunctionCall;
......@@ -24,9 +18,13 @@ import net.sf.saxon.value.StringValue;
*
*/
public abstract class OneWayHashFunction extends ExtensionFunctionDefinition {
private static final Logger log = Logger.getLogger(OneWayHashFunction.class.getName());
// private static final Logger log = Logger.getLogger(OneWayHashFunction.class.getName());
public static final String AKTIN_CDA_FUNCTIONS_NS = "http://aktin.org/cda/functions";
private Anonymizer anonymizer;
public OneWayHashFunction(Anonymizer anonymizer){
this.anonymizer = anonymizer;
}
protected static final StructuredQName buildFunctionQName(String funcName){
return new StructuredQName("", AKTIN_CDA_FUNCTIONS_NS, funcName);
}
......@@ -36,40 +34,6 @@ public abstract class OneWayHashFunction extends ExtensionFunctionDefinition {
return SequenceType.SINGLE_STRING;
}
/**
* Calculate a one way hash function for the given input.
* The algorithm is as follows:
* <ol>
* <li>Concatenate the arguments with a slash (/) as separator.</li>
* <li>Encode the input arguments with UTF-8 encoding
* <li>Generate a 160bit SHA-1 checksum</li>
* <li>Produce bas64 encoding with url-safe alphabet</li>
* </ol>
* The resulting string length will be less than 30 characters.
*
* @param strings input
* @return string hash
* @throws DigestException error calculating message digest
*/
public String calculateHash(String ...strings) throws DigestException{
MessageDigest digest;
try {
digest = MessageDigest.getInstance("SHA-1");
} catch (NoSuchAlgorithmException e) {
// should not happen. SHA-1 is guaranteed to be included in the JRE
throw new DigestException(e);
}
// join arguments
String composite = String.join("/", strings);
// logging
// encode to bytes
ByteBuffer input = Charset.forName("UTF-8").encode(composite);
// calculate digest and encode with base64
digest.update(input);
String result = Base64.getUrlEncoder().encodeToString(digest.digest());
log.info("Hash "+getFunctionQName().getDisplayName()+": "+composite+" -> "+result);
return result;
}
/**
* Implements a call to the hash function with variable arguments.
......@@ -89,11 +53,7 @@ public abstract class OneWayHashFunction extends ExtensionFunctionDefinition {
if( arguments.length == 0 ){
throw new XPathException("Need at least one argument for hash calculation");
}
try {
return new StringValue(calculateHash(strings));
} catch (DigestException e) {
throw new XPathException("Unable to calculate hash", e);
}
return new StringValue(anonymizer.calculateAbstractPseudonym(strings));
}
}
......
......@@ -141,6 +141,7 @@
<!-- Alle Fact-Templates auf Body/Component/Section Ebene aufrufen -->
<xsl:apply-templates select="/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section"/>
<xsl:apply-templates select="/cda:ClinicalDocument/cda:componentOf/cda:encompassingEncounter/cda:id[2]"/>
<xsl:apply-templates select="/cda:ClinicalDocument/cda:templateId"/>
</encounter>
</patient>
</eav-data>
......@@ -1000,6 +1001,10 @@
<xsl:template match="/cda:ClinicalDocument/cda:templateId">
<xsl:comment>Import Transformation/Version Information</xsl:comment>
<fact>
<!-- ACHTUNG: in anderen Modulen (z.B. Traumamodul) darf das nachfolgende fact nicht
genauso ausgegeben werden, da dann die Möglichkeit besteht dass das unique constraint
der observation_fact tabelle verletzt wird. Da die Software-Version aber gleich ist,
kann dies komplett weggelassen werden. -->
<xsl:attribute name="concept">
<xsl:value-of select="$ProjectVersion-Prefix"/><xsl:value-of select="$aktin.release.version"/>
</xsl:attribute>
......
......@@ -141,6 +141,7 @@
<xsl:apply-templates select="/cda:ClinicalDocument/cda:participant/cda:associatedEntity"/>
<!-- Alle Fact-Templates auf Body/Component/Section Ebene aufrufen -->
<xsl:apply-templates select="/cda:ClinicalDocument/cda:component/cda:structuredBody/cda:component/cda:section"/>
<xsl:apply-templates select="/cda:ClinicalDocument/cda:templateId"/>
</encounter>
</patient>
</eav-data>
......@@ -975,6 +976,10 @@
<xsl:template match="/cda:ClinicalDocument/cda:templateId">
<xsl:comment>Import Transformation/Version Information</xsl:comment>
<fact>
<!-- ACHTUNG: in anderen Modulen (z.B. Traumamodul) darf das nachfolgende fact nicht
genauso ausgegeben werden, da dann die Möglichkeit besteht dass das unique constraint
der observation_fact tabelle verletzt wird. Da die Software-Version aber gleich ist,
kann dies komplett weggelassen werden. -->
<xsl:attribute name="concept">
<xsl:value-of select="$ProjectVersion-Prefix"/><xsl:value-of select="$aktin.release.version"/>
</xsl:attribute>
......
......@@ -24,7 +24,7 @@ public class CDAImporterMockUp extends AbstractCDAImporter implements Consumer<O
private int insertCount;
public CDAImporterMockUp() throws IOException{
super();
super(new ConcatAnonymizer());
System.out.println("CONSTRUCT CDAImporterMockUp");
factory = new ObservationFactoryImpl(new SimplePatientExtension(), new SimpleVisitExtension());
insertCount = 0;
......
package org.aktin.cda.etl;
import org.aktin.dwh.Anonymizer;
/**
* Anonymizer for testing. Concatenates the source parts to
* produce a pseudonym string which should not be used for
* production.
* @author R.W.Majeed
*
*/
public class ConcatAnonymizer implements Anonymizer {
@Override
public String calculateAbstractPseudonym(String... parts) {
return String.join("/", parts);
}
}
package org.aktin.cda.etl;
import javax.inject.Inject;
import org.aktin.cda.CDAProcessor;
import org.aktin.cda.etl.xds.DocumentRepository;
import org.jboss.arquillian.container.test.api.Deployment;
import org.jboss.arquillian.junit.Arquillian;
import org.jboss.shrinkwrap.api.ShrinkWrap;
import org.jboss.shrinkwrap.api.asset.EmptyAsset;
import org.jboss.shrinkwrap.api.spec.WebArchive;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(Arquillian.class)
public class TestEEInject {
//private static final String WEBINF_DIR = "src/main/webapp/WEB-INF";
@Deployment
public static WebArchive createDeployment() {
return ShrinkWrap.create(WebArchive.class)
//.addPackage("org.aktin.cda")
.addPackage(CDAProcessor.class.getPackage())
// .addPackage(RestService.class.getPackage())
.addPackage(DocumentRepository.class.getPackage())
.addClass(CDAImporterMockUp.class)
.addAsManifestResource(EmptyAsset.INSTANCE, "META-INF/beans.xml")
//.setWebXML(new File(WEBINF_DIR,"web.xml"))
//.addAsManifestResource(EmptyAsset.INSTANCE, "beans.xml")
//.addAsWebInfResource(new File(WEBINF_DIR,"sun-jaxws.xml"))
;
}
@Inject
private DocumentRepository xdsService;
@Test
public void verifyInjectedBeans(){
// Assert.assertNotNull(restService);
Assert.assertNotNull(xdsService);
// TODO verify that CDAProcessor instanceof CDAImporter
}
@Test
public void assumeCorrectValidation(){
// TODO submit document to services
}
}
......@@ -10,6 +10,7 @@ public class TestTransformationLoader {
@Test
public void verifyTransformVariables() throws Exception{
TransformationFactory f = new TransformationFactory();
f.setAnonymizer(new ConcatAnonymizer());
//XMLStreamReader xsr = inputFactory.createXMLStreamReader(getClass().getResourceAsStream("/cda-eav.xsl"));
// InputSource input = new InputSource(getClass().getResourceAsStream("/cda-eav.xsl"));
Transformation t = f.getTransformation("1.2.276.0.76.10.1019");
......
......@@ -3,7 +3,6 @@ package org.aktin.cda.etl.fhir;
import java.io.BufferedWriter;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
......@@ -136,6 +135,7 @@ public class Binary implements ExternalInterface{
String templateId = null;
boolean isValid = false;
boolean importSuccessful = false;
URI location = null;
Document cda=null;
try {
cda = parser.buildDOM(doc);
......@@ -153,20 +153,18 @@ public class Binary implements ExternalInterface{
// otherwise return HTTP_BAD_REQUEST
// process document
CDAStatus stat = processor.createOrUpdate(cda, documentId, templateId);
// check whether document was created or updated, return 201 or 200
if( stat.getStatus() == Status.Created ){
try {
URI location = new URI("Binary/?_id="+stat.getDocumentId());
// create location conforming to FHIR specification
location = URI.create("Binary/"+stat.getDocumentId()+"/_history/0");
response = Response.created(location);
} catch (URISyntaxException e) {
log.log(Level.WARNING, "Unable to build URI for created resource", e);
response = Response.ok();
}
hallihallo2.addCreated();
importSuccessful = true;
}else if( stat.getStatus() == Status.Updated ){
response = Response.ok();
location = URI.create("Binary/"+stat.getDocumentId());
// Location header not allowed for status 200
hallihallo2.addUpdated();
importSuccessful = true;
}else{
......
......@@ -65,6 +65,7 @@ public class SimplifiedOperationOutcome {
public static class Issue{
Severity severity;
String details;
String diagnostics;
IssueType code;
public Issue(Severity severity, IssueType type, String details){
this.severity = severity;
......@@ -86,6 +87,11 @@ public class SimplifiedOperationOutcome {
break;
}
}
public static Issue diagnosticInfo(String message){
Issue i = new Issue(Severity.information, IssueType.informational, null);
i.diagnostics = "all ok";
return i;
}
}
private List<Issue> issues;
......@@ -110,6 +116,24 @@ public class SimplifiedOperationOutcome {
issues.add(new Issue(severity, details));
}
private void writeIssue(XMLStreamWriter writer, Issue issue) throws XMLStreamException{
writer.writeStartElement("issue");
// severity
writer.writeStartElement("severity");
writer.writeAttribute("value", issue.severity.name());
writer.writeEndElement();
// code
writer.writeStartElement("code");
writer.writeAttribute("value", issue.code.value);
writer.writeEndElement();
if( issue.details != null ){
writer.writeStartElement("details");
writer.writeAttribute("value", issue.details);
writer.writeEndElement();
}
writer.writeEndElement();