Commit 38257287 authored by Thiemann's avatar Thiemann
Browse files

Merge branch 'master' of gitlab.uni-oldenburg.de:AKTIN/dwh-import

parents 134129c0 ff673d88
......@@ -5,7 +5,7 @@
<groupId>org.aktin</groupId>
<artifactId>cda-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<description>
The CDA import module transforms received
......@@ -18,7 +18,7 @@
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<properties>
......@@ -61,12 +61,12 @@
<dependency>
<groupId>org.aktin</groupId>
<artifactId>cda-server</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.aktin</groupId>
<artifactId>dwh-api</artifactId>
<version>0.3</version>
<version>0.4</version>
</dependency>
......
......@@ -123,15 +123,17 @@ public abstract class AbstractCDAImporter implements CDAProcessor{
throw new CDAException("Transformation to EAV failed", e);
}
}
protected Anonymizer getAnonymizer() {
return cdaToDataWarehouse.getAnonymizer();
}
@Override
public CDAStatus createOrUpdate(Document document, String documentId, String templateId) throws CDAException{
public CDAStatus createOrUpdate(Document document, String documentId, String templateId, String[] patientId, String[] encounterId) throws CDAException{
// not using provided patientId, encounterId, documentId
// use IDs from EAV transformation result
// transform CDA document to EAV XML in temporary file
Path tempEAV = transform(document, templateId);
CDAStatus status;
try{
// parse EAV XML and insert into fact table
......
......@@ -32,6 +32,10 @@ import de.sekmi.histream.ObservationException;
import de.sekmi.histream.ObservationFactory;
import de.sekmi.histream.i2b2.DataDialect;
import de.sekmi.histream.i2b2.I2b2Inserter;
import de.sekmi.histream.i2b2.I2b2Patient;
import de.sekmi.histream.i2b2.I2b2Visit;
import de.sekmi.histream.i2b2.PostgresPatientStore;
import de.sekmi.histream.i2b2.PostgresVisitStore;
/**
* CDA importer pojo EJB. Processed CDA documents are loaded into
......@@ -45,10 +49,12 @@ public class CDAImporter extends AbstractCDAImporter implements AutoCloseable{
private I2b2Inserter inserter;
private ObservationFactory factory;
private ZoneId localZone;
private PostgresVisitStore visitStore;
private PostgresPatientStore patientStore;
/**
* Construct a CDAImporter
* @param factory observation factory
* @param factory observation factory, binding from dwh-db
* @param prefs preferences
* @param anonymizer anonymizer interface
* @throws NamingException i2b2 data sources could not be found by their names
......@@ -59,6 +65,10 @@ public class CDAImporter extends AbstractCDAImporter implements AutoCloseable{
public CDAImporter(ObservationFactory factory, Preferences prefs, Anonymizer anonymizer) throws NamingException, SQLException, IOException {
super(anonymizer);
this.factory = factory;
this.visitStore = (PostgresVisitStore)factory.getExtension(I2b2Visit.class);
// set visit store to reject patient changes, in the case we miss one here (should not happen, TODO remove later)
this.visitStore.setRejectPatientChange(true);
this.patientStore = (PostgresPatientStore)factory.getExtension(I2b2Patient.class);
this.localZone = ZoneId.of(prefs.get(PreferenceKey.timeZoneId));
log.info("Default timezone for CDA documents: "+localZone);
InitialContext ctx = new InitialContext();
......@@ -131,17 +141,72 @@ public class CDAImporter extends AbstractCDAImporter implements AutoCloseable{
return inserter;
}
private enum MergeResult{
NewVisit,
ExistingVisitNewPatient,
ExistingVisitDifferentPatient,
ExistingVisitSamePatient
}
private MergeResult encounterPatientMerge(String documentId, String[] patientId, String[] encounterId) {
// extract encounter id, patient id.
// if encounter exists and has different patient assigned, delete all facts for the encounter id. Then allow reassignment of new patient id
MergeResult result;
String encId = getAnonymizer().calculateEncounterPseudonym(encounterId[0], encounterId[1]);
String patId = getAnonymizer().calculatePatientPseudonym(patientId[0], patientId[1]);
log.info("Using patid="+patId+", encid="+encId+", docid="+documentId);
I2b2Visit visit = visitStore.findVisit(encId);
if( visit == null ) {
log.info("No existing visit found for "+encId);
result = MergeResult.NewVisit;
}else {
log.info("Existing visit found with patid="+visit.getPatientId());
// find new patient
I2b2Patient patient = patientStore.retrieve(patId);
boolean deleteFacts = false;
if( patient == null ) {
// new patient unknown, delete old encounter data
log.warning("Encounter "+encId+" assigned a new (unknown) patient "+patId);
deleteFacts = true;
result = MergeResult.ExistingVisitNewPatient;
}else if( patient.getNum() != visit.getPatientNum() ) {
// new patient differs from assigned patient
log.warning("Encounter "+encId+" assigned different patient "+patId);
// delete all facts for the encounter
result = MergeResult.ExistingVisitDifferentPatient;
deleteFacts = true;
}else {
result = MergeResult.ExistingVisitSamePatient;
deleteFacts = true;
}
if( deleteFacts ) {
try {
inserter.purgeVisit(visit.getNum());
} catch (SQLException e) {
log.log(Level.WARNING,"Failed to purge facts for visit "+visit.getNum(), e);
}
}
// TODO make sure that the new patient is assigned later when it is created
}
return result;
}
@Override
public synchronized CDAStatus createOrUpdate(Document document, String documentId, String templateId)
public synchronized CDAStatus createOrUpdate(Document document, String documentId, String templateId, String[] patientId, String[] encounterId)
throws CDAException {
//log.info("Using patid="+patientId+", encid="+encounterId+", docid="+documentId);
MergeResult merge = encounterPatientMerge(documentId, patientId, encounterId);
if( merge == MergeResult.ExistingVisitNewPatient || merge == MergeResult.ExistingVisitDifferentPatient ) {
log.info("Rejecting change of patient for existing visit");
// for now, reject the update
// TODO make sure that the previous facts are deleted
return CDAStatus.rejected(documentId);
}
final List<ObservationException> insertErrors = new LinkedList<>();
inserter.setErrorHandler(insertErrors::add);
// process document
CDAStatus status = null;
try{
status = super.createOrUpdate(document, documentId, templateId);
status = super.createOrUpdate(document, documentId, templateId, patientId, encounterId);
}finally{
inserter.setErrorHandler(null);
inserter.resetErrorCount();
......
......@@ -102,7 +102,7 @@ public class TransformationFactory {
// declared template does not match template name
// there is an error in the template,
// this should be reported to the developers
log.warning("Mismatch between template name="+templateId+" and declared template="+declaredTemplate);
log.severe("Mismatch between template name="+templateId+" and declared template="+declaredTemplate);
}
return new Transformation(moduleId, templateId, doc, anonymizer);
}
......@@ -110,6 +110,9 @@ public class TransformationFactory {
public void setAnonymizer(Anonymizer anonymizer){
this.anonymizer = anonymizer;
}
public Anonymizer getAnonymizer() {
return anonymizer;
}
public Transformation getTransformation(String templateId) throws IOException, TransformerConfigurationException, TransformerFactoryConfigurationError{
// look in cache
Transformation transform = cache.get(templateId);
......
......@@ -91,7 +91,7 @@
<xsl:template match="/">
<eav-data>
<meta>
<etl-strategy>replace-source</etl-strategy>
<etl-strategy>insert</etl-strategy>
<source>
<xsl:attribute name="timestamp">
<xsl:value-of select="func:ConvertDateTime(/cda:ClinicalDocument/cda:effectiveTime/@value)"/>
......
......@@ -91,7 +91,7 @@
<xsl:template match="/">
<eav-data>
<meta>
<etl-strategy>replace-source</etl-strategy>
<etl-strategy>insert</etl-strategy>
<source>
<xsl:attribute name="timestamp">
<xsl:value-of select="func:ConvertDateTime(/cda:ClinicalDocument/cda:effectiveTime/@value)"/>
......
......@@ -4,14 +4,14 @@
<groupId>org.aktin</groupId>
<artifactId>cda-ontology</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<description>
CDA Ontology definitions
</description>
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<build>
<resources>
......
......@@ -4,7 +4,7 @@
<groupId>org.aktin</groupId>
<artifactId>cda-server</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<description>
The CDA server provides web service interfaces
......@@ -16,7 +16,7 @@
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<properties>
......@@ -77,7 +77,7 @@
<dependency>
<groupId>org.aktin</groupId>
<artifactId>dwh-api</artifactId>
<version>0.3</version>
<version>0.4</version>
</dependency>
<!-- dependency injection -->
......@@ -98,7 +98,7 @@
<dependency>
<groupId>org.aktin</groupId>
<artifactId>cda-validation</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
......@@ -16,7 +16,7 @@ public interface CDAProcessor {
* @throws CDAException processing error
* @throws UnsupportedTemplateException template not supported
*/
public CDAStatus createOrUpdate(Document document, String documentId, String templateId) throws CDAException, UnsupportedTemplateException;
public CDAStatus createOrUpdate(Document document, String documentId, String templateId, String[] patientId, String[] encounterId) throws CDAException, UnsupportedTemplateException;
public Path transform(Document cda, String templateId) throws CDAException;
/**
......
......@@ -5,10 +5,12 @@ import java.util.Date;
public class CDAStatus {
private CDASummary summary;
private Status status;
// XXX maybe add information/warning messages
public enum Status{
Created,
Updated
Updated,
Rejected
}
public CDAStatus(CDASummary summary, Status status){
......@@ -22,6 +24,9 @@ public class CDAStatus {
public static CDAStatus updated(CDASummary summary){
return new CDAStatus(summary, Status.Updated);
}
public static CDAStatus rejected(String documentId) {
return new CDAStatus(new DocumentIdSummary(documentId), Status.Rejected);
}
public Date getLastModified(){
return summary.getLastModified();
......
package org.aktin.cda;
import java.util.Date;
public class DocumentIdSummary implements CDASummary {
private String documentId;
public DocumentIdSummary(String documentId) {
this.documentId = documentId;
}
@Override
public String getDocumentId() {
return documentId;
}
@Override
public Date getLastModified() {
return null;
}
@Override
public Date getCreated() {
return null;
}
@Override
public String getVersion() {
return null;
}
}
......@@ -151,8 +151,10 @@ public class Binary implements ExternalInterface{
if( isValid ){
// check arguments/valid id
// otherwise return HTTP_BAD_REQUEST
String[] patientId = parser.extractPatientId(cda);
String[] encounterId = parser.extractEncounterId(cda);
// process document
CDAStatus stat = processor.createOrUpdate(cda, documentId, templateId);
CDAStatus stat = processor.createOrUpdate(cda, documentId, templateId, patientId, encounterId);
// check whether document was created or updated, return 201 or 200
if( stat.getStatus() == Status.Created ){
// create location conforming to FHIR specification
......@@ -167,6 +169,8 @@ public class Binary implements ExternalInterface{
// Location header not allowed for status 200
hallihallo2.addUpdated();
importSuccessful = true;
}else if( stat.getStatus() == Status.Rejected ) {
response = Response.status(409); // HTTP conflict
}else{
throw new UnsupportedOperationException("Unexpected CDA status "+stat.getStatus());
}
......
......@@ -119,7 +119,10 @@ public class DocumentRepository implements DocumentRepositoryPortType, ExternalI
//ids = parser.extractIDs(cda);
// TODO compare to IDs from XDS call
// process document (XXX catch errors)
processor.createOrUpdate(cda, documentId, templateId);
String[] patientId = parser.extractPatientId(cda);
String[] encounterId = parser.extractEncounterId(cda);
processor.createOrUpdate(cda, documentId, templateId, patientId, encounterId);
resp.setStatus(XDSConstants.RESPONSE_SUCCESS);
} catch (CDAException e) {
log.log(Level.WARNING, "Unable to import CDA", e);
......@@ -128,6 +131,9 @@ public class DocumentRepository implements DocumentRepositoryPortType, ExternalI
Throwable cause = e.getCause();
if( null == cause )cause = e;
return createErrorResponse(XDSConstants.ERR_REPO_ERROR, "Error during import of CDA. See server log", cause);
} catch (XPathExpressionException e) {
log.log(Level.WARNING, "Unable to extract patient or encounter id", e);
return createErrorResponse(XDSConstants.ERR_REPO_ERROR, "Error during import of CDA. See server log", e);
}
}else{
// failed
......
......@@ -4,12 +4,12 @@
<groupId>org.aktin</groupId>
<artifactId>cda-validation</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<properties>
<schematronVersion>20170302T140447</schematronVersion>
......
......@@ -98,6 +98,20 @@ public class CDAParser {
(String)xee.evaluate(cda.getDocumentElement(), XPathConstants.STRING)
};
}
/**
* Find the encounter id for a given CDA document
* @param cda CDA document
* @return two part encounter id: root, extension
* @throws XPathExpressionException XPath error
*/
public String[] extractEncounterId(Document cda) throws XPathExpressionException{
XPathExpression xer = xpath.compile(CDAConstants.XPATH_CDA_ENCOUNTER_ID_ROOT);
XPathExpression xee = xpath.compile(CDAConstants.XPATH_CDA_ENCOUNTER_ID_EXT);
return new String[]{
(String)xer.evaluate(cda.getDocumentElement(), XPathConstants.STRING),
(String)xee.evaluate(cda.getDocumentElement(), XPathConstants.STRING)
};
}
/**
* Find the document id for a given CDA document. It should be globally
......
......@@ -6,19 +6,19 @@
<groupId>org.aktin</groupId>
<artifactId>demo-distribution</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>org.aktin</groupId>
<artifactId>demo-server</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</dependency>
</dependencies>
......
@ECHO OFF
REM AKTIN : DWH Import : Demo Distribution
SET mydir=%~dp0
java -Xmx1024m -Djava.util.logging.config.file="%mydir%\logging.properties" -cp "%mydir%\lib\*" --add-modules java.activation,java.xml.bind,java.xml.ws --patch-module java.xml.ws.annotation=lib/javax.annotation-api-1.2.jar org.aktin.cda.etl.demo.Server %*
......@@ -4,7 +4,7 @@
<groupId>org.aktin</groupId>
<artifactId>demo-server</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
<description>
Demo server application. The demo server can receive
......@@ -15,7 +15,7 @@
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</parent>
<properties>
......@@ -52,12 +52,12 @@
<dependency>
<groupId>org.aktin</groupId>
<artifactId>cda-validation</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.aktin</groupId>
<artifactId>cda-server</artifactId>
<version>0.11-SNAPSHOT</version>
<version>0.13-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
......@@ -61,7 +61,7 @@ public class HashtableStore implements CDAProcessor{
this.map = new Hashtable<>();
}
@Override
public CDAStatus createOrUpdate(Document document, String documentId, String templateId)
public CDAStatus createOrUpdate(Document document, String documentId, String templateId, String[] patientId, String[] encounterId)
throws CDAException {
VirtualDocument doc = map.get(documentId);
if( doc == null ){
......
......@@ -4,12 +4,12 @@
<groupId>org.aktin</groupId>
<artifactId>legacy-cda</artifactId>
<version>0.6-SNAPSHOT</version>
<version>0.11-SNAPSHOT</version>
<parent>
<groupId>org.aktin</groupId>
<artifactId>dwh-import</artifactId>
<version>0.6-SNAPSHOT</version>
<version>0.11-SNAPSHOT</version>
</parent>
<build>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment