Skip to content

GitLab

Commit 665e0a60 authored by R.W.Majeed's avatar R.W.Majeed
Browse files

modifier support for ontology processing

parent 78decaaa
Hide whitespace changes
Inline Side-by-side
Showing with 230 additions and 12 deletions
histream-core/src/main/java/de/sekmi/histream/ontology/Concept.java
View file @ 665e0a60
......@@ -84,10 +84,11 @@ public interface Concept {
* The part relationship is different from the narrower relationship
* in the sense that {@link #getNarrower()} will point to more specific concepts while
* {@link #getParts()} will divide the concept in the sense of a whole/part relationship.
*
* @param inherited whether to return parts which are inherited from broader concepts.
* @return concepts which are part of this concept, or {@code null} if this concept cannot be divided.
* @throws OntologyException for ontology errors
*/
Concept[] getParts();
Concept[] getParts(boolean inherited) throws OntologyException;
/**
* Get restrictions for the values of this concept.
......
histream-i2b2/src/main/java/de/sekmi/histream/i2b2/ont/Import.java
View file @ 665e0a60
......@@ -43,16 +43,20 @@ public class Import implements AutoCloseable{
private Locale locale;
private PreparedStatement insertMeta;
private PreparedStatement insertMetaModifier;
private PreparedStatement insertAccess;
private PreparedStatement insertConcept;
private PreparedStatement insertModifier;
// statistics
private int insertMetaCount;
private int insertAccessCount;
private int insertConceptCount;
private int insertModifierCount;
private int deleteMetaCount;
private int deleteAccessCount;
private int deleteConceptCount;
private int deleteModifierCount;
private String sourceId;
private String sourceIdDelete;
......@@ -63,6 +67,7 @@ public class Import implements AutoCloseable{
private String metaTable;
private String metaAccess;
private String dataConceptTable;
private String dataModifierTable;
private Consumer<String> warningHandler;
......@@ -100,6 +105,7 @@ public class Import implements AutoCloseable{
private String getMetaTable(){return metaTable;}
private String getAccessTable(){return metaAccess;}
private String getConceptTable(){return dataConceptTable;}
private String getModifierTable(){return dataModifierTable;}
public int getInsertMetaCount(){return this.insertMetaCount;}
public int getDeleteMetaCount(){return this.deleteMetaCount;}
......@@ -113,6 +119,7 @@ public class Import implements AutoCloseable{
}
private void prepareStatements() throws SQLException{
insertMeta = dbMeta.prepareStatement("INSERT INTO "+getMetaTable()+"(c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
insertMetaModifier = dbMeta.prepareStatement("INSERT INTO "+getMetaTable()+"(c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'modifier_cd','modifier_dimension','modifier_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
String access_table_name = getMetaTable();
if( access_table_name.indexOf('.') >= 0 ){
// name contains tablespace
......@@ -121,6 +128,8 @@ public class Import implements AutoCloseable{
}
insertAccess = dbMeta.prepareStatement("INSERT INTO "+getAccessTable()+"(c_table_cd,c_table_name,c_protected_access,c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_facttablecolumn,c_dimtablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip)VALUES(?,'"+access_table_name+"','N',?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?)");
insertConcept = dbData.prepareStatement("INSERT INTO "+getConceptTable()+"(concept_path,concept_cd,name_char,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,current_timestamp,?,current_timestamp,?)");
insertModifier = dbData.prepareStatement("INSERT INTO "+getModifierTable()+"(modifier_path, modifier_cd, name_char, update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,current_timestamp,?,current_timestamp,?)");
}
public void setWarningHandler(Consumer<String> warningHandler){
......@@ -142,10 +151,12 @@ public class Import implements AutoCloseable{
this.metaAccess = config.get("meta.access");
this.dataConceptTable = config.get("data.concept.table");
this.dataModifierTable = config.get("data.modifier.table");
Objects.requireNonNull(metaTable, "Need configuration: meta.table");
Objects.requireNonNull(metaAccess, "Need configuration: meta.access");
Objects.requireNonNull(dataConceptTable, "Need configuration: data.concept.table");
Objects.requireNonNull(dataConceptTable, "Need configuration: data.modifier.table");
// ontology configuration
// parse language for locale
if( config.get("ont.language") == null ){
......@@ -165,12 +176,18 @@ public class Import implements AutoCloseable{
}
PreparedStatement deleteOnt = dbMeta.prepareStatement("DELETE FROM "+getMetaTable()+" WHERE sourcesystem_cd=?");
PreparedStatement deleteAccess = dbMeta.prepareStatement("DELETE FROM "+getAccessTable()+" WHERE c_table_cd LIKE ?");
PreparedStatement deleteModifiers = dbMeta.prepareStatement("DELETE FROM "+getAccessTable()+" WHERE c_table_cd LIKE ?");
PreparedStatement deleteConcepts = dbData.prepareStatement("DELETE FROM "+getConceptTable()+" WHERE sourcesystem_cd=?");
deleteConcepts.setString(1, sourceIdDelete);
this.deleteConceptCount = deleteConcepts.executeUpdate();
// System.out.println("Deleted "+deleteConceptCount+" rows from "+getConceptTable());
deleteConcepts.close();
// delete modifiers
deleteModifiers.setString(1, sourceIdDelete);
this.deleteModifierCount = deleteModifiers.executeUpdate();
deleteModifiers.close();
deleteAccess.setString(1, sourceIdDelete+"%");
this.deleteAccessCount = deleteAccess.executeUpdate();
......@@ -218,6 +235,15 @@ public class Import implements AutoCloseable{
insertConcept.executeUpdate();
insertConceptCount ++;
}
private void insertModifierDimension(String path, String name, String concept_cd) throws SQLException{
insertModifier.setString(1, path);
insertModifier.setString(2, concept_cd);
insertModifier.setString(3, name);
insertModifier.setTimestamp(4, sourceTimestamp);
insertModifier.setString(5, sourceId);
insertModifier.executeUpdate();
insertModifierCount ++;
}
private String generateMetadataXML(ValueRestriction vr) throws XMLStreamException, OntologyException{
StringWriter xmlbuf = new StringWriter();
XMLOutputFactory xmlout = XMLOutputFactory.newInstance();
......@@ -348,10 +374,17 @@ public class Import implements AutoCloseable{
}while( !done );
return b.toString();
}
private String buildConceptPath(String parentPath, Concept concept){
String path_part = concept.getID(); // TODO unique key sufficient, e.g. try label.hashCode()
return parentPath + path_part + "\\";
}
private String buildModifierPath(String parentPath, Concept modifier){
return buildConceptPath(parentPath, modifier);
}
private void insertMeta(int level, String path_prefix, Concept concept, boolean accessRoot) throws SQLException, OntologyException{
insertMeta.setInt(1, level);
String label = concept.getPrefLabel(locale);
String path_part = concept.getID(); // TODO unique key sufficient, e.g. try label.hashCode()
if( label == null ){
// no label for language, try to get neutral label
......@@ -364,7 +397,7 @@ public class Import implements AutoCloseable{
}
// use hashcode
String path = path_prefix + path_part+"\\";
String path = buildConceptPath(path_prefix, concept);
insertMeta.setString(2, path);
insertMeta.setString(3, label);
// c_synonym_cd
......@@ -403,9 +436,14 @@ public class Import implements AutoCloseable{
}
// c_basecode
// TODO support multiple ids (e.g. adding virtual leaves)
if( conceptIds.length != 0 ){
if( conceptIds.length == 1 ){
// single concept code
insertConceptDimension(path, label, conceptIds[0]);
}else if( conceptIds.length > 1 ){
// multiple concept codes.
// TODO add virtual leaves to tree
// insert into concept_dimension
// XXX this will violate the primary key (concept_path) if multiple notations are used
insertConceptDimension(path, label, conceptIds[0]);
// concept has id and can occur in fact table
// TODO make sure, each concept_path is inserted only once
......@@ -453,6 +491,9 @@ public class Import implements AutoCloseable{
insertMeta.executeUpdate();
insertMetaCount ++;
// insert modifiers
insertModifiers(concept, path);
if( accessRoot ){
insertAccess.setString(1, sourceId+"_"+Integer.toHexString(concept.hashCode()));
insertAccess.setInt(2, level);
......@@ -471,6 +512,60 @@ public class Import implements AutoCloseable{
insertMeta(level+1, path, sub, false);
}
}
private void insertModifiers(Concept concept, String concept_path) throws OntologyException, SQLException{
// TODO write modifier dimension
Concept[] parts = concept.getParts(false);
if( parts == null ){
return; // nothing to do
}
for( Concept part : parts ){
insertModifierMetaTree(concept_path, 1, part, "\\");
}
}
private void insertModifierMetaTree(String concept_path, int level, Concept modifier, String path_prefix) throws OntologyException, SQLException{
// fill modifier dimension first
String[] ids = modifier.getNotations();
String path = buildModifierPath(path_prefix, modifier);
String label = modifier.getPrefLabel(locale);
for( int i=0; i<ids.length; i++ ){
// XXX will fail if multiple notations are present for a single path
insertModifierDimension(path, label, ids[i]);
}
// "INSERT INTO i2b2 (c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
insertMetaModifier.setInt(1, level); // c_hlevel
insertMetaModifier.setString(2, path); // c_fullname
// c_name
insertMetaModifier.setString(3, label);
// c_synonym_cd
insertMetaModifier.setString(4, "N");
// c_visualattributes
insertMetaModifier.setString(5, "RA");
// c_basecode
insertMetaModifier.setString(6, ids[0]);
// c_metadataxml
insertMetaModifier.setString(7, null);
// c_dimcode
insertMetaModifier.setString(8, label);
// c_tooltip
insertMetaModifier.setString(9, modifier.getDescription(locale));
// m_appliedpath
insertMetaModifier.setString(10, concept_path+"%");
// update, download, import dates
insertMetaModifier.setTimestamp(11, sourceTimestamp);
// sourcesystem_cd
insertMetaModifier.setString(12, sourceId);
insertMetaModifier.executeUpdate();
insertMetaCount ++;
// recurse into nested modifier structure
for( Concept nested : modifier.getNarrower() ){
insertModifierMetaTree(concept_path, level+1, nested, path);
}
}
/*
public void loadOntology(Class<?> ontologyClass, Map<String,String> config) throws Exception{
Plugin plugin = Plugin.newInstance(ontologyClass, config);
......
histream-i2b2/src/test/java/de/sekmi/histream/i2b2/ont/TestOntologyImport.java
View file @ 665e0a60
......@@ -26,7 +26,8 @@ public class TestOntologyImport {
props.put("meta.sourcesystem_cd", "test");
props.put("meta.table", "i2b2metadata.i2b2");
props.put("meta.access", "i2b2metadata.table_access");
props.put("data.concept.table", "i2b2crcdata.concepts");
props.put("data.concept.table", "i2b2crcdata.concept_dimension");
props.put("data.concept.table", "i2b2crcdata.modifier_dimension");
props.put("ont.language", "en");
imp = new Import(dbcMeta, dbcData, props);
}
......
histream-maven-plugin/src/test/resources/ontology.properties
View file @ 665e0a60
......@@ -7,7 +7,7 @@ meta.basepath=\\i2b2\\
meta.sourcesystem_cd=test
data.concept.table=i2b2demodata.concept_dimension
data.modifier.table=i2b2demodata.modifier_dimension
# language to use for accessing ontology
ont.language=DE
histream-maven-plugin/src/test/resources/skos-aktin-cda.ttl
View file @ 665e0a60
......@@ -497,11 +497,18 @@
dwh:restriction dwh:Integer ;
skos:notation "LOINC:9268-4" .
:RightPupil a skos:Concept ;
dwh:isPartOf :Pupilsize ;
skos:prefLabel "Right Pupil"@en ;
skos:prefLabel "Rechte Pupille"@de ;
skos:notation "right".
:Pupilsize a skos:Concept ;
skos:inScheme :EmergencyNote ;
skos:prefLabel "Size of pupil"@en ;
skos:prefLabel "Pupillenweite"@de ;
dwh:hasPart :RightPupil ;
dc:description "Pupillenweite: Auswahlliste" ;
skos:narrower :dilated ;
......
histream-maven-plugin/src/test/resources/test-ontology.ttl 0 → 100644
View file @ 665e0a60
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix dc: <http://purl.org/dc/elements/1.1/> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@prefix dwh: <http://sekmi.de/histream/dwh#> .
@prefix : <http://sekmi.de/histream/skos/tests#> .
: dc:title "HIStream concept ontology for testing".
:TestScheme a skos:ConceptScheme ;
skos:hasTopConcept :TestData .
:TestData a skos:Concept ;
skos:prefLabel "TestData_label_en"@en ;
skos:prefLabel "TestData_label_de"@de ;
skos:narrower :TestStringConcept ;
skos:narrower :TestIntegerConcept ;
skos:narrower :TestDecimalConcept ;
skos:narrower :TestEnumConcept ;
dc:description "Top concept for test data" .
:TestEnumConcept a skos:Concept;
skos:inScheme :TestData;
dwh:restriction [ a owl:Restriction ;
owl:onProperty rdf:value;
owl:oneOf (
[rdf:value "1"^^xsd:string; skos:prefLabel "1_de"@de; skos:prefLabel "1_en"@en ]
[rdf:value "2"^^xsd:string; skos:prefLabel "2_de"@de; skos:prefLabel "2_en"@en ]
)
] ;
skos:prefLabel "Enum_label_de"@de;
skos:prefLabel "Enum_label_en"@en;
skos:notation "T:Enum" .
####### skos:notation "Enum"^^:TestCS .
dwh:Integer a owl:Restriction ;
owl:onProperty rdf:value ;
owl:allValuesFrom xsd:integer .
dwh:String a owl:Restriction ;
owl:onProperty rdf:value ;
owl:allValuesFrom xsd:string .
:TestIntegerConcept a skos:Concept;
skos:inScheme :TestData ;
dwh:restriction dwh:Integer ;
skos:prefLabel "Integer_label_en"@en ;
skos:prefLabel "Integer_label_de"@de ;
dc:description "Description_de"@de ;
dc:description "Description_neutral" ;
skos:notation "T:type:int" .
:TestStringConcept a skos:Concept;
skos:inScheme :TestData ;
dwh:restriction dwh:String ;
skos:prefLabel "String_label_en"@en ;
skos:notation "T:type:str" .
:TestDecimalConcept a skos:Concept;
skos:inScheme :TestData ;
dwh:restriction [ a owl:Restriction ;
owl:onProperty rdf:value ;
owl:allValuesFrom[ a rdfs:Datatype;
owl:onDatatype xsd:decimal;
owl:withRestrictions(
[ xsd:minInclusive 200 ]
[ xsd:maxInclusive 400 ]
)
]
] ;
skos:prefLabel "Decimal_label_en"@en ;
skos:notation "T:type:dec" .
# Test inferred topConcept and narrower
:OtherScheme a skos:ConceptScheme .
:OtherTop a skos:Concept ;
skos:inScheme :OtherScheme ;
skos:topConceptOf :OtherScheme ;
skos:prefLabel "OtherTop_label_en"@en .
:OtherSub a skos:Concept ;
skos:inScheme :OtherScheme ;
skos:inScheme :TestScheme ;
skos:broader :OtherTop ;
skos:prefLabel "OtherSub_label_en"@en ;
skos:notation "other" .
# Second dimension / modifiers
:Mod1 a skos:Concept ;
dwh:isPartOf :TestData ;
skos:prefLabel "Mod1 en"@en ;
skos:notation "m1" .
\ No newline at end of file
histream-skos/examples/test-ontology.ttl
View file @ 665e0a60
......@@ -88,3 +88,9 @@ dwh:String a owl:Restriction ;
skos:broader :OtherTop ;
skos:prefLabel "OtherSub_label_en"@en ;
skos:notation "other" .
# Second dimension / modifiers
:Mod1 a skos:Concept ;
dwh:isPartOf :TestData ;
skos:prefLabel "Mod1 en"@en ;
skos:notation "m1" .
\ No newline at end of file
histream-skos/src/main/java/de/sekmi/histream/ontology/skos/ConceptImpl.java
View file @ 665e0a60
......@@ -89,9 +89,8 @@ public class ConceptImpl implements Concept {
}
@Override
public Concept[] getParts() {
// TODO implement
return null;
public Concept[] getParts(boolean inherited) throws OntologyException {
return store.getParts(this, inherited);
}
@Override
......
histream-skos/src/main/java/de/sekmi/histream/ontology/skos/HIStreamOntology.java
View file @ 665e0a60
......@@ -16,6 +16,8 @@ public class HIStreamOntology {
public static final URI DWH_OTHERWISE;
public static final URI DWH_XPATH;
public static final URI DWH_ECMASCRIPT;
public static final URI DWH_HAS_PART;
public static final URI DWH_IS_PART_OF;
static {
final ValueFactory f = ValueFactoryImpl.getInstance();
......@@ -28,6 +30,8 @@ public class HIStreamOntology {
DWH_OTHERWISE = f.createURI(DWH_NAMESPACE, "otherwise");
DWH_XPATH = f.createURI(DWH_NAMESPACE, "XPath");
DWH_ECMASCRIPT = f.createURI(DWH_NAMESPACE, "ECMAScript");
DWH_HAS_PART = f.createURI(DWH_NAMESPACE, "hasPart"); // TODO use correct names/URIs
DWH_IS_PART_OF = f.createURI(DWH_NAMESPACE, "isPartOf"); // XXX see above
}
}
histream-skos/src/main/java/de/sekmi/histream/ontology/skos/Store.java
View file @ 665e0a60
......@@ -6,6 +6,7 @@ import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
......@@ -48,6 +49,8 @@ public class Store implements Ontology, Plugin {
private Resource inferredContext;
// SKOS scheme to enforce unique notations
private Resource scheme;
private Map<String, String> namespaceShortener;
/// XXX namespace resolver
/**
* Plugin constructor which accepts configuration parameters.
......@@ -139,6 +142,7 @@ public class Store implements Ontology, Plugin {
return count;
}
private void initializeRepo(Iterable<File> files, Iterable<URL> urls, String baseURI) throws RepositoryException, IOException{
namespaceShortener = new HashMap<>();
repo = new SailRepository(new MemoryStore());
repo.initialize();
rc = repo.getConnection();
......@@ -171,6 +175,7 @@ public class Store implements Ontology, Plugin {
inferred += inferInverseRelations(SKOS.TOP_CONCEPT_OF, SKOS.HAS_TOP_CONCEPT);
inferred += inferInverseRelations(SKOS.BROADER, SKOS.NARROWER);
inferred += inferInverseRelations(SKOS.NARROWER, SKOS.BROADER);
// TODO a isPartOf b -> b hasPart a
if( inferred != 0 ){
log.fine("Inferred "+inferred+" statements.");
}
......@@ -230,6 +235,9 @@ public class Store implements Ontology, Plugin {
Concept[] getNarrower(ConceptImpl concept)throws OntologyException{
return getRelatedConcepts(concept.getResource(), SKOS.NARROWER);
}
Concept[] getParts(ConceptImpl concept, boolean inherited)throws OntologyException{
return getRelatedConcepts(concept.getResource(), HIStreamOntology.DWH_HAS_PART);
}
public void printTopConcepts() throws OntologyException{
for( Concept c : getTopConcepts() ){
......
histream-skos/src/test/java/de/sekmi/histream/ontology/skos/OntologyTest.java
View file @ 665e0a60
......@@ -13,6 +13,7 @@ import de.sekmi.histream.ontology.Concept;
import de.sekmi.histream.ontology.OntologyException;
import de.sekmi.histream.ontology.ValueRestriction;
import de.sekmi.histream.ontology.skos.Store;
import de.sekmi.histream.xml.NamespaceResolver;
public class OntologyTest {
Store store;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment