modifier support for ontology processing

histream-core/src/main/java/de/sekmi/histream/ontology/Concept.java

@@ -84,10 +84,11 @@ public interface Concept {
 	 * The part relationship is different from the narrower relationship
 	 * in the sense that {@link #getNarrower()} will point to more specific concepts while
 	 * {@link #getParts()} will divide the concept in the sense of a whole/part relationship.
-	 * 
+	 * @param inherited whether to return parts which are inherited from broader concepts.
 	 * @return concepts which are part of this concept, or {@code null} if this concept cannot be divided.
+	 * @throws OntologyException for ontology errors
 	 */
-	Concept[] getParts();
+	Concept[] getParts(boolean inherited) throws OntologyException;
 	
 	/**
 	 * Get restrictions for the values of this concept.

histream-i2b2/src/main/java/de/sekmi/histream/i2b2/ont/Import.java

@@ -43,16 +43,20 @@ public class Import implements AutoCloseable{
 	private Locale locale;
 	
 	private PreparedStatement insertMeta;
+	private PreparedStatement insertMetaModifier;
 	private PreparedStatement insertAccess;
 	private PreparedStatement insertConcept;
+	private PreparedStatement insertModifier;
 
 	// statistics
 	private int insertMetaCount;
 	private int insertAccessCount;
 	private int insertConceptCount;
+	private int insertModifierCount;
 	private int deleteMetaCount;
 	private int deleteAccessCount;
 	private int deleteConceptCount;
+	private int deleteModifierCount;
 
 	private String sourceId;
 	private String sourceIdDelete;
@@ -63,6 +67,7 @@ public class Import implements AutoCloseable{
 	private String metaTable;
 	private String metaAccess;
 	private String dataConceptTable;
+	private String dataModifierTable;
 
 	private Consumer<String> warningHandler;
 
@@ -100,6 +105,7 @@ public class Import implements AutoCloseable{
 	private String getMetaTable(){return metaTable;}
 	private String getAccessTable(){return metaAccess;}
 	private String getConceptTable(){return dataConceptTable;}
+	private String getModifierTable(){return dataModifierTable;}
 
 	public int getInsertMetaCount(){return this.insertMetaCount;}
 	public int getDeleteMetaCount(){return this.deleteMetaCount;}
@@ -113,6 +119,7 @@ public class Import implements AutoCloseable{
 	}
 	private void prepareStatements() throws SQLException{
 		insertMeta = dbMeta.prepareStatement("INSERT INTO "+getMetaTable()+"(c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
+		insertMetaModifier = dbMeta.prepareStatement("INSERT INTO "+getMetaTable()+"(c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'modifier_cd','modifier_dimension','modifier_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
 		String access_table_name = getMetaTable();
 		if( access_table_name.indexOf('.') >= 0 ){
 			// name contains tablespace
@@ -121,6 +128,8 @@ public class Import implements AutoCloseable{
 		}
 		insertAccess = dbMeta.prepareStatement("INSERT INTO "+getAccessTable()+"(c_table_cd,c_table_name,c_protected_access,c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_facttablecolumn,c_dimtablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip)VALUES(?,'"+access_table_name+"','N',?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?)");
 		insertConcept = dbData.prepareStatement("INSERT INTO "+getConceptTable()+"(concept_path,concept_cd,name_char,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,current_timestamp,?,current_timestamp,?)");
+		insertModifier = dbData.prepareStatement("INSERT INTO "+getModifierTable()+"(modifier_path, modifier_cd, name_char, update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,current_timestamp,?,current_timestamp,?)");
+		
 	}
 
 	public void setWarningHandler(Consumer<String> warningHandler){
@@ -142,10 +151,12 @@ public class Import implements AutoCloseable{
 		this.metaAccess = config.get("meta.access");
 		
 		this.dataConceptTable = config.get("data.concept.table");
+		this.dataModifierTable = config.get("data.modifier.table");
 
 		Objects.requireNonNull(metaTable, "Need configuration: meta.table");
 		Objects.requireNonNull(metaAccess, "Need configuration: meta.access");
 		Objects.requireNonNull(dataConceptTable, "Need configuration: data.concept.table");
+		Objects.requireNonNull(dataConceptTable, "Need configuration: data.modifier.table");
 		// ontology configuration
 		// parse language for locale
 		if( config.get("ont.language") == null ){
@@ -165,12 +176,18 @@ public class Import implements AutoCloseable{
 		}
 		PreparedStatement deleteOnt = dbMeta.prepareStatement("DELETE FROM "+getMetaTable()+" WHERE sourcesystem_cd=?");
 		PreparedStatement deleteAccess = dbMeta.prepareStatement("DELETE FROM "+getAccessTable()+" WHERE c_table_cd LIKE ?");
+		PreparedStatement deleteModifiers = dbMeta.prepareStatement("DELETE FROM "+getAccessTable()+" WHERE c_table_cd LIKE ?");
 		PreparedStatement deleteConcepts = dbData.prepareStatement("DELETE FROM "+getConceptTable()+" WHERE sourcesystem_cd=?");
 		
 		deleteConcepts.setString(1, sourceIdDelete);
 		this.deleteConceptCount = deleteConcepts.executeUpdate();
 //		System.out.println("Deleted "+deleteConceptCount+" rows from "+getConceptTable());
 		deleteConcepts.close();
+		
+		// delete modifiers
+		deleteModifiers.setString(1, sourceIdDelete);
+		this.deleteModifierCount = deleteModifiers.executeUpdate();
+		deleteModifiers.close();
 
 		deleteAccess.setString(1, sourceIdDelete+"%");
 		this.deleteAccessCount = deleteAccess.executeUpdate();
@@ -218,6 +235,15 @@ public class Import implements AutoCloseable{
 		insertConcept.executeUpdate();
 		insertConceptCount ++;
 	}
+	private void insertModifierDimension(String path, String name, String concept_cd) throws SQLException{
+		insertModifier.setString(1, path);
+		insertModifier.setString(2, concept_cd);
+		insertModifier.setString(3, name);
+		insertModifier.setTimestamp(4, sourceTimestamp);
+		insertModifier.setString(5, sourceId);
+		insertModifier.executeUpdate();
+		insertModifierCount ++;
+	}
 	private String generateMetadataXML(ValueRestriction vr) throws XMLStreamException, OntologyException{
 		StringWriter xmlbuf = new StringWriter();
 		XMLOutputFactory xmlout = XMLOutputFactory.newInstance();
@@ -348,10 +374,17 @@ public class Import implements AutoCloseable{
 		}while( !done );
 		return b.toString();
 	}
+
+	private String buildConceptPath(String parentPath, Concept concept){
+		String path_part = concept.getID(); // TODO unique key sufficient, e.g. try label.hashCode()
+		return parentPath + path_part + "\\";
+	}
+	private String buildModifierPath(String parentPath, Concept modifier){
+		return buildConceptPath(parentPath, modifier);
+	}
 	private void insertMeta(int level, String path_prefix, Concept concept, boolean accessRoot) throws SQLException, OntologyException{
 		insertMeta.setInt(1, level);
 		String label = concept.getPrefLabel(locale);
-		String path_part = concept.getID(); // TODO unique key sufficient, e.g. try label.hashCode()
 		
 		if( label == null ){
 			// no label for language, try to get neutral label
@@ -364,7 +397,7 @@ public class Import implements AutoCloseable{
 		}
 		
 		// use hashcode
-		String path = path_prefix + path_part+"\\";
+		String path = buildConceptPath(path_prefix, concept);
 		insertMeta.setString(2, path);
 		insertMeta.setString(3, label);
 		// c_synonym_cd
@@ -403,9 +436,14 @@ public class Import implements AutoCloseable{
 		}
 
 		// c_basecode
-		// TODO support multiple ids (e.g. adding virtual leaves)
-		if( conceptIds.length != 0 ){			
+		if( conceptIds.length == 1 ){
+			// single concept code
+			insertConceptDimension(path, label, conceptIds[0]);
+		}else if( conceptIds.length > 1 ){
+			// multiple concept codes. 
+			// TODO add virtual leaves to tree
 			// insert into concept_dimension
+			// XXX this will violate the primary key (concept_path) if multiple notations are used
 			insertConceptDimension(path, label, conceptIds[0]);
 			// concept has id and can occur in fact table			
 			// TODO make sure, each concept_path is inserted only once
@@ -453,6 +491,9 @@ public class Import implements AutoCloseable{
 		insertMeta.executeUpdate();
 		insertMetaCount ++;
 		
+		// insert modifiers
+		insertModifiers(concept, path);
+
 		if( accessRoot ){
 			insertAccess.setString(1, sourceId+"_"+Integer.toHexString(concept.hashCode()));
 			insertAccess.setInt(2, level);
@@ -471,6 +512,60 @@ public class Import implements AutoCloseable{
 			insertMeta(level+1, path, sub, false);
 		}
 	}
+
+	private void insertModifiers(Concept concept, String concept_path) throws OntologyException, SQLException{
+		// TODO write modifier dimension
+		Concept[] parts = concept.getParts(false);
+		if( parts == null ){
+			return; // nothing to do
+		}
+		for( Concept part : parts ){
+			insertModifierMetaTree(concept_path, 1, part, "\\");
+		}
+	}
+
+	private void insertModifierMetaTree(String concept_path, int level, Concept modifier, String path_prefix) throws OntologyException, SQLException{
+		// fill modifier dimension first
+		String[] ids = modifier.getNotations();
+		String path = buildModifierPath(path_prefix, modifier);
+		String label = modifier.getPrefLabel(locale);
+		for( int i=0; i<ids.length; i++ ){
+			// XXX will fail if multiple notations are present for a single path
+			insertModifierDimension(path, label, ids[i]);
+		}
+		// "INSERT INTO i2b2 (c_hlevel,c_fullname,c_name,c_synonym_cd,c_visualattributes,c_basecode,c_metadataxml,c_facttablecolumn,c_tablename,c_columnname,c_columndatatype,c_operator,c_dimcode,c_tooltip,m_applied_path,update_date,download_date,import_date,sourcesystem_cd)VALUES(?,?,?,?,?,?,?,'concept_cd','concept_dimension','concept_path','T','LIKE',?,?,?,current_timestamp,?,current_timestamp,?)");
+
+		insertMetaModifier.setInt(1, level); // c_hlevel
+		insertMetaModifier.setString(2, path); // c_fullname
+		// c_name
+		insertMetaModifier.setString(3, label);
+		// c_synonym_cd
+		insertMetaModifier.setString(4, "N");
+		// c_visualattributes
+		insertMetaModifier.setString(5, "RA");
+		// c_basecode
+		insertMetaModifier.setString(6, ids[0]);
+		// c_metadataxml
+		insertMetaModifier.setString(7, null);
+		// c_dimcode
+		insertMetaModifier.setString(8, label);
+		// c_tooltip
+		insertMetaModifier.setString(9, modifier.getDescription(locale));
+		// m_appliedpath
+		insertMetaModifier.setString(10, concept_path+"%");
+		// update, download, import dates
+		insertMetaModifier.setTimestamp(11, sourceTimestamp);
+		// sourcesystem_cd
+		insertMetaModifier.setString(12, sourceId);
+		insertMetaModifier.executeUpdate();
+		insertMetaCount ++;
+
+		// recurse into nested modifier structure
+		for( Concept nested : modifier.getNarrower() ){
+			insertModifierMetaTree(concept_path, level+1, nested, path);
+		}
+		
+	}
 	/*
 	public void loadOntology(Class<?> ontologyClass, Map<String,String> config) throws Exception{
 		Plugin plugin = Plugin.newInstance(ontologyClass, config);

histream-i2b2/src/test/java/de/sekmi/histream/i2b2/ont/TestOntologyImport.java

@@ -26,7 +26,8 @@ public class TestOntologyImport {
 		props.put("meta.sourcesystem_cd", "test");
 		props.put("meta.table", "i2b2metadata.i2b2");
 		props.put("meta.access", "i2b2metadata.table_access");
-		props.put("data.concept.table", "i2b2crcdata.concepts");
+		props.put("data.concept.table", "i2b2crcdata.concept_dimension");
+		props.put("data.concept.table", "i2b2crcdata.modifier_dimension");
 		props.put("ont.language", "en");
 		imp = new Import(dbcMeta, dbcData, props);
 	}

histream-maven-plugin/src/test/resources/ontology.properties

@@ -7,7 +7,7 @@ meta.basepath=\\i2b2\\
 meta.sourcesystem_cd=test
 
 data.concept.table=i2b2demodata.concept_dimension
-
+data.modifier.table=i2b2demodata.modifier_dimension
 
 # language to use for accessing ontology
 ont.language=DE

histream-maven-plugin/src/test/resources/skos-aktin-cda.ttl

@@ -497,11 +497,18 @@
     dwh:restriction dwh:Integer ;
     skos:notation	"LOINC:9268-4" .
     
-    
+
+:RightPupil a skos:Concept ;
+	dwh:isPartOf :Pupilsize ;
+    skos:prefLabel "Right Pupil"@en ;
+    skos:prefLabel "Rechte Pupille"@de ;
+    skos:notation	"right".
+	
 :Pupilsize a skos:Concept ;
     skos:inScheme :EmergencyNote ;
     skos:prefLabel "Size of pupil"@en ;
     skos:prefLabel "Pupillenweite"@de ;
+	dwh:hasPart :RightPupil ;
     dc:description "Pupillenweite: Auswahlliste" ;
     
     skos:narrower :dilated ;

histream-maven-plugin/src/test/resources/test-ontology.ttl

+@prefix rdf:	<http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs:	<http://www.w3.org/2000/01/rdf-schema#> .
+@prefix owl:	<http://www.w3.org/2002/07/owl#> .
+@prefix dc:	<http://purl.org/dc/elements/1.1/> .
+@prefix skos: 	<http://www.w3.org/2004/02/skos/core#> .
+@prefix xsd:	<http://www.w3.org/2001/XMLSchema#> .
+@prefix dwh:	<http://sekmi.de/histream/dwh#> .
+@prefix :	<http://sekmi.de/histream/skos/tests#> .
+
+: dc:title "HIStream concept ontology for testing".
+
+:TestScheme a skos:ConceptScheme ;
+	skos:hasTopConcept :TestData .
+
+:TestData a skos:Concept ;
+	skos:prefLabel "TestData_label_en"@en ;
+	skos:prefLabel "TestData_label_de"@de ;
+	skos:narrower :TestStringConcept ;
+	skos:narrower :TestIntegerConcept ;
+	skos:narrower :TestDecimalConcept ;
+	skos:narrower :TestEnumConcept ;
+	
+	dc:description "Top concept for test data" .
+
+:TestEnumConcept a skos:Concept;
+	skos:inScheme 	:TestData;
+	dwh:restriction [ a	owl:Restriction ;
+				owl:onProperty rdf:value;
+				owl:oneOf (
+					[rdf:value "1"^^xsd:string; skos:prefLabel "1_de"@de; skos:prefLabel "1_en"@en ]
+					[rdf:value "2"^^xsd:string; skos:prefLabel "2_de"@de; skos:prefLabel "2_en"@en ]
+				)
+			] ;
+	skos:prefLabel 	"Enum_label_de"@de;
+	skos:prefLabel 	"Enum_label_en"@en;
+	skos:notation 	"T:Enum" .
+#######	skos:notation 	"Enum"^^:TestCS .
+
+dwh:Integer a owl:Restriction ;
+	owl:onProperty rdf:value ;
+	owl:allValuesFrom xsd:integer .
+	
+dwh:String a owl:Restriction ;
+	owl:onProperty rdf:value ;
+	owl:allValuesFrom xsd:string .
+
+:TestIntegerConcept a skos:Concept;
+	skos:inScheme	:TestData ;
+	dwh:restriction dwh:Integer ;
+	skos:prefLabel	"Integer_label_en"@en ;
+	skos:prefLabel	"Integer_label_de"@de ;
+	dc:description	"Description_de"@de ;
+	dc:description	"Description_neutral" ;
+	skos:notation	"T:type:int" .
+
+:TestStringConcept a skos:Concept;
+	skos:inScheme	:TestData ;
+	dwh:restriction dwh:String ;
+	skos:prefLabel	"String_label_en"@en ;
+	skos:notation	"T:type:str" .
+	
+:TestDecimalConcept a skos:Concept;
+	skos:inScheme	:TestData ;
+	dwh:restriction [ a	owl:Restriction ;
+				owl:onProperty	rdf:value ;
+				owl:allValuesFrom[	a		rdfs:Datatype; 
+							owl:onDatatype	xsd:decimal; 
+							owl:withRestrictions( 
+									[ xsd:minInclusive 200 ] 	
+									[ xsd:maxInclusive 400 ]
+							)
+						]
+			] ;
+	skos:prefLabel	"Decimal_label_en"@en ;
+	skos:notation	"T:type:dec" .
+
+# Test inferred topConcept and narrower
+:OtherScheme a skos:ConceptScheme .
+
+:OtherTop a skos:Concept ;
+	skos:inScheme	:OtherScheme ;
+	skos:topConceptOf	:OtherScheme ;
+	skos:prefLabel "OtherTop_label_en"@en .
+
+:OtherSub a skos:Concept ;
+	skos:inScheme	:OtherScheme ;
+	skos:inScheme	:TestScheme ;
+	skos:broader	:OtherTop ;
+	skos:prefLabel "OtherSub_label_en"@en ;
+	skos:notation	"other" .
+
+# Second dimension / modifiers
+:Mod1 a skos:Concept ;
+	dwh:isPartOf	:TestData ;
+	skos:prefLabel	"Mod1 en"@en ;
+	skos:notation	"m1" .
\ No newline at end of file

histream-skos/examples/test-ontology.ttl

@@ -88,3 +88,9 @@ dwh:String a owl:Restriction ;
 	skos:broader	:OtherTop ;
 	skos:prefLabel "OtherSub_label_en"@en ;
 	skos:notation	"other" .
+
+# Second dimension / modifiers
+:Mod1 a skos:Concept ;
+	dwh:isPartOf	:TestData ;
+	skos:prefLabel	"Mod1 en"@en ;
+	skos:notation	"m1" .
\ No newline at end of file

histream-skos/src/main/java/de/sekmi/histream/ontology/skos/ConceptImpl.java

@@ -89,9 +89,8 @@ public class ConceptImpl implements Concept {
 	}
 
 	@Override
-	public Concept[] getParts() {
-		// TODO implement
-		return null;
+	public Concept[] getParts(boolean inherited) throws OntologyException {
+		return store.getParts(this, inherited);
 	}
 
 	@Override

histream-skos/src/main/java/de/sekmi/histream/ontology/skos/HIStreamOntology.java

@@ -16,6 +16,8 @@ public class HIStreamOntology {
 	public static final URI DWH_OTHERWISE;
 	public static final URI DWH_XPATH;
 	public static final URI DWH_ECMASCRIPT;
+	public static final URI DWH_HAS_PART;
+	public static final URI DWH_IS_PART_OF;
 	
 	static {
 		final ValueFactory f = ValueFactoryImpl.getInstance();
@@ -28,6 +30,8 @@ public class HIStreamOntology {
 		DWH_OTHERWISE = f.createURI(DWH_NAMESPACE, "otherwise");
 		DWH_XPATH = f.createURI(DWH_NAMESPACE, "XPath");
 		DWH_ECMASCRIPT = f.createURI(DWH_NAMESPACE, "ECMAScript");
+		DWH_HAS_PART = f.createURI(DWH_NAMESPACE, "hasPart"); // TODO use correct names/URIs
+		DWH_IS_PART_OF = f.createURI(DWH_NAMESPACE, "isPartOf"); // XXX see above
 		
 	}
 }

histream-skos/src/main/java/de/sekmi/histream/ontology/skos/Store.java

@@ -6,6 +6,7 @@ import java.io.IOException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -48,6 +49,8 @@ public class Store implements Ontology, Plugin {
 	private Resource inferredContext;
 	// SKOS scheme to enforce unique notations
 	private Resource scheme;
+	private Map<String, String> namespaceShortener;
+	
 	/// XXX  namespace resolver
 	/**
 	 * Plugin constructor which accepts configuration parameters.
@@ -139,6 +142,7 @@ public class Store implements Ontology, Plugin {
 		return count;
 	}
 	private void initializeRepo(Iterable<File> files, Iterable<URL> urls, String baseURI) throws RepositoryException, IOException{
+	    namespaceShortener = new HashMap<>();
 	    repo = new SailRepository(new MemoryStore());
 	    repo.initialize();
 		rc = repo.getConnection();
@@ -171,6 +175,7 @@ public class Store implements Ontology, Plugin {
 	    inferred += inferInverseRelations(SKOS.TOP_CONCEPT_OF, SKOS.HAS_TOP_CONCEPT);
 	    inferred += inferInverseRelations(SKOS.BROADER, SKOS.NARROWER);
 	    inferred += inferInverseRelations(SKOS.NARROWER, SKOS.BROADER);
+	    // TODO a isPartOf b -> b hasPart a
 	    if( inferred != 0 ){
 	    	log.fine("Inferred "+inferred+" statements.");
 	    }
@@ -230,6 +235,9 @@ public class Store implements Ontology, Plugin {
 	Concept[] getNarrower(ConceptImpl concept)throws OntologyException{
 		return getRelatedConcepts(concept.getResource(), SKOS.NARROWER);
 	}
+	Concept[] getParts(ConceptImpl concept, boolean inherited)throws OntologyException{
+		return getRelatedConcepts(concept.getResource(), HIStreamOntology.DWH_HAS_PART);
+	}
 	
 	public void printTopConcepts() throws OntologyException{
 		for( Concept c : getTopConcepts() ){

histream-skos/src/test/java/de/sekmi/histream/ontology/skos/OntologyTest.java

@@ -13,6 +13,7 @@ import de.sekmi.histream.ontology.Concept;
 import de.sekmi.histream.ontology.OntologyException;
 import de.sekmi.histream.ontology.ValueRestriction;
 import de.sekmi.histream.ontology.skos.Store;
+import de.sekmi.histream.xml.NamespaceResolver;
 
 public class OntologyTest {
 	Store store;