Commit ef79d79b authored by R.W.Majeed's avatar R.W.Majeed

unify configuration properties with prefixes jdbc.*, data.jdbc.*, meta.jdbc.*

parent 5315e37d
......@@ -30,7 +30,6 @@ import java.sql.Timestamp;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Logger;
......@@ -43,7 +42,7 @@ import de.sekmi.histream.Value;
import de.sekmi.histream.impl.AbstractObservationHandler;
/**
* Inserts observtions in to the i2b2 observation_fact table.
* Inserts observations in to the i2b2 observation_fact table.
* <p>
* Need to specify default provider id for i2b2 in property 'nullProvider'.
* Null values in providerId will be be replaced with this value and stored in
......@@ -173,12 +172,13 @@ public class I2b2Inserter extends AbstractObservationHandler implements Observat
* @throws ClassNotFoundException if database driver not found
*/
private void open(Map<String,String> props)throws SQLException, ClassNotFoundException{
Properties jdbc = new Properties();
PostgresExtension.copyProperties(props, "jdbc", jdbc);
PostgresExtension.copyProperties(props, "data.jdbc", jdbc);
db = PostgresExtension.getConnection(jdbc);
db = PostgresExtension.getConnection(props, new String[]{"jdbc.","data.jdbc."});
db.setAutoCommit(false);
this.nullProviderId = props.get("nullProvider");
if( this.nullProviderId == null ){
log.warning("property 'nullProvider' missing, using '@' (may violate foreign keys)");
this.nullProviderId = "@";
}
prepareStatements(props);
}
......@@ -415,8 +415,15 @@ public class I2b2Inserter extends AbstractObservationHandler implements Observat
throw new IllegalArgumentException("Unknown etl strategy "+value);
}
}else{
throw new IllegalArgumentException("Unknown meta key "+value);
throw new IllegalArgumentException("Unknown meta key "+key);
}
}
public int getInsertCount(){
return insertCount;
}
public void resetInsertCount(){
this.insertCount = 0;
}
}
......@@ -52,15 +52,15 @@ public abstract class PostgresExtension<T> implements Extension<T>, Plugin {
this.config = configuration;
}
private static Connection getConnection(Map<String,String> props) throws SQLException, ClassNotFoundException{
Class.forName(driver);
Properties jdbcProps = new Properties();
// TODO put only properties relevant to jdbc
jdbcProps.putAll(props);
return DriverManager.getConnection("jdbc:postgresql://"+props.get("host")+":"+props.get("port")+"/"+props.get("database"), jdbcProps);
public static Connection getConnection(Map<String,String> props, String[] prefixes) throws SQLException, ClassNotFoundException{
Properties jdbc = new Properties();
for( String prefix : prefixes ){
PostgresExtension.copyProperties(props, prefix, jdbc);
}
return getConnection(jdbc);
}
public static Connection getConnection(Properties props) throws SQLException, ClassNotFoundException{
private static Connection getConnection(Properties props) throws SQLException, ClassNotFoundException{
Class.forName(driver);
return DriverManager.getConnection("jdbc:postgresql://"+props.get("host")+":"+props.get("port")+"/"+props.get("database"), props);
}
......@@ -81,8 +81,8 @@ public abstract class PostgresExtension<T> implements Extension<T>, Plugin {
);
}
protected void open() throws ClassNotFoundException, SQLException{
db = getConnection(config);
protected void openDatabase(String[] propertyPrefixes) throws ClassNotFoundException, SQLException{
db = getConnection(config, propertyPrefixes);
prepareStatements();
}
......
......@@ -108,7 +108,12 @@ public class PostgresPatientStore extends PostgresExtension<I2b2Patient> impleme
* Construct new postgres patient store. In addition to properties
* needed by {@link PostgresExtension#PostgresExtension(Map)},
* the following properties are needed:
* <p>projectId,
* <p>jdbc.{host|port|database} or data.jdbc.{host|port|database} to
* construct the database URI.
* Any other parameters under jdbc. or data.jdbc. are passed to the
* JDBC connect method.
*
* <p>project,
* <p>Optional properties:
* <p>
* idSourceDefault ('HIVE'), idSourceSeparator (single char, ':')
......@@ -128,7 +133,9 @@ public class PostgresPatientStore extends PostgresExtension<I2b2Patient> impleme
// this.autoInsertSourceId = "HS.auto";
patientCache = new Hashtable<>(1000);
idCache = new Hashtable<>(1000);
open();
openDatabase(new String[]{"jdbc.","data.jdbc."});
loadMaxPatientNum();
batchLoad();
}
private I2b2Patient getCached(int patient_num){
......@@ -188,13 +195,6 @@ public class PostgresPatientStore extends PostgresExtension<I2b2Patient> impleme
public int size(){
return patientCache.size();
}
@Override
public void open()throws SQLException, ClassNotFoundException{
super.open();
loadMaxPatientNum();
batchLoad();
}
private void batchLoad() throws SQLException{
try( ResultSet rs = selectAll.executeQuery() ){
......@@ -210,8 +210,9 @@ public class PostgresPatientStore extends PostgresExtension<I2b2Patient> impleme
ArrayList<String> ids = new ArrayList<>(16);
int num = 0; // current patient number
while( rs.next() ){
if( num == 0 )num = rs.getInt(1);
else if( num != rs.getInt(1) ){
if( num == 0 ){ // first patient
num = rs.getInt(1);
}else if( num != rs.getInt(1) ){
// next patient
// cache ids for current patients
p = getCached(num);
......
......@@ -87,14 +87,16 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit>{
projectId = config.get("project");
idSourceDefault = "HIVE";
idSourceSeparator = ':';
open();
openDatabase(new String[]{"jdbc.","data.jdbc."});
db.setAutoCommit(true);
loadMaxEncounterNum();
batchLoad();
}
@Override
protected void open()throws SQLException, ClassNotFoundException{
super.open();
// db is already opened by super.connect()
db.setAutoCommit(true);
protected void prepareStatements() throws SQLException {
// TODO: use prefix from configuration to specify tablespace
insert = db.prepareStatement("INSERT INTO visit_dimension(encounter_num, patient_num, import_date, download_date, sourcesystem_cd) VALUES(?,?,current_timestamp,?,?)");
insertMapping = db.prepareStatement("INSERT INTO encounter_mapping(encounter_num, encounter_ide, encounter_ide_source, patient_ide, patient_ide_source, encounter_ide_status, project_id, import_date, download_date, sourcesystem_cd) VALUES(?,?,?,?,?,'A','"+projectId+"',current_timestamp,?,?)");
......@@ -106,12 +108,9 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit>{
selectMappingsAll.setFetchSize(getFetchSize());
deleteSource = db.prepareStatement("DELETE FROM visit_dimension WHERE sourcesystem_cd=?");
deleteMapSource = db.prepareStatement("DELETE FROM encounter_mapping WHERE sourcesystem_cd=?");
loadMaxEncounterNum();
batchLoad();
deleteMapSource = db.prepareStatement("DELETE FROM encounter_mapping WHERE sourcesystem_cd=?");
}
public int size(){
return visitCache.size();
}
......@@ -611,11 +610,6 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit>{
loadMaxEncounterNum();
}
@Override
protected void prepareStatements() throws SQLException {
// TODO Auto-generated method stub
}
@Override
public I2b2Visit createInstance(Object... args) throws UnsupportedOperationException {
......@@ -626,8 +620,7 @@ public class PostgresVisitStore extends PostgresExtension<I2b2Visit>{
{
throw new IllegalArgumentException("Need arguments String visitId, I2b2Patient patient, ExternalSourceType source");
}
// TODO: implement
throw new UnsupportedOperationException("TODO: implement");
return getOrCreateInstance((String)args[0], (I2b2Patient)args[1], (ExternalSourceType)args[2]);
}
@Override
......
......@@ -391,21 +391,13 @@ public class Import implements AutoCloseable{
sourceId = config.get("meta.sourcesystem_cd");
//String connectString = "jdbc:postgresql://"+props.get("jdbc.host")+":"+props.get("jdbc.port")+"/"+props.get("jdbc.database");
Properties jdbc;
// use only properties relevant to JDBC
// meta connection
jdbc = new Properties();
PostgresExtension.copyProperties(config, "jdbc.", jdbc);
PostgresExtension.copyProperties(config, "meta.jdbc.", jdbc);
dbMeta = PostgresExtension.getConnection(jdbc);
dbMeta = PostgresExtension.getConnection(config,new String[]{"jdbc.","meta.jdbc."});
dbMeta.setAutoCommit(true);
// data connection
jdbc = new Properties();
PostgresExtension.copyProperties(config, "jdbc.", jdbc);
PostgresExtension.copyProperties(config, "data.jdbc.", jdbc);
dbData = PostgresExtension.getConnection(jdbc);
dbData = PostgresExtension.getConnection(config,new String[]{"jdbc.","data.jdbc."});
dbData.setAutoCommit(true);
prepareStatements();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment