|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |
java.lang.Objectorg.jhuapl.edu.sages.etl.strategy.SagesOpenCsvJar
public abstract class SagesOpenCsvJar
SagesOpenCsvJar
is the domain class and controls the execution of the overall ETL process.
It contains an ETLStrategyTemplate
object that implements most of the ETL processing logic.
Field Summary | |
---|---|
protected File[] |
csvFiles
List of files that ETL loads into the production table |
protected ArrayList<String[]> |
currentEntries
Current file records that the ETL will load into the production table via SQL statements |
protected int |
currentRecNum
TODO: not used yet |
protected String |
dbms
target database connection settings |
protected String |
dbName
|
protected Map<String,String> |
DEST_COLTYPE_MAP
maps the destination columns to their sql-datatype qualifier for generating the schema |
protected Map<String,Integer> |
DEST_SQLTYPE_MAP
maps the destination columns to their java.sql.Types for setting ? parameters on prepared statements |
protected String |
dst_table_name
|
protected int |
errorFlag
errorFlag to control what to do on certain errors |
protected static String |
ETL_CLEANSE_TABLE
|
protected static String |
ETL_STAGING_DB
|
protected ETLStrategyTemplate |
etlStrategy
The ETLStrategyTemplate object |
protected File |
fileMarkedForDeletion
TODO: not used yet. |
protected String[] |
header_src
header columns used to define the CLEANSE table schema |
protected String |
inputdir_csvfiles
ETL looks at the input directory for files to process |
protected Map<String,String> |
MAPPING_MAP
maps the source:destination columns |
protected Map<String,String> |
MAPPING_REV_MAP
maps the destination:source columns |
protected String |
outputdir_csvfiles
ETL moves successfully processed files to the output directory |
protected Map<String,Integer> |
PARAMINDX_DST
maps destination column name to its parameter index in the destination table, indexing starts at 1 |
protected Map<String,Integer> |
PARAMINDX_SRC
maps source column name to its parameter index in the source table, indexing starts at 1 |
protected String |
password
|
protected int |
portNumber
|
protected String |
prod_table_name
|
protected Properties |
props_customsql_cleanse
|
protected Properties |
props_customsql_final_to_prod
|
protected Properties |
props_customsql_staging
|
protected Properties |
props_dateformats
|
protected Properties |
props_etlconfig
properties holders |
protected Properties |
props_mappings
|
protected Map<String,Savepoint> |
savepoints
The savepoints |
protected String |
serverName
|
protected String |
src_table_name
|
protected String |
userName
|
Constructor Summary | |
---|---|
SagesOpenCsvJar()
|
Methods inherited from class java.lang.Object |
---|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
Field Detail |
---|
protected ETLStrategyTemplate etlStrategy
ETLStrategyTemplate
object
protected Map<String,Savepoint> savepoints
protected File[] csvFiles
protected ArrayList<String[]> currentEntries
protected File fileMarkedForDeletion
protected int currentRecNum
protected String inputdir_csvfiles
protected String outputdir_csvfiles
protected static final String ETL_CLEANSE_TABLE
protected static final String ETL_STAGING_DB
protected String src_table_name
protected String dst_table_name
protected String prod_table_name
protected Map<String,String> DEST_COLTYPE_MAP
protected Map<String,Integer> DEST_SQLTYPE_MAP
protected Map<String,String> MAPPING_MAP
protected Map<String,String> MAPPING_REV_MAP
protected Properties props_etlconfig
protected Properties props_mappings
protected Properties props_dateformats
protected Properties props_customsql_cleanse
protected Properties props_customsql_staging
protected Properties props_customsql_final_to_prod
protected String dbms
protected int portNumber
protected String serverName
protected String dbName
protected String userName
protected String password
protected Map<String,Integer> PARAMINDX_SRC
protected Map<String,Integer> PARAMINDX_DST
protected String[] header_src
protected int errorFlag
Constructor Detail |
---|
public SagesOpenCsvJar() throws SagesEtlException
SagesEtlException
Method Detail |
---|
public void setEtlStrategy(ETLStrategyTemplate strategy)
public void extractHeaderColumns(SagesOpenCsvJar socj) throws FileNotFoundException, IOException
FileNotFoundException
IOException
public Savepoint buildCleanseTable(Connection c, SagesOpenCsvJar socj, Savepoint save1) throws SQLException, SagesEtlException
SQLException
SagesEtlException
public Savepoint buildEtlStatusTable(Connection c, SagesOpenCsvJar socj, Savepoint save1) throws SQLException, SagesEtlException
SQLException
SagesEtlException
public void truncateCleanseAndStagingTables(DumbTestOpenCsvJar socj_dumb, Connection c, File file, Savepoint baseLine) throws SagesEtlException, SQLException
SagesEtlException
SQLException
public Savepoint buildStagingTable(Connection c, SagesOpenCsvJar socj, Savepoint save1) throws SQLException, SagesEtlException
SQLException
SagesEtlException
public void generateSourceDestMappings(SagesOpenCsvJar socj)
public void setAndExecuteInsertIntoCleansingTablePreparedStatement(Connection c, SagesOpenCsvJar socj, ArrayList<String[]> entries_rawdata, Savepoint save2, PreparedStatement ps_INSERT_CLEANSE) throws SQLException
SQLException
public String buildInsertIntoCleansingTableSql(Connection c, SagesOpenCsvJar socj) throws SQLException
SQLException
public void copyFromCleanseToStaging(Connection c, SagesOpenCsvJar socj, Savepoint save2) throws SQLException, SagesEtlException
SQLException
SagesEtlException
public int errorCleanup(SagesOpenCsvJar socj, Savepoint savepoint, Connection connection, File currentCsv, String failedDirPath, Exception e)
public void alterCleanseTableAddFlagColumn(Connection c, Savepoint save1, Savepoint createCleanseSavepoint) throws SQLException, SagesEtlException
SQLException
SagesEtlException
public void alterStagingTableAddFlagColumn(Connection c, Savepoint save1, Savepoint createCleanseSavepoint) throws SQLException, SagesEtlException
SQLException
SagesEtlException
protected void initializeProperties(ETLProperties etlProperties) throws SagesEtlException
SagesOpenCsvJar
' s etl properties
etlProperties
- - these are configured by updating the set of ETL properties files
SagesEtlException
- - if property doesn't exist, or issue loading properties file occurspublic Connection getConnection() throws SagesEtlException
SQLException
SagesEtlException
protected static File etlMoveFile(File file, File destinationDir) throws IOException
file
- destinationDir
-
IOException
protected static void runCustomSql(Connection c, Properties customSql, String targetTableName) throws SQLException
socj_dumb
- c
-
SQLException
public static void logFileOutcome(SagesOpenCsvJar socj, Connection c, File file, String outcome) throws SagesEtlException
socj
- TODOc
- outcome
- sql
- canonicalPath
- fileName
- processtime
-
SagesEtlException
SQLException
public static void closeDbConnection(Connection c)
c
- database connectionpublic static SagesEtlException abort(String msg, Throwable e)
msg
- SAGES ETL message to displaye
- the original exception
public String getFaileddir_csvfiles()
public void setFaileddir_csvfiles(String faileddir_csvfiles)
public File getCurrentFile()
public void setCurrentFile(File currentFile)
public static List<File> getFailedCsvFiles()
public static void setFailedCsvFiles(List<File> failedCsvFiles)
public boolean isSuccess()
public void setSuccess(boolean success)
public static SimpleDateFormat getSimpleDateFormat()
|
||||||||||
PREV CLASS NEXT CLASS | FRAMES NO FRAMES | |||||||||
SUMMARY: NESTED | FIELD | CONSTR | METHOD | DETAIL: FIELD | CONSTR | METHOD |