THIS IS A TEST INSTANCE. ALL YOUR CHANGES WILL BE LOST!!!!
Summary
We currently have two entities in sqoop that expose configs, connector and driver ( after it was renamed from framework-driver in SQOOP-1498 ). Driver is not persisted in the repository like the various connectors. The proposal is to change this and treat both of them as configurable entities in the sqoop repository.
Requirements
- Do not use "null" as an primary key indicator to identify Driver
- Connectors and Drivers expose configs and hence both should be in one table and they can uniquely identified by the primary key
- Make sure the driver and its config upgrade works
- Remove DIRECTION field out of SQ_CONFIG, since it is not applicable to all configurable. For instance Driver has no DIRECTION. Connectors have DIRECTION and it should be only relevant to connectors.
Design & Implementation Details
Note: The work related to removing DIRECTION field out of config was done in a separate ticket after this proposal.
Repository Schema changes
- Rename the SQ_CONNECTOR to SQ_CONFIGURABLE
- Rename relevant FK fields to reflect the SQ_CONFIGURABLE
- Driver is now added a first class row in the SQ_CONFIGURABLE, and has a real persistence ID, instead of a fake Id/ null
| Code Block |
|---|
// SQOOP-1557 upgrade queries for table rename for CONNECTOR-> CONFIGURABLE
// drop the SQ_CONFIG FK for connector table
public static final String QUERY_UPGRADE_DROP_TABLE_SQ_CONFIG_CONNECTOR_CONSTRAINT = "ALTER TABLE "
+ TABLE_SQ_CONFIG + " DROP CONSTRAINT " + CONSTRAINT_SQ_CFG_SQC;
// drop the SQ_LINK FK for connector table
public static final String QUERY_UPGRADE_DROP_TABLE_SQ_LINK_CONSTRAINT = "ALTER TABLE "
+ TABLE_SQ_LINK + " DROP CONSTRAINT " + CONSTRAINT_SQ_LNK_SQC;
// drop the SQ_CONNECTOR_DIRECTION FK for connector table
public static final String QUERY_UPGRADE_DROP_TABLE_SQ_CONNECTOR_DIRECTION_CONSTRAINT = "ALTER TABLE "
+ TABLE_SQ_CONNECTOR_DIRECTIONS + " DROP CONSTRAINT " + CONSTRAINT_SQCD_SQC;
// rename
public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_CONNECTOR_TO_SQ_CONFIGURABLE = "RENAME TABLE "
+ TABLE_SQ_CONNECTOR + " TO SQ_CONFIGURABLE";
public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_CONFIG_COLUMN_1 = "RENAME COLUMN "
+ TABLE_SQ_CONFIG + "." + COLUMN_SQ_CFG_CONNECTOR + " TO " + COLUMN_SQ_CFG_CONFIGURABLE;
public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_LINK_COLUMN_1 = "RENAME COLUMN "
+ TABLE_SQ_LINK + "." + COLUMN_SQ_LNK_CONNECTOR + " TO " + COLUMN_SQ_LNK_CONFIGURABLE;
// add a type column to the configurable
public static final String QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ADD_COLUMN_SQC_TYPE = "ALTER TABLE "
+ TABLE_SQ_CONFIGURABLE + " ADD COLUMN " + COLUMN_SQC_TYPE + " VARCHAR(32)";
// add the constraints back for SQ_CONFIG
public static final String QUERY_UPGRADE_ADD_TABLE_SQ_CONFIG_CONFIGURABLE_CONSTRAINT = "ALTER TABLE "
+ TABLE_SQ_CONFIG
+ " ADD CONSTRAINT "
+ CONSTRAINT_SQ_CFG_SQC
+ " "
+ "FOREIGN KEY ("
+ COLUMN_SQ_CFG_CONFIGURABLE
+ ") "
+ "REFERENCES "
+ TABLE_SQ_CONFIGURABLE
+ " ("
+ COLUMN_SQC_ID + ")";
// add the constraints back for SQ_LINK
public static final String QUERY_UPGRADE_ADD_TABLE_SQ_LINK_CONFIGURABLE_CONSTRAINT = "ALTER TABLE "
+ TABLE_SQ_LINK
+ " ADD CONSTRAINT "
+ CONSTRAINT_SQ_LNK_SQC
+ " "
+ "FOREIGN KEY ("
+ COLUMN_SQ_LNK_CONFIGURABLE
+ ") "
+ "REFERENCES "
+ TABLE_SQ_CONFIGURABLE
+ " ("
+ COLUMN_SQC_ID + ")"; |
Repository Schema upgrade changes
Upgrade code in derby added for renames of table and field names
Code Block Override public void createOrUpgradeRepository(Connection conn) { ..} private void renameConnectorToConfigurable(Connection conn) { // SQ_CONNECTOR to SQ_CONFIGURABLE upgrade runQuery(QUERY_UPGRADE_DROP_TABLE_SQ_CONFIG_CONNECTOR_CONSTRAINT, conn); runQuery(QUERY_UPGRADE_DROP_TABLE_SQ_LINK_CONSTRAINT, conn); runQuery(QUERY_UPGRADE_DROP_TABLE_SQ_CONNECTOR_DIRECTION_CONSTRAINT, conn); runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_CONNECTOR_TO_SQ_CONFIGURABLE, conn); runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_CONFIG_COLUMN_1, conn); runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_LINK_COLUMN_1, conn); runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ADD_COLUMN_SQC_TYPE, conn); runQuery(QUERY_UPGRADE_ADD_TABLE_SQ_CONFIG_CONFIGURABLE_CONSTRAINT, conn); runQuery(QUERY_UPGRADE_ADD_TABLE_SQ_LINK_CONFIGURABLE_CONSTRAINT, conn); runQuery(QUERY_UPGRADE_ADD_TABLE_SQ_CONNECTOR_DIRECTION_CONSTRAINT, conn); LOG.info("CONNECTOR TABLE altered and constraints added for CONFIGURABLE"); }- Force register the driver as an entity for the "Driver" upgrade path to work across release
| Code Block |
|---|
/**
* Pre-register Driver since the 1.99.3 release NOTE: This should be used only
* in the upgrade path
*/
@Deprecated
protected long registerDriver(Connection conn) {
if (LOG.isTraceEnabled()) {
LOG.trace("Begin Driver loading.");
}
PreparedStatement baseDriverStmt = null;
try {
baseDriverStmt = conn.prepareStatement(STMT_INSERT_INTO_CONFIGURABLE,
Statement.RETURN_GENERATED_KEYS);
baseDriverStmt.setString(1, MDriver.DRIVER_NAME);
baseDriverStmt.setString(2, Driver.getClassName());
baseDriverStmt.setString(3, "1");
baseDriverStmt.setString(4, MConfigurableType.DRIVER.name());
int baseDriverCount = baseDriverStmt.executeUpdate();
if (baseDriverCount != 1) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0003, Integer.toString(baseDriverCount));
}
ResultSet rsetDriverId = baseDriverStmt.getGeneratedKeys();
if (!rsetDriverId.next()) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0004);
}
return rsetDriverId.getLong(1);
} catch (SQLException ex) {
throw new SqoopException(DerbyRepoError.DERBYREPO_0009, ex);
} finally {
closeStatements(baseDriverStmt);
}
} |
Repository API changes
registerDrivermethod, makes a actual call to the repository and persists and entry into the SQ_CONFIGURABLEfindDriversimilarly fetches from the repository