Page History

/**
 * An intermediate layer for passing data from the execution engine
 * to the ETL engine.
 */
public abstract class DataReader {
  /**
   * Read data from the execution engine as an object array.
   * @return - array of objects with each column represented as an object
   * @throws Exception
   */
  public abstract Object[] readArrayRecord() throws Exception;
  /**
   * Read data from execution engine as text - as a CSV record.
   * public abstract Object readContent(int type) throws Exception;
   * @return - CSV formatted data.
   * @throws Exception
   */
  public abstract String readTextRecord() throws Exception;
  /**
   * Read data from execution engine as a native format.
   * @return - the content in the native format of the intermediate data
   * format being used.
   * @throws Exception
   */
  public abstract Object readContent() throws Exception;
}

/**
 * An intermediate layer for passing data from the ETL framework
 * to the MR framework.
 */
public abstract class DataWriter {
  /**
   * Write an array of objects into the execution framework
   * @param array - data to be written
   */
  public abstract void writeArrayRecord(Object[] array);
  /**
   * Write data into execution framework as text. The Intermediate Data Format
   * may choose to convert the data to another format based on how the data
   * format is implemented
   * @param text - data represented as CSV text.
   */
  public abstract void writeStringRecord(String text);
  /**
   * Write data in the intermediate data format's native format.
   * @param obj - data to be written
   */
  public abstract void writeRecord(Object obj);
}

 /**
   * Get one row of data.
   *
   * @return - One row of data, represented in the internal/native format of
   *         the intermediate data format implementation.
   */
  public T getData() {
    return data;
  }
  /**
   * Set one row of data. If validate is set to true, the data is validated
   * against the schema.
   *
   * @param obj - A single row of data to be moved.
   */
  public void setData(T obj) {
    this.data = obj;
  }
  /**
   * Get one row of data as CSV text. Use {@link #SqoopIDFUtils} for reading and writing
   * into the sqoop specified CSV text format for each {@link #ColumnType} field in the row
   * Why a "native" internal format and then return CSV text too?
   * Imagine a connector that moves data from a system that stores data as a
   * serialization format called FooFormat. If I also need the data to be
   * written into HDFS as FooFormat, the additional cycles burnt in converting
   * the FooFormat to text and back is useless - so using the sqoop specified
   * CSV text format saves those extra cycles
   * <p/>
   * Most fast access mechanisms, like mysqldump or pgsqldump write the data
   * out as CSV, and most often the source data is also represented as CSV
   * - so having a minimal CSV support is mandated for all IDF, so we can easily read the
   * data out as text and write as text.
   * <p/>
   * @return - String representing the data in CSV text format.
   */
  public abstract String getCSVTextData();
  /**
   * Set one row of data as CSV.
   */
  public abstract void setCSVTextData(String csvText);
  /**
   * Get one row of data as an Object array.
   * Sqoop uses defined object representation
   * for each column type. For instance org.joda.time to represent date.
   * Use {@link #SqoopIDFUtils} for reading and writing into the sqoop
   * specified object format for each {@link #ColumnType} field in the row
   * </p>
   * @return - String representing the data as an Object array
   * If FROM and TO schema exist, we will use SchemaMatcher to get the data according to "TO" schema
   */
  public abstract Object[] getObjectData();
 /**
  * Set one row of data as an Object array.
  * It also should construct the data representation
  * that the IDF represents so that the object is ready to
  * consume when getData is invoked. Custom implementations
  * will override this method to convert form object array
  * to the data format

public abstract class IntermediateDataFormat<T> {
  protected volatile T data;
  public int hashCode() {
    return data.hashCode();
  }

  /**
   * Get one row of data.
   *
   * @return - One row of data, represented in the internal/native format of
   *         the intermediate data format implementation.
   */
  public T getData() {
    return data;
  }
  /**
   * Set one row of data. If validate is set to true, the data is validated
   * against the schema.
   *
   * @param data - A single row of data to be moved.
   */
  public void setData(T data) {
    this.data = data;
  }
  /**
   * Get one row of data as CSV text. Use SqoopDataUtils for reading and writing
   * into the sqoop specified CSV text format for each {@link #ColumnType} field in the row
   * Why a "native" internal format and then return CSV text too?
   * Imagine a connector that moves data from a system that stores data as a
   * serialization format called FooFormat. If I also need the data to be
   * written into HDFS as FooFormat, the additional cycles burnt in converting
   * the FooFormat to text and back is useless - so using the sqoop specified
   * CSV text format saves those extra cycles
   * <p/>
   * Most fast access mechanisms, like mysqldump or pgsqldump write the data
   * out as CSV, and most often the source data is also represented as CSV
   * - so having a minimal CSV support is mandated for all IDF, so we can easily read the
   * data out as text and write as text.
   * <p/>
   * @return - String representing the data in CSV text format.
   */
  public abstract Stringvoid getCSVTextDatasetObjectData(Object[] data);
  /**
   * Set the schema for serializing/de-serializing data.
   /**
   * Set@param oneschema
 row of data* as CSV.
   *
   */
  public- abstractthe voidschema setCSVTextData(String csvText);
  /**used for serializing/de-serializing data
   */
 Get onepublic rowvoid of data as an Object array. Sqoop uses defined object representation
   * for each column type. For instance org.joda.time to represent date.Use SqoopDataUtils
   * for reading and writing into the sqoop specified object format
   * for each {@link #ColumnType} field in the row
   * </p>setSchema(Schema schema) {
    if (schema == null) {
      // TODO(SQOOP-1956): throw an exception since working without a schema is dangerous
      return;
    }
    this.schema = schema;
  ..
  }
  /**
   * @returnSerialize -the Stringfields representingof thethis dataobject as an Object arrayto <code>out</code>.
   *
 If FROM and* TO@param schema exist, we will use SchemaMatcher to get the data according to "TO" schemaout <code>DataOuput</code> to serialize this object into.
   * @throws IOException
   */
  public abstract Object[]void getObjectDatawrite(DataOutput out) throws IOException;
  /**
   * SetDeserialize onethe rowfields of datathis asobject an Object arrayfrom <code>in</code>.
   *
   */
  public abstract void setObjectData(Object[] data);
  /**
   * Set the schema for serializing/de-serializing  data.<p>For efficiency, implementations should attempt to re-use storage in the
   * existing object where possible.</p>
   *
   * @param schema - the schema used for serializing/de-serializing  datain <code>DataInput</code> to deseriablize this object from.
   * @throws IOException
   */
  public abstract void setSchemaread(SchemaDataInput schemain) throws IOException;
  /**
   * SerializeProvide the fieldsexternal ofjars thisthat objectthe to <code>out</code>.
   *IDF depends on
   * @param@return outset <code>DataOuput</code> to serialize this object into.of jars
   */
  public *Set<String> @throwsgetJars() IOException{
   */
  public abstract void write(DataOutput out) throws IOException;
  /**
   * Deserialize the fields of this object from <code>in</code>.
   *
   * <p>For efficiency, implementations should attempt to re-use storage in the
   * existing object where possible.</p>
   *
   * @param in <code>DataInput</code> to deseriablize this object from.
   * @throws IOException
   */
  public abstract void read(DataInput in) throws IOException;
} return new HashSet<String>();
  }
  @Override
  public int hashCode() {
    final int prime = 31;
    int result = 1;
    result = prime * result + ((data == null) ? 0 : data.hashCode());
    result = prime * result + ((schema == null) ? 0 : schema.hashCode());
    return result;
  }