Page History

...

Code Block

language	java
title	Collector.java

/** This class take response for collecting data to output stream. */
public interface Collector<OUT> {
    /**
     * Collect record to output stream.
     *
     * @param record to be collected.
     */
    void collect(OUT record);      
    
    /**
     * Overwrite the timestamp of this record and collect it to output stream.
     *
     * @param record to be collected.
     * @param timestamp of the processed data.
     */
    void collectAndOverwriteTimestamp(OUT record, long timestamp); 
}

...

NonPartitionedContext

Sometimes it is not possible to decide on the current partition in the context of the processing. For example, when processing the input records from the broadcast edge.

Code Block

language	java
title	OneInputStreamProcessFunctionCollector.java

/**
 * This class containsinterface represents the context associated with all logicaloperations relatedmust tobe processapplied recordsto fromall
 single* inputpartitions.
 */
 public interface OneInputStreamProcessFunction<IN, OUT> extends ProcessFunctionNonPartitionedContext<OUT> {
    /**
     * ProcessApply recorda andfunction emitto data through {@link Collector}.
     *
     * @param record to process.
     * @param output to emit processed recordsall partitions. For keyed stream, it will apply to all keys. For
     * non-keyed stream, it will apply to single partition.
     */
 @param ctx, runtime contextvoid in which this function is executed.
     */
    void processRecord(IN record, Collector<OUT> output, RuntimeContext ctx) throws Exception;

applyToAllPartitions(ApplyPartitionFunction<OUT> applyPartitionFunction);
}

/** A function to be applied to all partitions . */
@FunctionalInterface
public interface ApplyPartitionFunction<OUT> {
    /**
     * ThisThe isactual amethod life-cycleto methodbe indicatesapplied thatto thiseach functionpartition.
 will no longer receive any input*
     * data.
@param collector to output  *data.
     * @param ctx, theruntime context in which this function is executed.
     */
    default void endInput(NonPartitionedContextapply(Collector<OUT> collector, RuntimeContext ctx) throws {}Exception;
}

...

OneInputStreamProcessFunction

Code Block

language	java
title	TwoInputStreamProcessFunctionOneInputStreamProcessFunction.java

/** This class contains all logical related to process records from twosingle input. */
public interface TwoInputStreamProcessFunction<IN1OneInputStreamProcessFunction<IN, IN2, OUT> extends ProcessFunction {
    /**
     * Process record from first input and emit data through {@link Collector}.
     *
     * @param record to process.
     * @param output to emit processed records.
     * @param ctx, runtime context in which this function is executed.
     */
    void processRecordFromFirstInputprocessRecord(IN1IN record, Collector<OUT> output, RuntimeContext ctx)
            throws Exception;

    /**
     * ProcessThis recordis froma secondlife-cycle inputmethod andindicates emitthat datathis throughfunction {@link Collector}.
     *will no longer receive any input
     * @param record to processdata.
     * @param output to emit processed records.
     * @param ctx, runtimethe context in which this function is executed.
     */
    default void endInput(NonPartitionedContext ctx) {}
}

TwoInputStreamProcessFunction

Code Block

language	java
title	TwoInputStreamProcessFunction.java

/** This class contains all logical related to process records from two input. */
public interface TwoInputStreamProcessFunction<IN1, IN2, OUT> extends ProcessFunction {
processRecordFromSecondInput(IN2 record, Collector<OUT> output, RuntimeContext ctx)
            throws Exception;

    /**
     * ThisProcess isrecord afrom life-cyclefirst methodinput indicatesand thatemit thisdata functionthrough will no longer receive any data from{@link Collector}.
     *
     * the@param record firstto inputprocess.
     * @param output to emit processed records.
     * @param ctx, theruntime context in which this function is executed.
     */
    default void endFirstInput(NonPartitionedContextprocessRecordFromFirstInput(IN1 record, Collector<OUT> output, RuntimeContext ctx) {}

    /**
     * This is a life-cycle method indicates that this function will no longer receive any data from
     * the second inputthrows Exception;

    /**
     * Process record from second input and emit data through {@link Collector}.
     *
     * @param record to process.
     * @param output to emit processed records.
     * @param ctx, theruntime context in which this function is executed.
     */
    default void endSecondInput(NonPartitionedContextprocessRecordFromSecondInput(IN2 record, Collector<OUT> output, RuntimeContext ctx) {}
}

BroadcastTwoInputStreamProcessFunction

Code Block

language	java
title	BroadcastTwoInputStreamProcessFunction

/**
 * This class contains all logical related to process records from a broadcast stream and a
 * non-broadcast stream.
 */
public interface BroadcastTwoInputStreamProcessFunction<IN1, IN2, OUT> extends ProcessFunction {
    /**
     * Process record from non-broadcast input and emit data through {@link Collector}.
     *
            throws Exception;

    /**
     * This is a life-cycle method indicates that this function will no longer receive any data from
     * @paramthe recordfirst to processinput.
     * @param output to emit processed records.
     * @param ctx, runtimethe context in which this function is executed.
     */
    default void processRecordFromNonBroadcastInputendFirstInput(IN1NonPartitionedContext record, Collector<OUT> output, RuntimeContext ctx)ctx) {}

    /**
     * This is a life-cycle method indicates throws Exception;

    /**
     * Process record from broadcast input.In general, the broadcast side is not allowed tothat this function will no longer receive any data from
     * the second input.
     *
 manipulate state and output data* because@param itctx, correspondsthe tocontext allin partitionswhich insteadthis offunction ais singleexecuted.
     */
 partition. But you coulddefault usevoid endSecondInput(NonPartitionedContext ctx) {}
}

BroadcastTwoInputStreamProcessFunction

Code Block

language	java
title	BroadcastTwoInputStreamProcessFunction

/**
 * This class contains all logical related to process records from a broadcast stream and a
 * non-broadcast stream.
 */
public interface BroadcastTwoInputStreamProcessFunction<IN1, IN2, OUT> extends ProcessFunction {broadcast context to process all the partitions at once.
     *
     * @param record to process.
     * @param ctx, the context in which this function is executed.
     */
    void processRecordFromBroadcastInput(IN2 record, NonPartitionedContext<OUT> ctx) throws Exception;

    /**
     * ThisProcess isrecord afrom lifenon-cyclebroadcast methodinput indicatesand thatemit thisdata functionthrough will no longer receive any data from{@link Collector}.
     *
     * @param therecord non-broadcastto inputprocess.
     * @param output to emit processed records.
     * @param ctx, theruntime context in which this function is executed.
     */
    default void endNonBroadcastInput(NonPartitionedContextprocessRecordFromNonBroadcastInput(IN1 record, Collector<OUT> output, RuntimeContext ctx) {}
            throws Exception;

    /**
     * ThisProcess record from broadcast input.In general, the broadcast side is anot life-cycle method indicates that this function will no longer receive any data from
     * the broadcast input.allowed to
     * manipulate state and output data because it corresponds to all partitions instead of a single
     *
 partition. But you could *use @parambroadcast ctx,context theto contextprocess inall whichthe thispartitions functionat is executedonce.
     */
     default* void@param endBroadcastInput(NonPartitionedContext<OUT> ctx) {}
}   

/**
record to process.
     * This@param interfacectx, represents the context associated with all operations must be applied to all
 * partitions.
 */ public interface NonPartitionedContext<OUT> {in which this function is executed.
     */
    void processRecordFromBroadcastInput(IN2 record, NonPartitionedContext<OUT> ctx) throws Exception;

    /**
     * This Applyis a life-cycle method indicates that this function to all partitions. For keyed stream, it will apply to all keys. For will no longer receive any data from
     * the non-broadcast input.
     *
     * non-keyed@param streamctx, it will apply to single partitionthe context in which this function is executed.
     */
    default void applyToAllPartitionsendNonBroadcastInput(ApplyPartitionFunction<OUT>NonPartitionedContext applyPartitionFunctionctx);
 {}

/** A function to be applied to all partitions . */
@FunctionalInterface
public interface ApplyPartitionFunction<OUT> {
    /**
     * The actual method to be applied to each partition. /**
     * This is a life-cycle method indicates that this function will no longer receive any data from
     *
 the broadcast input.
  * @param collector to output data. *
     * @param ctx, runtimethe context in which this function is executed.
     */
    default void applyendBroadcastInput(Collector<OUT>NonPartitionedContext<OUT> collector, RuntimeContext ctx) throws Exception;{}
}

TwoOutputStreamProcessFunction

...

Page tree

Versions Compared

Old Version 33

New Version 34

Key

NonPartitionedContext

OneInputStreamProcessFunction

TwoInputStreamProcessFunction

BroadcastTwoInputStreamProcessFunction

BroadcastTwoInputStreamProcessFunction

TwoOutputStreamProcessFunction