DUE TO SPAM, SIGN-UP IS DISABLED. Goto Selfserve wiki signup and request an account.
...
| Code Block | ||||
|---|---|---|---|---|
| ||||
{
"apiKey": TBD,
"type": "request",
"listeners": ["broker", "zkBroker"],
"name": "StreamsGroupHeartbeatRequest",
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "GroupId", "type": "string", "versions": "0+", "entityType": "groupId",
"about": "The group identifier." },
{ "name": "MemberId", "type": "string", "versions": "0+",
"about": "The member ID generated by the streams consumer. The member ID must be kept during the entire lifetime of the streams consumer process." },
{ "name": "MemberEpoch", "type": "int32", "versions": "0+",
"about": "The current member epoch; 0 to join the group; -1 to leave the group; -2 to indicate that the static member will rejoin." },
{ "name": "InstanceId", "type": "string", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "null if not provided or if it didn't change since the last heartbeat; the instance ID for static membership otherwise." },
{ "name": "RackId", "type": "string", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "null if not provided or if it didn't change since the last heartbeat; the rack ID of the member otherwise." },
{ "name": "RebalanceTimeoutMs", "type": "int32", "versions": "0+", "default": -1,
"about": "-1 if it didn't change since the last heartbeat; the maximum time in milliseconds that the coordinator will wait on the member to revoke its tasks otherwise." },
{ "name": "Topology", "type": "Topology", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "The topology metadata of the streams application. Used to initialize the topology of the group and to check if the topology corresponds to the topology initialized for the group. Only sent when memberEpoch = 0, must be non-empty. Null otherwise.",
"fields": [
{ "name": "Epoch", "type": "int32", "versions": "0+",
"about": "The epoch of the topology. Used to check if the topology corresponds to the topology initialized on the brokers." },
{ "name": "Subtopologies", "type": "[]Subtopology", "versions": "0+",
"about": "The sub-topologies of the streams application.",
"fields": [
{ "name": "SubtopologyId", "type": "string", "versions": "0+",
"about": "String to uniquely identify the sub-topologysubtopology. Deterministically generated from the topology" },
{ "name": "SourceTopics", "type": "[]string", "versions": "0+",
"about": "The topics the topology reads from." },
{ "name": "SourceTopicRegex", "type": "[]string", "versions": "0+",
"about": "The regular expressions identifying topics the sub-topologysubtopology reads from." },
{ "name": "StateChangelogTopics", "type": "[]TopicInfo", "versions": "0+",
"about": "The set of state changelog topics associated with this sub-topologysubtopology. Created automatically." },
{ "name": "RepartitionSinkTopics", "type": "[]string", "versions": "0+",
"about": "The repartition topics the sub-topologysubtopology writes to." },
{ "name": "RepartitionSourceTopics", "type": "[]TopicInfo", "versions": "0+",
"about": "The set of source topics that are internally created repartition topics. Created automatically." },
{ "name": "CopartitionGroups", "type": "[]CopartitionGroup", "versions": "0+",
"about": "A subset of source topics that must be copartitioned.",
"fields": [
{ "name": "SourceTopics", "type": "[]int16", "versions": "0+",
"about": "The topics the topology reads from. Index into the array on the subtopology level." },
{ "name": "SourceTopicRegex", "type": "[]int16", "versions": "0+",
"about": "Regular expressions identifying topics the subtopology reads from. Index into the array on the subtopology level." },
{ "name": "RepartitionSourceTopics", "type": "[]int16", "versions": "0+",
"about": "The set of source topics that are internally created repartition topics. Index into the array on the subtopology level." }
]}
]}
]
},
{ "name": "ActiveTasks", "type": "[]TaskIds", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Currently owned active tasks for this client. Null if unchanged since last heartbeat." },
{ "name": "StandbyTasks", "type": "[]TaskIds", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Currently owned standby tasks for this client. Null if unchanged since last heartbeat." },
{ "name": "WarmupTasks", "type": "[]TaskIds", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Currently owned warm-up tasks for this client. Null if unchanged since last heartbeat." },
{ "name": "ProcessId", "type": "string", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Identity of the streams instance that may have multiple consumers. Null if unchanged since last heartbeat." },
{ "name": "UserEndpoint", "type": "Endpoint", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "User-defined endpoint for Interactive Queries. Null if unchanged since last heartbeat or if not defined on the client." },
{ "name": "ClientTags", "type": "[]KeyValue", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Used for rack-aware assignment algorithm. Null if unchanged since last heartbeat." },
{ "name": "TaskOffsets", "type": "[]TaskOffset", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Cumulative changelog offsets for tasks. Only updated when a warm-up task has caught up, and according to the task offset interval. Null if unchanged since last heartbeat." },
{ "name": "TaskEndOffsets", "type": "[]TaskOffset", "versions": "0+", "nullableVersions": "0+", "default": "null",
"about": "Cumulative changelog end-offsets for tasks. Only updated when a warm-up task has caught up, and according to the task offset interval. Null if unchanged since last heartbeat." },
{ "name": "ShutdownApplication", "type": "bool", "versions": "0+", "default": false,
"about": "Whether all Streams clients in the group should shut down." }
],
"commonStructs": [
{ "name": "KeyValue", "versions": "0+", "fields": [
{ "name": "Key", "type": "string", "versions": "0+",
"about": "key of the config" },
{ "name": "Value", "type": "string", "versions": "0+",
"about": "value of the config" }
]},
{ "name": "TopicInfo", "versions": "0+", "fields": [
{ "name": "Name", "type": "string", "versions": "0+",
"about": "The name of the topic." },
{ "name": "Partitions", "type": "int32", "versions": "0+",
"about": "The number of partitions in the topic. Can be 0 if no specific number of partitions is enforced. Always 0 for changelog topics." },
{ "name": "ReplicationFactor", "type": "int16", "versions": "0+",
"about": "The replication factor of the topic. Can be 0 if the default replication factor should be used." },
{ "name": "TopicConfigs", "type": "[]KeyValue", "versions": "0+",
"about": "Topic-level configurations as key-value pairs."
}
]},
{ "name": "Endpoint", "versions": "0+", "fields": [
{ "name": "Host", "type": "string", "versions": "0+",
"about": "host of the endpoint" },
{ "name": "Port", "type": "uint16", "versions": "0+",
"about": "port of the endpoint" }
]},
{ "name": "TaskOffset", "versions": "0+", "fields": [
{ "name": "SubtopologyId", "type": "string", "versions": "0+",
"about": "The subtopology identifier." },
{ "name": "Partition", "type": "int32", "versions": "0+",
"about": "The partition." },
{ "name": "Offset", "type": "int64", "versions": "0+",
"about": "The offset." }
]},
{ "name": "TaskIds", "versions": "0+", "fields": [
{ "name": "SubtopologyId", "type": "string", "versions": "0+",
"about": "The subtopology identifier." },
{ "name": "Partitions", "type": "[]int32", "versions": "0+",
"about": "The partitions of the input topics processed by this member." }
]}
]
} |
...
| Code Block | ||||
|---|---|---|---|---|
| ||||
{
"type": "data",
"name": "StreamsGroupTopologyValue",
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "Epoch", "type": "int32", "versions": "0+",
"about": "The epoch of the topology. Must be non-zero. " },
{ "name": "Subtopologies", "type": "[]Subtopology", "versions": "0+",
"about": "The sub-topologies of the streams application.",
"fields": [
{ "name": "SubtopologyId", "type": "string", "versions": "0+",
"about": "String to uniquely identify the subtopology." },
{ "name": "SourceTopics", "type": "[]string", "versions": "0+",
"about": "The topics the topology reads from." },
{ "name": "SourceTopicRegex", "type": "[]string", "versions": "0+",
"about": "Regular expressions identifying topics the subtopology reads from." },
{ "name": "StateChangelogTopics", "type": "[]TopicInfo", "versions": "0+",
"about": "The set of state changelog topics associated with this subtopology." },
{ "name": "RepartitionSinkTopics", "type": "[]string", "versions": "0+",
"about": "The repartition topics the subtopology writes to." },
{ "name": "RepartitionSourceTopics", "type": "[]TopicInfo", "versions": "0+",
"about": "The set of source topics that are internally created repartition topics." },
{ "name": "CopartitionGroups", "type": "[]CopartitionGroup", "versions": "0+",
"about": "A subset of source topics that must be copartitioned.",
"fields": [
{ "name": "SourceTopics", "type": "[]int16", "versions": "0+",
"about": "The topics the topology reads from. Index into the array on the subtopology level." },
{ "name": "SourceTopicRegex", "type": "[]int16", "versions": "0+",
"about": "Regular expressions identifying topics the subtopology reads from. Index into the array on the subtopology level." },
{ "name": "RepartitionSourceTopics", "type": "[]int16", "versions": "0+",
"about": "The set of source topics that are internally created repartition topics. Index into the array on the subtopology level." }
]
}
]
}
],
"commonStructs": [
{ "name": "TopicConfig", "versions": "0+", "fields": [
{ "name": "key", "type": "string", "versions": "0+",
"about": "The key of the topic-level configuration." },
{ "name": "value", "type": "string", "versions": "0+",
"about": "The value of the topic-level configuration," }
]
},
{ "name": "TopicInfo", "versions": "0+", "fields": [
{ "name": "Name", "type": "string", "versions": "0+",
"about": "The name of the topic." },
{ "name": "Partitions", "type": "int32", "versions": "0+",
"about": "The number of partitions in the topic. Can be 0 if no specific number of partitions is enforced. Always 0 for changelog topics." },
{ "name": "ReplicationFactor", "type": "int16", "versions": "0+",
"about": "The replication factor of the topic. Can be 0 if the default replication factor should be used." },
{ "name": "TopicConfigs", "type": "[]TopicConfig", "versions": "0+",
"about": "Topic-level configurations as key-value pairs."
}
]}
]
} |
...
| Code Block | ||||||
|---|---|---|---|---|---|---|
| ||||||
/**
* A detailed description of a single member in the group.
*/
public class StreamsGroupMemberDescription {
public StreamsGroupMemberDescription(
final String memberId,
final int memberEpoch,
final Optional<String> instanceId,
final String rackId,
final String clientId,
final String clientHost,
final int topologyEpoch,
final String processId,
final Endpoint userEndpoint,
final Map<String, String> clientTags,
final List<TaskOffset> taskOffsets,
final List<TaskOffset> taskEndOffsets,
final StreamsGroupMemberAssignment assignment,
final Optional<StreamsGroupMemberAssignment> targetAssignment,
final boolean isClassic
);
/**
* The id of the group member.
*/
public String memberId();
/**
* The epoch of the group member.
*/
public int memberEpoch();
/**
* The id of the instance, used for static membership, if available.
*/
public Optional<String> instanceId();
/**
* The rack ID of the group member.
*/
public String rackId();
/**
* The client id of the group member.
*/
public String clientId();
/**
* The host of the group member.
*/
public String clientHost();
/**
* The epoch of the group member.
*/
public int memberEpoch() {
return memberEpoch;
}
/**
* The epoch of the topology present on the client.
*/
public int topologyEpoch();
/**
* Identity of the streams instance that may have multiple clients.
*/
public String processId();
/**
* User-defined endpoint for Interactive Queries.
*/
public Endpoint userEndpoint() {
return userEndpoint;
}
/**
* Used for rack-aware assignment algorithm.
*/
public Map<String, String> clientTags();
/**
* Cumulative offsets for tasks.
*/
public List<TaskOffset> taskOffsets();
/**
* Cumulative task changelog end offsets for tasks.
*/
public List<TaskOffset> taskEndOffsets();
/**
* The current assignment.
*/
public StreamsGroupMemberAssignment assignment();
/**
* The target assignment.
*/
public Optional<StreamsGroupMemberAssignment> targetAssignment();
/**
* The flag indicating whether a member is classic.
*/
public boolean isClassic() {
return isClassic;
}
/**
* The cumulative offset for one task.
*/
public static class TaskOffset {
public TaskOffset(final String subtopologysubtopologyId, final int partition, final long offset);
/**
* The subtopology identifier.
*/
public String subtopologysubtopologyId();
/**
* The partition of the task.
*/
public int partition();
/**
* The cumulative offset (sum of offsets in all input partitions).
*/
public long offset();
}
} |
...
| Code Block | ||||||
|---|---|---|---|---|---|---|
| ||||||
/**
* A description of the assignments of a specific streams group member.
*/
public class StreamsGroupMemberAssignment {
public StreamsGroupMemberAssignment(
final List<TaskIds> activeTasks,
final List<TaskIds> standbyTasks,
final List<TaskIds> warmupTasks);
/**
* Active tasks for this client.
*/
public List<TaskIds> activeTasks();
/**
* Standby tasks for this client.
*/
public List<TaskIds> standbyTasks();
/**
* Warmup tasks for this client.
*/
public List<TaskIds> warmupTasks();
/**
* All tasks for one subtopology of a member.
*/
public static class TaskIds {
public TaskIds(final String subtopologysubtopologyId, final List<Integer> partitions);
/**
* The subtopology identifier.
*/
public String subtopologysubtopologyId();
/**
* The partitions of the input topics processed by this member.
*/
public List<Integer> partitions();
}
} |
...
| Code Block | ||||||
|---|---|---|---|---|---|---|
| ||||||
/**
* A detailed description of a single subtopology
*/
public class StreamsGroupSubtopologyDescription {
public StreamsGroupSubtopologyDescription(
final String subtopologysubtopologyId,
final List<String> sourceTopics,
final List<String> repartitionSinkTopics,
final Map<String, TopicInfo> stateChangelogTopics,
final Map<String, TopicInfo> repartitionSourceTopics);
/**
* String to uniquely identify the subtopology.
*/
public String subtopologysubtopologyId();
/**
* The topics the topology reads from.
*/
public List<String> sourceTopics();
/**
* The repartition topics the topology writes to.
*/
public List<String> repartitionSinkTopics();
/**
* The set of state changelog topics associated with this sub-topologysubtopology.
*/
public Map<String, TopicInfo> stateChangelogTopics();
/**
* The set of source topics that are internally created repartition topics.
*/
public Map<String, TopicInfo> repartitionSourceTopics();
/**
* Information about a topic.
*/
public static class TopicInfo {
public TopicInfo(final int partitions, final Map<String, String> topicConfigs);
/**
* The number of partitions in the topic.
*/
public int partitions();
/**
* Configurations of the topic.
*/
public Map<String, String> topicConfigs();
}
} |
...