THIS IS A TEST INSTANCE. ALL YOUR CHANGES WILL BE LOST!!!!
...
| Code Block | ||
|---|---|---|
| ||
{
"apiKey": 0,
"type": "data",
"name": "RemoteLogSegmentMetadataRecord",
"validVersions": "0",
"flexibleVersions": "none",
"fields": [
{
"name": "RemoteLogSegmentId",
"type": "RemoteLogSegmentIdEntry",
"versions": "0+",
"about": "Unique idrepresentation of the remote log segment",
"fields": [
{
"name": "TopicIdPartition",
"type": "TopicIdPartitionEntry",
"versions": "topicName",
"0+",
"about": "Represents unique topic partition",
"fields": [
{
"name": "Name",
"type": "string",
"versions": "0+",
"about": "Topic name"
},
{
"name": "topicIdId",
"type": "uuid",
"versions": "0+",
"about": "Topic idUnique identifier of the topic"
},
{
"name": "partitionPartition",
"type": "int32",
"versions": "0+",
"about": "Partition number"
}
]
},
{
"name": "segmentIdId",
"type": "uuid",
"versions": "0+",
"about": "Unique identifier of the remote log segment"
}
]
},
{
"name": "StartOffset",
"type": "int64",
"versions": "0+",
"about": "Start offset of the segment."
},
{
"name": "endOffsetEndOffset",
"type": "int64",
"versions": "0+",
"about": "End offset of the segment."
},
{
"name": "LeaderEpoch",
"type": "int32",
"versions": "0+",
"about": "Leader epoch from which this segment instance is created or updated"
},
{
"name": "MaxTimestamp",
"type": "int64",
"versions": "0+",
"about": "Maximum timestamp with in this segment."
},
{
"name": "EventTimestamp",
"type": "int64",
"versions": "0+",
"about": "Event timestamp of this segment."
},
{
"name": "SegmentLeaderEpochs",
"type": "[]SegmentLeaderEpochEntry",
"versions": "0+",
"about": "Leader epoch cache.",
"fields": [
{
"name": "LeaderEpoch",
"type": "int32",
"versions": "0+",
"about": "Leader epoch"
},
{
"name": "Offset",
"type": "int64",
"versions": "0+",
"about": "Start offset for the leader epoch"
}
]
},
{
"name": "SegmentSizeInBytes",
"type": "int32",
"versions": "0+",
"about": "Segment size in bytes"
},
{
"name": "RemoteLogSegmentState",
"type": "int8",
"versions": "0+",
"about": "State of the remote log segment"
}
]
}
{
"apiKey": 1,
"type": "data",
"name": "RemoteLogSegmentMetadataRecordUpdate",
"validVersions": "0",
"flexibleVersions": "none",
"fields": "none",
[
{
"name": "RemoteLogSegmentId",
"type": "RemoteLogSegmentIdEntry",
"versions": "0+",
"about": "Unique representation of the remote log segment",
"fields": [
{
"name": "RemoteLogSegmentIdTopicIdPartition",
"type": "RemoteLogSegmentIdEntryTopicIdPartitionEntry",
"versions": "0+",
"about": "UniqueRepresents unique id of the remote log segment",
topic partition",
"fields": [
{
"name": "topicName",
"type": "string",
"versions": "0+",
"about": "Topic name"
},
{
"name": "topicIdId",
"type": "uuid",
"versions": "0+",
"about": "Unique identifier of the topic id"
},
{
"name": "partitionPartition",
"type": "int32",
"versions": "0+",
"about": "Partition number"
}
]
},
{
"name": "idId",
"type": "uuid",
"versions": "0+",
"about": "Unique identifier of the remote log segment"
}
]
},
{
"name": "LeaderEpoch",
"type": "int32",
"versions": "0+",
"about": "Leader epoch from which this segment instance is created or updated"
},
{
"name": "EventTimestamp",
"type": "int64",
"versions": "0+",
"about": "Event timestamp of this segment."
},
{
"name": "RemoteLogSegmentState",
"type": "int8",
"versions": "0+",
"about": "State of the remote segment"
}
]
}
{
"apiKey": 2,
"type": "data",
"name": "RemotePartitionDeleteMetadataRecord",
"validVersions": "0",
"flexibleVersions": "none",
"fields": [
{
"name": "TopicIdPartition",
"type": "TopicIdPartitionEntry",
"versions": "0+",
"about": "TopicRepresents unique topic partition",
"fields": [
{
"name": "nameName",
"type": "string",
"versions": "0+",
"about": "Topic name"
},
{
"name": "topicIdId",
"type": "uuid",
"versions": "0+",
"about": "Unique identifier of the topic id"
},
{
"name": "partitionPartition",
"type": "int32",
"versions": "0+",
"about": "Partition number"
}
]
},
{
"name": "epochEpoch",
"type": "int32",
"versions": "0+",
"about": "Epoch (controller or leader) from which this event is created. DELETE_PARTITION_MARKED is sent by the controller. DELETE_PARTITION_STARTED and DELETE_PARTITION_FINISHED are sent by remote log metadata topic partition leader."
},
{
"name": "EventTimestamp",
"type": "int64",
"versions": "0+",
"about": "Event timestamp of this segment."
},
{
"name": "RemotePartitionDeleteState",
"type": "int8",
"versions": "0+",
"about": "Deletion state of the remote partition, its value is RemotePartitionDeleteState.id"
}
]
}
package org.apache.kafka.server.log.remote.storage;
...
/**
* It indicates the deletion state of the remote topic partition. This will be based on the action executed on this
* partition by the remote log service implementation.
*/
public enum RemotePartitionDeleteState {
/**
* This is used when a topic/partition is determined to be deleted by controller.
* This partition is marked for delete by controller. That means, all its remote log segments are eligible for
* deletion so that remote partition removers can start deleting them.
*/
DELETE_PARTITION_MARKED((byte) 0),
/**
* This state indicates that the partition deletion is started but not yet finished.
*/
DELETE_PARTITION_STARTED((byte) 1),
/**
* This state indicates that the partition is deleted successfully.
*/
DELETE_PARTITION_FINISHED((byte) 2);
...
}
package org.apache.kafka.server.log.remote.storage;
...
/**
* It indicates the state of the remote log segment or partition. This will be based on the action executed on this
* segment or partition by the remote log service implementation.
* <p>
*/
public enum RemoteLogSegmentState {
/**
* This state indicates that the segment copying to remote storage is started but not yet finished.
*/
COPY_SEGMENT_STARTED((byte) 0),
/**
* This state indicates that the segment copying to remote storage is finished.
*/
COPY_SEGMENT_FINISHED((byte) 1),
/**
* This state indicates that the segment deletion is started but not yet finished.
*/
DELETE_SEGMENT_STARTED((byte) 2),
/**
* This state indicates that the segment is deleted successfully.
*/
DELETE_SEGMENT_FINISHED((byte) 3),
...
} |
...
| Code Block | ||||
|---|---|---|---|---|
| ||||
<magic><topic-name><topic-id><metadata-topic-offset><sequence-of-serialized-entries>
magic:
unsigned var int, version of this file format.
topic-name:
string, topic name.
topic-id:
uuid, uuid of topic
metadata-topic-offset:
var long, offset of the remote log metadata topic partition upto which this topic partition's remote log metadata is fetched.
serialized-entries:
sequence of serialized entries defined as below, more types can be added later if needed.
Serialization of entry is done as mentioned below. This is very similar to the message format mentioned earlier for storing into the metadata topic.
length : unsigned var int, length of this entry which is sum of sizes of type, version, and data.
type : unsigned var int, represents the value type. This value is 'apikey' as mentioned in the schema.
version : unsigned var int, the 'version' number of the type as mentioned in the schema.
data : record payload in kafka protocol message format, the schema is given below.
Both type and version are added before the data is serialized into record value. Schema can be evolved by adding a new version with the respective changes. A new type can also be supported by adding the respective type and its version.
{
"apiKey": 0,
"type": "data",
"name": "RemoteLogSegmentMetadataRecordStored",
"validVersions": "0",
"flexibleVersions": "none",
"fields": [
{
"name": "segmentIdSegmentId",
"type": "uuid",
"versions": "0+",
"about": "Unique identifier of the log segment"
},
{
"name": "StartOffset",
"type": "int64",
"versions": "0+",
"about": "Start offset of the segment."
},
{
"name": "endOffsetEndOffset",
"type": "int64",
"versions": "0+",
"about": "End offset of the segment."
},
{
"name": "LeaderEpoch",
"type": "int32",
"versions": "0+",
"about": "Leader epoch from which this segment instance is created or updated"
},
{
"name": "MaxTimestamp",
"type": "int64",
"versions": "0+",
"about": "Maximum timestamp with in this segment."
},
{
"name": "EventTimestamp",
"type": "int64",
"versions": "0+",
"about": "Event timestamp of this segment."
},
{
"name": "SegmentLeaderEpochs",
"type": "[]SegmentLeaderEpochEntry",
"versions": "0+",
"about": "Event timestamp of this segment.",
"fields": [
{
"name": "LeaderEpoch",
"type": "int32",
"versions": "0+",
"about": "Leader epoch"
},
{
"name": "Offset",
"type": "int64",
"versions": "0+",
"about": "Start offset for the leader epoch"
}
]
},
{
"name": "SegmentSizeInBytes",
"type": "int32",
"versions": "0+",
"about": "Segment size in bytes"
},
{
"name": "RemoteLogSegmentState",
"type": "int8",
"versions": "0+",
"about": "State of the remote log segment"
}
]
}
{
"apiKey": 1,
"type": "data",
"name": "DeletePartitionStateRecord",
"validVersions": "0",
"flexibleVersions": "none",
"fields": [
{
"name": "epochEpoch",
"type": "int32",
"versions": "0+",
"about": "Epoch (controller or leader) from which this event is created. DELETE_PARTITION_MARKED is sent by the controller. DELETE_PARTITION_STARTED and DELETE_PARTITION_FINISHED are sent by remote log metadata topic partition leader."
},
{
"name": "EventTimestamp",
"type": "int64",
"versions": "0+",
"about": "Event timestamp of this segment."
},
{
"name": "RemotePartitionDeleteState",
"type": "int8",
"versions": "0+",
"about": "Deletion state of the remote partition"
}
]
} |
...