DUE TO SPAM, SIGN-UP IS DISABLED. Goto Selfserve wiki signup and request an account.
...
| Code Block | ||||
|---|---|---|---|---|
| ||||
{
"apiKey": 100,
"type": "request",
"listeners": ["controller"],
"name": "DisklessDescribeFilesRequest",
// Version 0 is the initial version.
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "Files", "type": "[]DisklessDescribeFileEntry", "versions": "0+",
"about": "Each file to describe.", "fields": [
{ "name": "ObjectKey", "type": "string", "versions": "0+", "mapKey": true,
"about": "The object key identifying the file in object storage." }
]},
{ "name": "TimeoutMs", "type": "int32", "versions": "0+", "default": "60000",
"about": "How long to wait in milliseconds before timing out the request." }
]
}
{
"apiKey": 100,
"type": "response",
"name": "DisklessDescribeFilesResponse",
// Version 0 is the initial version.
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true,
"about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
{ "name": "Files", "type": "[]DisklessDescribeFileResult", "versions": "0+",
"about": "Results for each file.", "fields": [
{ "name": "ObjectKey", "type": "string", "versions": "0+", "mapKey": true,
"about": "The object key identifying the file in object storage." },
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
"about": "The error code, or 0 if there was no error." },
{ "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", "ignorable": true, "default": "null",
"about": "The error message, or null if there was no error." },
{ "name": "HasLiveBatches", "type": "bool", "versions": "0+",
"about": "True if the file still has live batches tracked by this coordinator." }
]}
]
} |
Metadata records
DisklessPartitionCreatedRecord
| Code Block | ||||
|---|---|---|---|---|
| ||||
{
"apiKey": 100,
"type": "data",
"name": "DisklessPartitionsCreatedRecordDisklessPartitionCreatedRecord",
// Version 0 is the initial version.
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "TopicsTopicId", "type": "[]DisklessCreatedPartitionTopicuuid", "versions": "0+",
"about": "The topicsunique withtopic created partitionsID.", "fields": [
},
{ "name": "TopicIdName", "type": "uuidstring", "versions": "0+",
"entityType": "topicName",
"about": "The unique topic IDname." },
{ "name": "NamePartitionIndex", "type": "stringint32", "versions": "0+", "entityType": "topicName",
"about": "The topicpartition nameindex." },
{ "name": "Partitions",]
} |
DisklessTopicDeletedRecord
| Code Block | ||||
|---|---|---|---|---|
| ||||
{ "apiKey": 101, "type": "[]DisklessCreatedPartitiondata", "versionsname": "0+DisklessTopicDeletedRecord", // Version 0 is the initial version. "aboutvalidVersions": "0"The, partitions created for this topic.","flexibleVersions": "0+", "fields": [ { "name": "PartitionIndexTopicId", "type": "int32uuid", "versions": "0+", "about": "The unique partitiontopic indexID." } ]} ]} ] } | ||||
| Code Block | ||||
| language | js | title | DisklessPartitionCreatedRecord
DisklessFileCommittedRecord
TODO: optimize for storing fewer bytes per record
| Code Block | ||||
|---|---|---|---|---|
| ||||
| Code Block | ||||
|
Configuration
TBD
Monitoring
| |
{
"apiKey": 102,
"type": "data",
"name": "DisklessFileCommittedRecord",
// Version 0 is the initial version.
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "ObjectKey", "type": "string", "versions": "0+",
"about": "The object key identifying the file in object storage." },
{ "name": "ObjectFormat", "type": "int8", "versions": "0+",
"about": "The format of the object (e.g. 1 = WRITE_AHEAD_MULTI_SEGMENT)." },
{ "name": "FileSize", "type": "int64", "versions": "0+",
"about": "The total size of the file in bytes." },
{ "name": "UploaderBrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
"about": "The broker that uploaded the file." },
{ "name": "OwnerIds", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
"about": "The list of Diskless Coordinator IDs that share ownership of this file." },
{ "name": "Topics", "type": "[]DisklessCommittedTopic", "versions": "0+",
"about": "Each topic with accepted batches in this file.", "fields": [
{ "name": "TopicId", "type": "uuid", "versions": "0+",
"about": "The unique topic ID." },
{ "name": "Partitions", "type": "[]DisklessCommittedPartition", "versions": "0+",
"about": "Each partition with accepted batches for this topic.", "fields": [
{ "name": "PartitionIndex", "type": "int32", "versions": "0+",
"about": "The partition index." },
{ "name": "Batches", "type": "[]DisklessCommittedBatch", "versions": "0+",
"about": "The accepted batches for this partition.", "fields": [
{ "name": "Magic", "type": "int8", "versions": "0+",
"about": "The record batch magic byte." },
{ "name": "ByteOffset", "type": "int32", "versions": "0+",
"about": "The byte offset of this batch within the file." },
{ "name": "ByteSize", "type": "int32", "versions": "0+",
"about": "The size of the batch in bytes." },
{ "name": "AssignedBaseOffset", "type": "int64", "versions": "0+",
"about": "The base offset assigned by the coordinator." },
{ "name": "AssignedLastOffset", "type": "int64", "versions": "0+",
"about": "The last offset assigned by the coordinator." },
{ "name": "LogAppendTimestamp", "type": "int64", "versions": "0+",
"about": "The log append timestamp assigned by the coordinator." },
{ "name": "BatchMaxTimestamp", "type": "int64", "versions": "0+",
"about": "The maximum timestamp in the batch." },
{ "name": "TimestampType", "type": "int8", "versions": "0+",
"about": "The timestamp type (0 = CreateTime, 1 = LogAppendTime)." },
{ "name": "ProducerId", "type": "int64", "versions": "0+",
"about": "The producer ID, or -1 if not set." },
{ "name": "ProducerEpoch", "type": "int16", "versions": "0+",
"about": "The producer epoch, or -1 if not set." },
{ "name": "BaseSequence", "type": "int32", "versions": "0+",
"about": "The base sequence number, or -1 if not set." },
{ "name": "LastSequence", "type": "int32", "versions": "0+",
"about": "The last sequence number, or -1 if not set." }
]}
]}
]}
]
} |
DisklessRecordsDeletedRecord
| Code Block | ||||
|---|---|---|---|---|
| ||||
{
"apiKey": 103,
"type": "data",
"name": "DisklessRecordsDeletedRecord",
// Version 0 is the initial version.
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "TopicId", "type": "uuid", "versions": "0+",
"about": "The unique topic ID." },
{ "name": "PartitionIndex", "type": "int32", "versions": "0+",
"about": "The partition index." },
{ "name": "NewLogStartOffset", "type": "int64", "versions": "0+",
"about": "The new log start offset after deletion. All batches with last offset less than this value are removed." }
]
} |
Configuration
Broker configuration
More to be added as discussion unrolls
| Configuration | Description | Values |
|---|---|---|
diskless.metadata.topic.num.partitions | The number of Diskless metadata partitions and the number of Diskless coordinators. | |
diskless.coordinator.orphan.scan.interval.ms | The interval for scanning for orphan files. | |
diskless.coordinator.orphan.scan.grace.period.ms | The grace period to give to deletable files. |
Monitoring
More to be added as discussion unrollsTBD
Compatibility, Deprecation, and Migration Plan
...