Versions Compared

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.
Comment: Add some Configuration

...

DisklessFileCommittedRecord

TODO: optimize for storing fewer bytes per record

Code Block
languagejs
titleDisklessFileCommittedRecord
{
  "apiKey": 102,
  "type": "data",
  "name": "DisklessFileCommittedRecord",
  // Version 0 is the initial version.
  "validVersions": "0",
  "flexibleVersions": "0+",
  "fields": [
    { "name": "ObjectKey", "type": "string", "versions": "0+",
      "about": "The object key identifying the file in object storage." },
    { "name": "ObjectFormat", "type": "int8", "versions": "0+",
      "about": "The format of the object (e.g. 1 = WRITE_AHEAD_MULTI_SEGMENT)." },
    { "name": "FileSize", "type": "int64", "versions": "0+",
      "about": "The total size of the file in bytes." },
    { "name": "UploaderBrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
      "about": "The broker that uploaded the file." },
    { "name": "OwnerIds", "type": "[]int32", "versions": "0+", "entityType": "brokerId",
      "about": "The list of Diskless Coordinator IDs that share ownership of this file." },
    { "name": "BatchesTopics", "type": "[]DisklessCommittedBatchDisklessCommittedTopic", "versions": "0+",
      "about": "TheEach topic with accepted batches in this file.", "fields": [
      { "name": "MagicTopicId", "type": "int8uuid", "versions": "0+",
        "about": "The recordunique batchtopic magic byteID." },
      { "name": "TopicIdPartitions", "type": "uuid[]DisklessCommittedPartition", "versions": "0+",
        "about": "TheEach topicpartition IDwith ofaccepted thebatches partitionfor this batch belongs to." },topic.", "fields": [
        { "name": "PartitionIndex", "type": "int32", "versions": "0+",
          "about": "The partition index this batch belongs to." },
        { "name": "ByteOffsetBatches", "type": "int32[]DisklessCommittedBatch", "versions": "0+",
          "about": "The byteaccepted offsetbatches offor this batch within the file." },
 partition.", "fields": [
          { "name": "ByteSizeMagic", "type": "int32int8", "versions": "0+",
            "about": "The size of therecord batch in magic byte." },
          { "name": "ByteOffset", "type": "int32", "versions": "0+",
            "about": "The byte offset of this batch within the file." },
          { "name": "ByteSize", "type": "int32", "versions": "0+",
            "about": "The size of the batch in bytes." },
          { "name": "AssignedBaseOffset", "type": "int64", "versions": "0+",
            "about": "The base offset assigned by the coordinator." },
          { "name": "AssignedLastOffset", "type": "int64", "versions": "0+",
            "about": "The last offset assigned by the coordinator." },
          { "name": "LogAppendTimestamp", "type": "int64", "versions": "0+",
            "about": "The log append timestamp assigned by the coordinator." },
          { "name": "BatchMaxTimestamp", "type": "int64", "versions": "0+",
            "about": "The maximum timestamp in the batch." },
          { "name": "TimestampType", "type": "int8", "versions": "0+",
            "about": "The timestamp type (0 = CreateTime, 1 = LogAppendTime)." },
          { "name": "ProducerId", "type": "int64", "versions": "0+",
            "about": "The producer ID, or -1 if not set." },
          { "name": "ProducerEpoch", "type": "int16", "versions": "0+",

            "about": "The producer epoch, or -1 if not set." },
          { "name": "BaseSequence", "type": "int32", "versions": "0+",
            "about": "The base sequence number, or -1 if not set." },
          { "name": "LastSequence", "type": "int32", "versions": "0+",
            "about": "The last sequence number, or -1 if not set." }
        ]}
      ]}
    ]}
  ]
}

DisklessRecordsDeletedRecord

Code Block
languagejs
titleDisklessRecordsDeletedRecord
{
  "apiKey": 103,
  "type": "data",
  "name": "DisklessRecordsDeletedRecord",
  // Version 0 is the initial version.
  "validVersions": "0",
  "flexibleVersions": "0+",
  "fields": [
    { "name": "TopicId", "type": "uuid", "versions": "0+",
      "about": "The unique topic ID." },
    { "name": "PartitionIndex", "type": "int32", "versions": "0+",
      "about": "The partition index." },
    { "name": "NewLogStartOffset", "type": "int64", "versions": "0+",
      "about": "The new log start offset after deletion. All batches with last offset less than this value are removed." }
  ]
}

Configuration

TBD

Monitoring

Broker configuration

More to be added as discussion unrolls

ConfigurationDescriptionValues
diskless.metadata.topic.num.partitionsThe number of Diskless metadata partitions and the number of Diskless coordinators.
diskless.coordinator.orphan.scan.interval.msThe interval for scanning for orphan files.
diskless.coordinator.orphan.scan.grace.period.msThe grace period to give to deletable files.

Monitoring

More to be added as discussion unrollsTBD

Compatibility, Deprecation, and Migration Plan

...