From 4d608cb1894e2a426cb148023bb23b09b34863f2 Mon Sep 17 00:00:00 2001 From: Jorge Esteban Quilcate Otoya Date: Tue, 31 Mar 2026 12:34:22 -0500 Subject: [PATCH] feat(inkless:docker): add diskless tiered storage unification demo Bundle the Aiven Tiered Storage plugin (v1.1.1: core, s3, gcs, azure) in the main Inkless Dockerfile. Replace managed-replicas overlay with ts-unification overlay that enables classic-to-diskless migration testing with managed replicas, rack-aware placement, and the classic remote storage bridge. Classic TS and Inkless share the same bucket with path-based separation. Co-Authored-By: Claude Opus 4.6 --- .../docker-compose-files/inkless/Makefile | 8 +- .../docker-compose-files/inkless/README.md | 100 ++++++++---------- .../docker-compose.managed-replicas.yml | 30 ------ .../inkless/docker-compose.ts-unification.yml | 56 ++++++++++ docker/inkless/Dockerfile | 19 ++++ 5 files changed, 125 insertions(+), 88 deletions(-) delete mode 100644 docker/examples/docker-compose-files/inkless/docker-compose.managed-replicas.yml create mode 100644 docker/examples/docker-compose-files/inkless/docker-compose.ts-unification.yml diff --git a/docker/examples/docker-compose-files/inkless/Makefile b/docker/examples/docker-compose-files/inkless/Makefile index a9a493ed225..cbb2266292c 100644 --- a/docker/examples/docker-compose-files/inkless/Makefile +++ b/docker/examples/docker-compose-files/inkless/Makefile @@ -31,10 +31,10 @@ s3-aws: KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.monitoring.yml -f docker-compose.demo.yml -f docker-compose.s3-aws.yml up $(MAKE) destroy -# Multi-AZ cluster with managed replicas (RF > 1) -.PHONY: managed-replicas -managed-replicas: - KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.s3-local.yml -f docker-compose.managed-replicas.yml up +# Diskless tiered storage unification +.PHONY: ts-unification +ts-unification: + KAFKA_VERSION=$(KAFKA_VERSION) $(DOCKER) compose -f docker-compose.yml -f docker-compose.s3-local.yml -f docker-compose.ts-unification.yml up $(MAKE) destroy .PHONY: destroy diff --git a/docker/examples/docker-compose-files/inkless/README.md b/docker/examples/docker-compose-files/inkless/README.md index 3e405dfa595..88fc9a3a7ae 100644 --- a/docker/examples/docker-compose-files/inkless/README.md +++ b/docker/examples/docker-compose-files/inkless/README.md @@ -12,7 +12,7 @@ See the [Quickstart guide](../../../../docs/inkless/QUICKSTART.md#dockerized-dem | `make gcs-local` | Fake GCS server | | `make azure-local` | Azurite Azure-compatible storage | | `make s3-aws` | Real AWS S3 (requires AWS credentials) | -| `make managed-replicas` | Multi-AZ cluster with managed replicas (RF=2) | +| `make ts-unification` | Diskless tiered storage unification | | `make destroy` | Manual cleanup (run after stopping the demo) | ## Image version @@ -27,20 +27,34 @@ make s3-local KAFKA_VERSION=local # Locally built image (requires: make See [Releases](../../../../docs/inkless/RELEASES.md#docker-images) for all available tags. -## Testing Managed Replicas (Multi-AZ) +## Diskless Tiered Storage Unification -Test diskless topics with managed replicas (RF > 1) using the managed replicas overlay. +Test classic-to-diskless migration and tiered storage unification features. The Inkless image bundles the +[Aiven Tiered Storage plugin](https://github.com/Aiven-Open/tiered-storage-for-apache-kafka) (v1.1.1) — +no local plugin build required. ### Quick start ```bash -# Start 2-broker cluster with rack assignments (az1, az2) -make managed-replicas KAFKA_VERSION=local +# Start 2-broker cluster with classic TS + diskless + migration bridge +make ts-unification # Or manually: -docker compose -f docker-compose.yml -f docker-compose.s3-local.yml -f docker-compose.managed-replicas.yml up -d +docker compose -f docker-compose.yml -f docker-compose.s3-local.yml \ + -f docker-compose.ts-unification.yml up -d ``` +### What's enabled + +| Config | Value | Purpose | +|---------------------------------------|--------|--------------------------------------------------| +| `remote.log.storage.system.enable` | `true` | Classic tiered storage via Aiven TS plugin | +| `diskless.allow.from.classic.enable` | `true` | Allow classic-to-diskless topic migration | +| `classic.remote.storage.force.enable` | `true` | All classic topics get `remote.storage.enable=true` | +| `log.diskless.enable` | `false`| Topics start as classic (not diskless by default)| +| `diskless.managed.rf.enable` | `true` | Managed replicas with rack-aware placement | +| `default.replication.factor` | `2` | One replica per AZ | + ### Cluster topology | Broker | Node ID | Rack | Role | @@ -48,87 +62,65 @@ docker compose -f docker-compose.yml -f docker-compose.s3-local.yml -f docker-co | broker | 1 | az1 | broker+controller | | broker2 | 2 | az2 | broker+controller | +### Storage + +Both classic tiered storage and diskless (Inkless) share the same MinIO `inkless` bucket: +- **Classic tiered storage**: Aiven TS plugin → `tiered-storage/` prefix +- **Diskless (Inkless)**: Inkless → bucket root + ### Test procedure -**Step 1: Create a diskless topic** +**Step 1: Create a classic topic (default)** ```bash docker compose exec broker /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server broker:19092 \ - --create --topic test-managed \ - --config diskless.enable=true \ - --partitions 3 + --create --topic test-classic \ + --partitions 1 --replication-factor 2 ``` -**Step 2: Verify RF=2 (one replica per AZ)** +Verify it has `remote.storage.enable=true` and `diskless.enable=false`: ```bash docker compose exec broker /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server broker:19092 \ - --describe --topic test-managed -``` - -Example output (your exact leader/replica ordering may differ): -``` -Topic: test-managed PartitionCount: 3 ReplicationFactor: 2 - Partition: 0 Leader: 1 Replicas: 1,2 Isr: 1,2 - Partition: 1 Leader: 2 Replicas: 2,1 Isr: 2,1 - Partition: 2 Leader: 1 Replicas: 1,2 Isr: 1,2 + --describe --topic test-classic ``` -You should verify that `ReplicationFactor: 2` and that each partition's replicas include both broker IDs `1` and `2` (one replica per AZ). - -**Step 3: Produce messages** +**Step 2: Produce messages** ```bash docker compose exec broker /opt/kafka/bin/kafka-console-producer.sh \ --bootstrap-server broker:19092 \ - --topic test-managed + --topic test-classic ``` -**Step 4: Consume messages** +**Step 3: Migrate to diskless** ```bash -docker compose exec broker /opt/kafka/bin/kafka-console-consumer.sh \ +docker compose exec broker /opt/kafka/bin/kafka-configs.sh \ --bootstrap-server broker:19092 \ - --topic test-managed \ - --from-beginning + --alter --entity-type topics --entity-name test-classic \ + --add-config diskless.enable=true ``` -### Testing AZ-aware routing - -Use the `diskless_az=` prefix in client ID to enable AZ-aware routing: +**Step 4: Verify migration** ```bash -# Produce with AZ hint (prefers az1 replica) -docker compose exec broker /opt/kafka/bin/kafka-console-producer.sh \ - --bootstrap-server broker:19092 \ - --topic test-managed \ - --command-property client.id=diskless_az=az1 - -# Consume with AZ hint (prefers az2 replica) -docker compose exec broker /opt/kafka/bin/kafka-console-consumer.sh \ +docker compose exec broker /opt/kafka/bin/kafka-topics.sh \ --bootstrap-server broker:19092 \ - --topic test-managed \ - --from-beginning \ - --command-property client.id=diskless_az=az2 + --describe --topic test-classic ``` -### Verifying metrics +The topic should now show `diskless.enable=true`. -Check transformer metrics via Prometheus endpoint: +**Step 5: Consume from beginning (spans remote-classic + diskless)** ```bash -# View all AZ routing metrics -curl -s http://localhost:7070/metrics | grep -Ei "client[-_]az" - -# Key metrics: -# - client-az-hit-rate: Requests where broker found in client AZ -# - client-az-miss-rate: Requests routed to different AZ -# - client-az-unaware-rate: Requests without AZ hint -# - cross-az-routing-total: Cross-AZ routing events -# - fallback-total: Fallbacks to non-replica brokers -# - offline-replicas-routed-around: Routing around offline replicas +docker compose exec broker /opt/kafka/bin/kafka-console-consumer.sh \ + --bootstrap-server broker:19092 \ + --topic test-classic \ + --from-beginning ``` ### Cleanup diff --git a/docker/examples/docker-compose-files/inkless/docker-compose.managed-replicas.yml b/docker/examples/docker-compose-files/inkless/docker-compose.managed-replicas.yml deleted file mode 100644 index 8f8dae0bd47..00000000000 --- a/docker/examples/docker-compose-files/inkless/docker-compose.managed-replicas.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Overlay for testing diskless managed replicas (RF > 1) -# Usage: -# docker compose -f docker-compose.yml -f docker-compose.s3-local.yml -f docker-compose.managed-replicas.yml up -d -# -# This enables: -# - diskless.managed.rf.enable=true (user-defined RF via default.replication.factor) -# - Broker racks configured (az1, az2) -# -# Expected behavior: -# - New diskless topics with RF=-1 resolve to default.replication.factor -# - Use: kafka-topics.sh --describe --topic to verify placement - -services: - broker: - environment: - # Enable managed replicas for diskless topics - KAFKA_DISKLESS_MANAGED_RF_ENABLE: "true" - # RF=-1 resolves to this value (one replica per AZ) - KAFKA_DEFAULT_REPLICATION_FACTOR: "2" - # Configure rack/AZ - KAFKA_BROKER_RACK: "az1" - - broker2: - environment: - # Enable managed replicas for diskless topics - KAFKA_DISKLESS_MANAGED_RF_ENABLE: "true" - # RF=-1 resolves to this value (one replica per AZ) - KAFKA_DEFAULT_REPLICATION_FACTOR: "2" - # Configure rack/AZ - KAFKA_BROKER_RACK: "az2" diff --git a/docker/examples/docker-compose-files/inkless/docker-compose.ts-unification.yml b/docker/examples/docker-compose-files/inkless/docker-compose.ts-unification.yml new file mode 100644 index 00000000000..dbecb937cff --- /dev/null +++ b/docker/examples/docker-compose-files/inkless/docker-compose.ts-unification.yml @@ -0,0 +1,56 @@ +# Overlay for testing diskless tiered storage unification. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.s3-local.yml \ +# -f docker-compose.ts-unification.yml up -d +# +# This enables: +# - Classic tiered storage via Aiven TS plugin (same bucket, "tiered-storage/" prefix) +# - Diskless storage (Inkless, same bucket) +# - diskless.allow.from.classic.enable=true (migration bridge) +# - classic.remote.storage.force.enable=true (all classic topics get remote.storage.enable=true) +# - Managed replicas with rack-aware placement (az1, az2) + +services: + broker: + environment: &ts-unification-env + # --- Classic Tiered Storage (Aiven TS plugin) --- + KAFKA_REMOTE_LOG_STORAGE_SYSTEM_ENABLE: "true" + KAFKA_REMOTE_LOG_MANAGER_TASK_INTERVAL_MS: "5000" + # Remote Log Metadata Manager (built-in, topic-based) + KAFKA_REMOTE_LOG_METADATA_MANAGER_CLASS_NAME: "org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManager" + KAFKA_REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME: "PLAINTEXT" + KAFKA_RLMM_CONFIG_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR: "2" + # Remote Storage Manager (Aiven TS plugin) + KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/opt/tiered-storage-plugin/core/*:/opt/tiered-storage-plugin/s3/*" + KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager" + KAFKA_RSM_CONFIG_CHUNK_SIZE: "4194304" + KAFKA_RSM_CONFIG_STORAGE_BACKEND_CLASS: "io.aiven.kafka.tieredstorage.storage.s3.S3Storage" + KAFKA_RSM_CONFIG_STORAGE_S3_ENDPOINT_URL: "http://storage:9000" + KAFKA_RSM_CONFIG_STORAGE_S3_BUCKET_NAME: "inkless" + KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage/" + KAFKA_RSM_CONFIG_STORAGE_S3_REGION: "us-east-1" + KAFKA_RSM_CONFIG_STORAGE_S3_PATH_STYLE_ACCESS_ENABLED: "true" + KAFKA_RSM_CONFIG_STORAGE_AWS_ACCESS_KEY_ID: "minioadmin" + KAFKA_RSM_CONFIG_STORAGE_AWS_SECRET_ACCESS_KEY: "minioadmin" + + # --- Migration bridge --- + KAFKA_DISKLESS_ALLOW_FROM_CLASSIC_ENABLE: "true" + KAFKA_CLASSIC_REMOTE_STORAGE_FORCE_ENABLE: "true" + # Topics are classic by default; diskless only when explicitly switched + KAFKA_LOG_DISKLESS_ENABLE: "false" + + # --- Managed replicas (RF=2, rack-aware) --- + KAFKA_DISKLESS_MANAGED_RF_ENABLE: "true" + KAFKA_DEFAULT_REPLICATION_FACTOR: "2" + KAFKA_BROKER_RACK: "az1" + + # --- Faster segment rolling + retention for testing --- + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: "10000" + KAFKA_LOG_SEGMENT_BYTES: "1048576" + + broker2: + environment: + <<: *ts-unification-env + KAFKA_BROKER_RACK: "az2" + diff --git a/docker/inkless/Dockerfile b/docker/inkless/Dockerfile index 41ef8bf5644..4b259012d11 100644 --- a/docker/inkless/Dockerfile +++ b/docker/inkless/Dockerfile @@ -75,6 +75,25 @@ RUN set -eux ; \ apk del wget gpg gpg-agent; \ apk cache clean; +# Aiven Tiered Storage plugin (classic remote storage support) +ARG ts_plugin_version=1.1.1 +ARG ts_plugin_base_url=https://github.com/Aiven-Open/tiered-storage-for-apache-kafka/releases/download/v${ts_plugin_version} + +ADD ${ts_plugin_base_url}/core-${ts_plugin_version}.tgz /tmp/core.tgz +ADD ${ts_plugin_base_url}/s3-${ts_plugin_version}.tgz /tmp/s3.tgz +ADD ${ts_plugin_base_url}/gcs-${ts_plugin_version}.tgz /tmp/gcs.tgz +ADD ${ts_plugin_base_url}/azure-${ts_plugin_version}.tgz /tmp/azure.tgz + +RUN mkdir -p /opt/tiered-storage-plugin/core \ + /opt/tiered-storage-plugin/s3 \ + /opt/tiered-storage-plugin/gcs \ + /opt/tiered-storage-plugin/azure && \ + tar xzf /tmp/core.tgz -C /opt/tiered-storage-plugin/core --strip-components=1 && \ + tar xzf /tmp/s3.tgz -C /opt/tiered-storage-plugin/s3 --strip-components=1 && \ + tar xzf /tmp/gcs.tgz -C /opt/tiered-storage-plugin/gcs --strip-components=1 && \ + tar xzf /tmp/azure.tgz -C /opt/tiered-storage-plugin/azure --strip-components=1 && \ + rm /tmp/*.tgz + COPY --from=build-jsa kafka.jsa /opt/kafka/kafka.jsa COPY --from=build-jsa storage.jsa /opt/kafka/storage.jsa COPY --chown=appuser:appuser resources/common-scripts /etc/kafka/docker