[{"_1":2,"_189":-5,"_190":-5},"loaderData",{"_3":4,"_17":18,"_25":26,"_167":168},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","routes/_layout",{"_19":-5,"_20":21,"_22":21,"_23":11,"_24":21},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":89,"_110":111,"_156":-5,"_157":11,"_158":21,"_159":11,"_160":-5,"_161":11,"_162":-5,"_163":164,"_165":112,"_166":-7},"canonicalUrl","https://hub.docker.com/r/spoud/kafka-datagen/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","spoud","reponame","kafka-datagen","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":74,"_79":11,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2024-11-17T17:38:04.350147Z","description","","is_private","last_updated","2026-10-01T05:16:13.306974Z","last_modified","2026-10-01T05:15:29.900232Z","media_types",[56],"application/vnd.oci.image.index.v1+json","name","pull_count",1763,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# kafka-datagen\n\nThis is a reimplementation of the [kafka-datagen](https://github.com/confluentinc/kafka-connect-datagen) project in Quarkus.\n\nThe same data streams can be generated with much lower startup time and memory footprint.\n\n## Container quick start\n\nRun the published image with the minimum required settings:\n\n```bash\ndocker run --rm \\\n -e KAFKA_BOOTSTRAP_SERVERS=broker:9092 \\\n -e TOPIC=topic \\\n -e SCHEMA_FILE=clickstream_users_schema.avro \\\n -e SCHEMA_KEYFIELD=username \\\n spoud/kafka-datagen:latest\n```\n\nThe container:\n\n- listens on port `8080`\n- runs as a non-root user (`185:0`)\n- reads schemas from the image classpath or from a mounted file path\n- exposes metrics on `/q/metrics`\n- needs outbound access to Kafka and, for Avro production, Schema Registry\n\n## What an agent or operator needs to know\n\nIf you want an agent to deploy or operate this image safely, these are the facts it needs up front:\n\n1. Kafka connectivity\n - bootstrap servers\n - whether TLS is required\n - whether SASL is required\n - where credentials come from\n\n2. Data contract\n - topic name\n - schema file name or mounted schema path\n - output format (`json` or `avro`)\n - schema key field, if any\n\n3. Runtime behavior\n - target rate\n - whether generation is bounded with `MAX_RECORDS`\n - whether late events, duplicates, or poison pills are enabled\n\n4. Infrastructure assumptions\n - whether the platform allows a writable `/tmp`\n - whether the platform allows outbound access to Kafka and Schema Registry\n - CPU and memory limits\n\n5. Security posture\n - whether `/q/metrics` should be internal only\n - how secrets are injected\n - whether the deployment should use a read-only root filesystem\n\nThe repository does **not** know your Kafka auth model, your schema distribution approach, or your production network boundaries. An agent still needs those deployment-specific inputs.\n\n## Runtime contract\n\n### Configuration\n\nEnvironment variables are mapped to Quarkus config using the usual MicroProfile naming rules.\n\nCore settings:\n\n- `KAFKA_BOOTSTRAP_SERVERS`: The Kafka bootstrap servers to connect to. Default: `localhost:9092`\n- `TOPIC`: The Kafka topic to produce to. Default: `test`\n- `SCHEMA_FILE`: The Avro schema file to use. Default: `clickstream_users_schema.avro`\n- `SCHEMA_KEYFIELD`: The field in the Avro schema to use as the key. Default: `null`\n- `RATE`: The number of records to produce per second. Default: `10`\n- `MAX_RECORDS`: The maximum number of records to produce. Default: `0` (unlimited)\n- `FORMAT`: `json` or `avro`. Default: `json`\n- `SCHEMA_REGISTRY_URL`: The URL of the Confluent Schema Registry\n\nLate-event settings:\n\n- `LATE_EVENTS_PERCENTAGE`: Percentage of generated records whose event timestamp should be shifted into the past. Decimal values such as `12.5` are supported. Default: `0`\n- `LATE_EVENTS_MIN_MS`: Minimum lateness in milliseconds. Default: `0`\n- `LATE_EVENTS_MAX_MS`: Maximum lateness in milliseconds. Default: `0`\n- `LOG_LATE_EVENT`: If set to `true`, log late-event details without requiring special Quarkus logger-category configuration. Default: `false`\n\nPoison-pill settings:\n\n- `POISON_PILL_ENABLED`: Enable poison-pill injection. Default: `false`\n- `POISON_PILL_COUNT`: Number of poison pills to inject. Use `-1` to repeat forever. Default: `-1`\n- `POISON_PILL_INTERVAL`: Delay between poison pills as an ISO-8601 duration. Default: `PT5M`\n- `POISON_PILL_CLEAN_RECORDS`: Minimum number of normal records to emit before poison pills are allowed. Default: `0`\n- `POISON_PILL_CLEAN_DURATION`: Minimum startup time before poison pills are allowed, as an ISO-8601 duration. Default: `PT0S`\n- `POISON_PILL_TYPE`: Supported values: `invalid-string`, `random-bytes`. Default: `invalid-string`\n- `LOG_POISON_PILL`: If set to `true`, log poison-pill details without requiring special Quarkus logger-category configuration. Default: `false`\n\nDuplicate-event settings:\n\n- `DUPLICATE_EVENT_ENABLED`: Enable duplicate-event replay. Default: `false`\n- `DUPLICATE_EVENT_COUNT`: Number of duplicate events to inject. Use `-1` to repeat forever. Default: `-1`\n- `DUPLICATE_EVENT_INTERVAL`: Delay between duplicate events as an ISO-8601 duration. Default: `PT5M`\n- `DUPLICATE_EVENT_CLEAN_RECORDS`: Minimum number of normal records to emit before duplicates are allowed. Default: `0`\n- `DUPLICATE_EVENT_CLEAN_DURATION`: Minimum startup time before duplicates are allowed, as an ISO-8601 duration. Default: `PT0S`\n- `LOG_DUPLICATE_EVENT`: If set to `true`, log duplicate-event details without requiring special Quarkus logger-category configuration. Default: `false`\n\nFeature-specific headers are configured with these property prefixes:\n\n```properties\nlate.events.headers.=\npoison.pill.headers.=\nduplicate.event.headers.=\n```\n\nImportant behavior notes:\n\n- When using the Avro format, configure the Kafka value serializer accordingly:\n\n ```properties\n mp.messaging.outgoing.generated.value.serializer=io.confluent.kafka.serializers.KafkaAvroSerializer\n ```\n\n- The JSON generator still requires `SCHEMA_FILE`, because the schema drives field generation.\n- Late events require exactly one timestamp field in the Avro schema. The generator detects Avro logical timestamp fields (`timestamp-millis` or `timestamp-micros`) and fails fast when late events are enabled but the schema has none or more than one matching field.\n- `late.events.percentage` accepts decimal values in the range `0..100`.\n- Duplicate events replay an earlier generated event with the original key and payload. Only headers are overridden for the duplicate record.\n- Poison pills are emitted on a dedicated byte-oriented Kafka producer path so the payload can intentionally violate the normal topic contract.\n\n### Schema lookup\n\n`SCHEMA_FILE` can point to:\n\n- a schema bundled in `src/main/resources`\n- a path mounted into the container filesystem\n\nIf you mount your own schemas, make the path explicit:\n\n```bash\ndocker run --rm \\\n -v \"$PWD/schemas:/schemas:ro\" \\\n -e KAFKA_BOOTSTRAP_SERVERS=broker:9092 \\\n -e TOPIC=topic \\\n -e SCHEMA_FILE=/schemas/payroll_bonus.avro \\\n spoud/kafka-datagen:latest\n```\n\n### Observability\n\nThe application exposes:\n\n- metrics on `http://:8080/q/metrics`\n\nThere is currently no built-in authentication layer around that endpoint, so production deployments should normally expose it only on an internal network or through a secured ingress path.\n\n## Container hardening and supply chain\n\nThe published JVM image now:\n\n- runs as non-root (`185:0`)\n- keeps application files owned by the runtime user and group-accessible for OpenShift-style execution\n- removes world write access from `/deployments`\n- uses `/tmp` as the JVM temp directory\n\nRecommended runtime hardening:\n\n- run with a read-only root filesystem\n- mount a writable `emptyDir` or tmpfs at `/tmp`\n- drop all Linux capabilities\n- disallow privilege escalation\n- use the default runtime seccomp profile\n- avoid exposing `/q/metrics` publicly\n\nThe GitHub Actions image build publishes Docker BuildKit supply-chain metadata for pushed images:\n\n- SBOM\n- provenance attestation\n\n## Secure deployment example\n\nThe example below assumes:\n\n- non-root execution\n- read-only root filesystem\n- writable `/tmp`\n- secrets injected from Kubernetes\n- metrics exposed only inside the cluster\n\n```yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: kafka-datagen\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: kafka-datagen\n template:\n metadata:\n labels:\n app: kafka-datagen\n spec:\n containers:\n - name: kafka-datagen\n image: spoud/kafka-datagen:latest\n imagePullPolicy: IfNotPresent\n ports:\n - containerPort: 8080\n name: http\n env:\n - name: KAFKA_BOOTSTRAP_SERVERS\n value: kafka-bootstrap.kafka.svc.cluster.local:9092\n - name: TOPIC\n value: payroll-bonus\n - name: SCHEMA_FILE\n value: /schemas/payroll_bonus.avro\n - name: FORMAT\n value: json\n - name: RATE\n value: \"10\"\n - name: MAX_RECORDS\n value: \"0\"\n - name: QUARKUS_HTTP_PORT\n value: \"8080\"\n - name: MP_MESSAGING_OUTGOING_GENERATED_SECURITY_PROTOCOL\n value: SASL_SSL\n - name: MP_MESSAGING_OUTGOING_GENERATED_SASL_MECHANISM\n value: SCRAM-SHA-512\n - name: MP_MESSAGING_OUTGOING_GENERATED_SASL_JAAS_CONFIG\n valueFrom:\n secretKeyRef:\n name: kafka-datagen-kafka-auth\n key: sasl-jaas-config\n volumeMounts:\n - name: schemas\n mountPath: /schemas\n readOnly: true\n - name: tmp\n mountPath: /tmp\n securityContext:\n runAsNonRoot: true\n allowPrivilegeEscalation: false\n readOnlyRootFilesystem: true\n capabilities:\n drop: [\"ALL\"]\n seccompProfile:\n type: RuntimeDefault\n resources:\n requests:\n cpu: 100m\n memory: 256Mi\n limits:\n cpu: 500m\n memory: 512Mi\n volumes:\n - name: schemas\n configMap:\n name: kafka-datagen-schemas\n - name: tmp\n emptyDir: {}\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: kafka-datagen\nspec:\n selector:\n app: kafka-datagen\n ports:\n - name: http\n port: 8080\n targetPort: http\n```\n\nAdapt the Kafka security properties to your environment. For Avro output, also set the serializer and Schema Registry properties on the `generated` outgoing channel.\n\n## Local demo stack\n\nThe provided `docker-compose.yml` starts:\n\n- Redpanda\n- Redpanda Console\n- Prometheus\n- `kafka-datagen`\n\nStart it with:\n\n```bash\ndocker compose up --build\n```\n\n## Quarkus Project\n\nThis project uses Quarkus, the Supersonic Subatomic Java Framework.\n\nIf you want to learn more about Quarkus, please visit its website: .\n\n### Running the application in dev mode\n\nYou can run your application in dev mode that enables live coding using:\n\n```shell\n./mvnw compile quarkus:dev\n```\n\n> **NOTE:** Quarkus now ships with a Dev UI, which is available in dev mode only at .\n\n### Packaging and running the application\n\nThe application can be packaged using:\n\n```shell\n./mvnw package\n```\n\nIt produces the `quarkus-run.jar` file in the `target/quarkus-app/` directory.\n\nThe application is then runnable using:\n\n```shell\njava -jar target/quarkus-app/quarkus-run.jar\n```\n\nIf you want to build an _über-jar_, execute:\n\n```shell\n./mvnw package -Dquarkus.package.jar.type=uber-jar\n```\n\n### Creating a native executable\n\nYou can create a native executable using:\n\n```shell\n./mvnw package -Dnative\n```\n\nOr, if you do not have GraalVM installed:\n\n```shell\n./mvnw package -Dnative -Dquarkus.native.container-build=true\n```\n\nYou can then execute your native executable with:\n\n```shell\n./target/kafka-datagen-1.0.0-SNAPSHOT-runner\n```\n\nIf you want to learn more about building native executables, please consult .\n\n### Related Guides\n\n- Apache Kafka Client ([guide](https://quarkus.io/guides/kafka)): Connect to Apache Kafka with its native API\n- Messaging - Kafka Connector ([guide](https://quarkus.io/guides/kafka-getting-started)): Connect to Kafka with Reactive Messaging\n- Confluent Schema Registry - Avro ([guide](https://quarkus.io/guides/kafka-schema-registry-avro)): Use Confluent as Avro schema registry\n","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":21,"_83":11,"_84":11},"read","write","admin","user","storage_size",2599334888,"orgData",{"_90":91,"_92":93,"_94":95,"_96":33,"_97":48,"_98":99,"_100":101,"_102":103,"_104":105,"_106":48,"_107":48,"_108":48,"_109":21},"id","5576ab487371493e963ae878bbd894a4","uuid","5576ab48-7371-493e-963a-e878bbd894a4","type","organization","orgname","full_name","location","Bern","company","SPOUD","profile_url","https://spoud.io","date_joined","2017-06-16T09:20:06.376623Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[112],{"_113":44,"_114":115,"_116":117,"_118":119,"_90":120,"_121":122,"_50":144,"_145":115,"_146":147,"_148":56,"_57":149,"_150":151,"_152":129,"_153":144,"_154":66,"_155":21},"content_type","creator",1924715,"digest","sha256:8830293d70b5af2e02748d3cb21501d2f2fae20748ee323f4dddb4fc211befd0","full_size",219718337,796199855,"images",[123,138],{"_124":125,"_116":126,"_127":48,"_128":129,"_130":131,"_132":133,"_134":48,"_135":-5,"_136":119,"_63":66,"_137":-5},"architecture","amd64","sha256:e5d9885eaf107d7d450ce1775127d9d7dbbacc825f0762854f46a6cd7045313d","features","last_pulled","2026-10-09T10:20:12.876730929Z","last_pushed","2026-10-01T05:16:10.730016858Z","os","linux","os_features","os_version","size","variant",{"_124":139,"_116":140,"_127":48,"_128":141,"_130":142,"_132":133,"_134":48,"_135":-5,"_136":143,"_63":66,"_137":-5},"arm64","sha256:8986ed200356db28402db46c54c0719367241a1fb90cbc92f3dc7db3539fae11","2026-10-09T10:20:12.858549341Z","2026-10-01T05:16:11.157042128Z",217155771,"2026-10-01T05:16:12.587732Z","last_updater","last_updater_username","spoudbot","media_type","latest","repository",25690471,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":169,"_176":177,"_163":187,"_188":11},{"_40":170,"_42":171,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":172,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":173,"_79":11,"_80":175,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":174},[78],{"_82":21,"_83":11,"_84":11},"tags",{"_178":64,"_179":-5,"_180":-5,"_181":182},"count","next","previous","results",[183],{"_113":44,"_114":115,"_116":117,"_118":119,"_90":120,"_121":184,"_50":144,"_145":115,"_146":147,"_148":56,"_57":149,"_150":151,"_152":129,"_153":144,"_154":66,"_155":21},[185,186],{"_124":125,"_116":126,"_127":48,"_128":129,"_130":131,"_132":133,"_134":48,"_135":-5,"_136":119,"_63":66,"_137":-5},{"_124":139,"_116":140,"_127":48,"_128":141,"_130":142,"_132":133,"_134":48,"_135":-5,"_136":143,"_63":66,"_137":-5},[],"canDeleteTags","actionData","errors"]

Digest	OS/ARCH	Compressed size
e5d9885eaf10	linux/amd64	209.54 MB
8986ed200356	linux/arm64	207.1 MB

spoud/kafka-datagen

Tags for spoud/kafka-datagen