kafka Schema not found with schema registry - kotlin

Question

i'm studying kafka with schema registry.
i create schema with topic users-value to schema registry directly.
and than i use avro plugin(davidmc24) to convert.
servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is org.apache.kafka.common.errors.SerializationException: Error retrieving Avro schema{"type":"record","name":"User","namespace":"io.confluent.developer","fields":[{"name":"name","type":{"type":"string","avro.java.string":"String"},"avro.java.string":"String"},{"name":"age","type":"int"}]}] with root cause error occured.
i don't know what is the prolem.
can anyone help me please?
below is my code.
schema
it is also in src/main/avro/sample.avsc
{
"namespace": "io.confluent.developer",
"type": "record",
"name": "User",
"fields": [
{
"name": "name",
"type": "string",
"avro.java.string": "String"
},
{
"name": "age",
"type": "int"
}
]
}
application
#SpringBootApplication()
class Application
fun main(args: Array<String>) {
runApplication<Application>(*args)
}
consumer
#Service
class Consumer(
private val logger: Logger = LoggerFactory.getLogger(Producer::class.java)
) {
#KafkaListener(topics = ["users"], groupId = "group_id")
#Throws(IOException::class)
fun consume(message: ConsumerRecord<String, User>?) {
if (message != null) {
logger.info(String.format("#### -> Consumed message -> %s", message.value()))
}
}
}
controller
#RestController
#RequestMapping(value = ["/kafka"])
class KafkaController #Autowired internal constructor(private val producer: Producer) {
#PostMapping(value = ["/publish"])
fun sendMessageToKafkaTopic(#RequestParam("name") name: String, #RequestParam("age") age: Int) {
producer.sendMessage(User(name, age))
}
}
producer
#Service
class Producer (
private val kafkaTemplate: KafkaTemplate<String, User>? = null,
#Value("\${topic.name}")
private val TOPIC: String? = null
) {
fun sendMessage(user: User) {
this.kafkaTemplate?.send(this.TOPIC!!, user.getName(), user)
logger.info(String.format("Produced user -> %s", user))
}
companion object {
private val logger: Logger = LoggerFactory.getLogger(Producer::class.java)
}
}
application.yml
server:
port: 9000
spring:
kafka:
bootstrap-servers:
- localhost:9092
properties:
schema.registry.url: http://localhost:8085
auto:
register:
schemas: false
consumer:
group-id: group_id
auto-offset-reset: earliest
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
value-deserializer: io.confluent.kafka.serializers.KafkaAvroDeserializer
producer:
key-serializer: org.apache.kafka.common.serialization.StringSerializer
value-serializer: io.confluent.kafka.serializers.KafkaAvroSerializer
topic:
name: users
build.gradle
buildscript {
repositories {
maven {
url "https://plugins.gradle.org/m2/"
}
maven {
url "https://packages.confluent.io/maven/"
}
maven {
url = uri("https://jitpack.io")
}
}
}
plugins {
id "com.github.davidmc24.gradle.plugin.avro" version "1.2.0"
id "com.github.imflog.kafka-schema-registry-gradle-plugin" version "1.6.0"
}
apply plugin: 'kotlin-jpa'
apply plugin: 'org.springframework.boot'
apply plugin: 'kotlin-allopen'
apply plugin: "com.github.imflog.kafka-schema-registry-gradle-plugin"
group = 'com.example.exercise'
version = '0.0.1-alpha.kafka'
configurations {
developmentOnly
runtimeClasspath {
extendsFrom developmentOnly
}
}
processResources {
def props = ['version': project.properties.get('version')]
expand(props)
}
allOpen {
annotation 'javax.persistence.Entity'
}
ext {
avroVersion = "1.11.0"
}
dependencies {
implementation 'org.springframework.boot:spring-boot-starter-validation'
implementation 'org.springframework.kafka:spring-kafka:2.8.5'
implementation "org.apache.avro:avro:${avroVersion}"
implementation 'io.confluent:kafka-avro-serializer:7.1.1'
implementation 'org.springframework.boot:spring-boot-starter-web'
developmentOnly 'org.springframework.boot:spring-boot-devtools'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
}
schemaRegistry {
url = 'http://registry-url:8085/'
quiet = true
}
docker-compose.yml
version: '2'
services:
kafka:
image: wurstmeister/kafka
container_name: kafka-exercise
ports:
- "9092:9092"
environment:
- KAFKA_ADVERTISED_HOST_NAME=127.0.0.1
- KAFKA_ADVERTISED_PORT=9092
- KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
- KAFKA_LISTENERS=PLAINTEXT://:29092,EXTERNAL://:9092
- KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:29092,EXTERNAL://localhost:9092
- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
- KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
- KAFKA_SCHEMA_REGISTRY_URL=schemaregistry:8085
depends_on:
- zookeeper
zookeeper:
image: wurstmeister/zookeeper
container_name: zookeeper-exercise
ports:
- "2181:2181"
environment:
- KAFKA_ADVERTISED_HOST_NAME=zookeeper
schemaregistry:
image: confluentinc/cp-schema-registry:6.2.0
restart: always
container_name: registry-exercise
depends_on:
- zookeeper
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: "zookeeper:2181"
SCHEMA_REGISTRY_HOST_NAME: schemaregistry
SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8085"
ports:
- 8085:8085
kafka-ui:
image: provectuslabs/kafka-ui
container_name: kafka-ui-exercise
depends_on:
- kafka
- zookeeper
- schemaregistry
ports:
- "8083:8080"
restart: always
environment:
- KAFKA_CLUSTERS_0_NAME=local
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:29092
- KAFKA_CLUSTERS_0_ZOOKEEPER=zookeeper:2181
- KAFKA_CLUSTERS_0_SCHEMAREGISTRY=http://schemaregistry:8085
this is whole code that i use.
ps. if i comment out auto.register.schemas: false part, it run but creating new schema.
how to make it work, without commenting out auto.register.schemas: false?

Answer 1

i changed avro file.
the problem is avro.
{
"type": "record",
"name": "User",
"namespace": "io.confluent.developer",
"fields": [
{
"name": "name",
"type": {
"type": "string",
"avro.java.string": "String"
},
"avro.java.string": "String"
},
{
"name": "age",
"type": "int"
}
]
}
now it works!