Actor stats not being sent to StatsD from Kamon - akka-http

Question

I'm trying to use Kamon with StatsD backend for Akka monitoring/telemetry. I currently only have one actor and I'm sending an ask which replies to the caller. I can see my custom counter data in Graphana and Graphite as well as JVM and OS stats, but no actor stats from Akka. My project is setup just like this intro but I have added the Kamon stuff and my own HttpService. What am I missing?
Here's my config:
kamon {
trace.level = simple-trace
metric {
tick-interval = 1 second
filters {
akka-actor.includes = [ "**" ]
akka-router.includes = [ "**" ]
akka-dispatcher.includes = [ "**" ]
trace.includes = [ "**" ]
trace-segment.includes = [ "**" ]
histogram.includes = [ "**" ]
min-max-counter.includes = [ "**" ]
gauge.includes = [ "**" ]
counters.includes = [ "**" ]
# For web projects
http-server.includes = [ "**" ]
akka-actor.excludes = [ "**" ]
akka-router.excludes = [ "**" ]
akka-dispatcher.excludes = [ "**" ]
counters.excludes = [ "" ]
trace-segment.excludes = [ "" ]
histogram.excludes = [ "" ]
min-max-counter.excludes = [ "" ]
gauge.excludes = [ "" ]
http-server.excludes = [ "" ]
}
}
akka {
ask-pattern-timeout-warning = "lightweight"
}
statsd {
hostname = "localhost" //Graphite Host
port = "8125"
flush-interval = 1 second
max-packet-size = 1024 bytes
subscriptions {
histogram = ["**"]
min-max-counter = ["**"]
gauge = ["**"]
counter = ["**"]
trace = ["**"]
trace-segment = ["**"]
system-metric = ["**"]
akka-actor = ["**"]
akka-dispatcher = ["**"]
http-server = ["**"]
}
report-system-metrics = true
simple-metric-key-generator {
application = "data-id-generator"
}
}
system-metrics {
#sigar is enabled by default
sigar-enabled = false
#jmx related metrics are enabled by default
jmx-enabled = true
}
}
Here's my service:
import akka.event.Logging
import akka.http.scaladsl.server.Directives._
import akka.http.scaladsl.server.Route
import akka.pattern.ask
import akka.util.{ByteString, Timeout}
import com.incontact.data.GeneratorActor.GenerateId
import com.incontact.data.IdGenerator.{GeneratorError, IdResult}
import com.incontact.http.HttpService
import akka.http.scaladsl.model.StatusCodes._
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport
import spray.json._
import kamon.Kamon
import scala.concurrent.duration._
trait IdService extends HttpService with SprayJsonSupport with DefaultJsonProtocol {
private lazy val log = Logging(system, classOf[IdService])
implicit val timeout: Timeout = 1.second
//implicit val errorFormat = jsonFormat2(GeneratorError)
abstract override def route: Route = {
log.info("creating generator actor")
val generator = system.actorOf(GeneratorActor.props, "generator")
val counter = Kamon.metrics.counter("generate-long-id")
path("") {
get {
counter.increment()
log.info("IdService sending single request to actor")
onSuccess((generator ? GenerateId).mapTo[IdResult]) {
case Right(id) => complete(ByteString(s"${id}\n"))
case Left(e:GeneratorError) => failWith(e.Error)
}
}
}
} ~ super.route
}
Main.scala (startup)
import akka.event.Logging
import com.incontact.data.IdService
import com.incontact.http.HttpServer
import kamon.Kamon
import scala.io.StdIn
import scala.util.Success
object Main extends App
with HttpServer
with IdService {
Kamon.start()
private lazy val log = Logging(system, getClass)
bindingFuture.andThen {
case Success(binding) =>
// this really doesn't support any production behavior since [ENTER] won't be used to shut down the docker container
log.info("Press ENTER to exit")
StdIn.readLine()
log.info("Shutting down...")
Kamon.shutdown()
system.terminate()
}
}
build.sbt
scalaVersion := "2.11.8"
val akkaVersion = "2.4.14"
val kamonVersion = "0.6.3"
libraryDependencies ++= Seq(
// remove multiple-dependency warnings
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
"org.scala-lang.modules" %% "scala-xml" % "1.0.5",
// application dependencies
"com.typesafe.akka" %% "akka-actor" % akkaVersion,
"com.typesafe.akka" %% "akka-http" % "10.0.0",
"com.typesafe.akka" %% "akka-http-spray-json" % "10.0.0",
"com.amazonaws" % "aws-java-sdk-dynamodb" % "1.11.63",
// kamon dependencies
"io.kamon" %% "kamon-core" % kamonVersion,
"io.kamon" %% "kamon-statsd" % kamonVersion,
"io.kamon" %% "kamon-system-metrics" % kamonVersion,
"org.aspectj" % "aspectjweaver" % "1.8.6",
// test dependencies
"com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
"org.scalatest" %% "scalatest" % "3.0.1" % "test",
"com.typesafe.akka" %% "akka-http-testkit" % "10.0.0" % "test",
"org.pegdown" % "pegdown" % "1.1.0" % "test"
)
project/plugins.sbt
addSbtPlugin("io.kamon" % "aspectj-runner" % "0.1.3")
launch from command-line:
$ sbt aspectj-runner:run

Answer 1

It looks like you're excluding all the actors/dispatcher/router metrics in your config. This should rule all metrics out (even if you're also including them all).
akka-actor.excludes = [ "**" ]
akka-router.excludes = [ "**" ]
akka-dispatcher.excludes = [ "**" ]
Try and change it to
akka-actor.excludes = [ ]
akka-router.excludes = [ ]
akka-dispatcher.excludes = [ ]