RxJava different output between Flowable and Observable with Window and Groupby - kotlin

Question

I'm using RxJava2 with code that boils down to something like this:
val whitespaceRegex = Regex("\\s+")
val queryRegex = Regex("query=([^&]+)", RegexOption.IGNORE_CASE)
val dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME
#JvmStatic
fun main(args: Array<String>) {
val cnt = AtomicLong()
val templateStr = "|date| /ignored/ query=|query|"
val random = ThreadLocalRandom.current()
var curDate = ZonedDateTime.of(LocalDate.of(2016, Month.JANUARY, 1), LocalTime.MIDNIGHT, ZoneId.of("UTC"))
val generator = Flowable.generate<String> { emitter ->
// normally these are read from a file, this is for the example
val next = cnt.incrementAndGet()
if (next % 3000 == 0L) {
curDate = curDate.plusDays(1)
}
if (next < 100000) {
val curStr = templateStr
.replace("|date|", dateTimeFormatter.format(curDate))
.replace("|query|", random.nextInt(1, 1000).toString())
emitter.onNext(curStr)
} else {
emitter.onComplete()
}
}
val source = generator
.map { line ->
val cols = line.split(whitespaceRegex)
val queryRaw = queryRegex.find(cols[2])?.groupValues?.get(1) ?: ""
val query = URLDecoder.decode(queryRaw, Charsets.UTF_8.name()).toLowerCase().replace(whitespaceRegex, " ").trim()
val date = dateTimeFormatter.parse(cols[0])
Pair(LocalDate.from(date), query)
}
.share()
source
.window(source.map { it.first }.distinctUntilChanged())
.flatMap { window ->
window
.groupBy { pair -> pair }
.flatMap({ grouping ->
grouping
.count()
.map {
Pair(grouping.key, it)
}.toFlowable()
})
}
.subscribe({ println("Result: $it}") }, { it.printStackTrace() }, { println("Done") })
}
When I use Observable.generate it works fine, but with Flowable.generate there is no output. This is counting how many queries occurred on a given day. The day increase sequentially so I form a window of each day, then count the queries with a groupBy. Do I need to do this differently with Flowable?

Answer 1

As akarnokd mentioned, this was due to flatMap having a default maxConcurrency of 128. I found this issue, https://github.com/ReactiveX/RxJava/issues/5126, which describes the reason in more detail. This fixes the problem:
val cnt = AtomicLong()
val templateStr = "|date| /ignored/ query=|query|"
val random = ThreadLocalRandom.current()
var curDate = ZonedDateTime.of(LocalDate.of(2016, Month.JANUARY, 1), LocalTime.MIDNIGHT, ZoneId.of("UTC"))
val generator = Flowable.generate<String> { emitter ->
val next = cnt.incrementAndGet()
if (next % 3000 == 0L) {
curDate = curDate.plusDays(1)
}
if (next < 1000000) {
val curStr = templateStr
.replace("|date|", dateTimeFormatter.format(curDate))
.replace("|query|", random.nextInt(1, 1000).toString())
emitter.onNext(curStr)
} else {
emitter.onComplete()
}
}
val source = generator
.map { line ->
val cols = line.split(whitespaceRegex)
val queryRaw = queryRegex.find(cols[2])?.groupValues?.get(1) ?: ""
val query = URLDecoder.decode(queryRaw, Charsets.UTF_8.name()).toLowerCase().replace(whitespaceRegex, " ").trim()
val date = dateTimeFormatter.parse(cols[0])
Pair(LocalDate.from(date), query)
}
.share()
source
.window(source.map { it.first }.distinctUntilChanged().doOnEach({println("Win: $it")}))
.flatMap( { window ->
window
.groupBy { pair -> pair }
.flatMap({ grouping ->
grouping
.count()
.map {
Pair(grouping.key, it)
}.toFlowable()
// fix is here
}, Int.MAX_VALUE)
// and here
}, Int.MAX_VALUE)
.subscribe({ println("Result: $it}") }, { it.printStackTrace() }, { println("Done") })