Kotlin Ktor client 403 forbidden cloudflare - cloudflare

Question

So im trying to request a side with proxies that is protected with cloudflare. The problem is i get 403 forbidden cloduflare error but only when im using proxies without it works. But the proxies are not the problem i tried them with python(requests module) and in my browser there i dont get blocked. My code
suspend fun scrape() {
val client = HttpClient {
followRedirects = true
install(ContentNegotiation) {
json(Json {
ignoreUnknownKeys = true
})
}
engine {
proxy =
ProxyBuilder.http("http://ProxyIP:proxyPort")
}
defaultRequest {
val credentials = Base64.getEncoder().encodeToString("ProxyUser:ProxyPassword".toByteArray())
header(HttpHeaders.ProxyAuthorization, "Basic $credentials")
}
}
val response = client.get("http://example.com")
val body = response.bodyAsText()
println(body)
println(response.status.hashCode())
Fixxed it
suspend fun scrape() {
val client = HttpClient(Apache) {
install(ContentNegotiation) {
json(Json {
ignoreUnknownKeys = true
})
}
engine {
followRedirects = false
customizeClient {
setProxy(HttpHost("hostname", port))
val credentialsProvider = BasicCredentialsProvider()
credentialsProvider .setCredentials(
AuthScope("hostname", port),
UsernamePasswordCredentials("username", "password")
)
setDefaultCredentialsProvider(credentialsProvider )
}
}
}
val response =
client.get("http://example.com") {
}
val body = response.bodyAsText()
println(body)
println(response.status.hashCode())
}

Answer 1

There is a problem with making a request through a proxy server using the CIO engine. I've created an issue to address this problem. As a workaround, please use the OkHttp engine instead of CIO.
Here is how you can use a proxy with the Basic authentication:
import io.ktor.client.*
import io.ktor.client.engine.okhttp.*
import io.ktor.client.request.*
import kotlinx.coroutines.runBlocking
import okhttp3.Authenticator
import okhttp3.Credentials
import okhttp3.OkHttpClient
import java.net.Proxy
fun main(): Unit = runBlocking {
val proxyAuthenticator = Authenticator { _, response ->
response.request.newBuilder()
.header("Proxy-Authorization", Credentials.basic("<username>", "<password>"))
.build()
}
val client = HttpClient(OkHttp) {
engine {
preconfigured = OkHttpClient.Builder()
.proxy(Proxy(Proxy.Type.HTTP, java.net.InetSocketAddress("127.0.0.1", 3128)))
.proxyAuthenticator(proxyAuthenticator)
.build()
}
}
val response = client.get("http://eu.kith.com/products.json")
println(response.status)
}