Im trying to use crawlera alongside splash local instance, this is my lua script
function main(splash)
function use_crawlera(splash)
local user = splash.args.crawlera_user
local host = 'proxy.crawlera.com'
local port = 8010
local session_header = 'X-Crawlera-Session'
local session_id = 'create'
splash:on_request(function(request)
request:set_header('X-Crawlera-Cookies', 'disable')
request:set_header(session_header, session_id)
request:set_proxy { host, port, username = user, password = '' }
end)
splash:on_response_headers(function(response)
if type(response.headers[session_header]) ~= nil then
session_id = response.headers[session_header]
end
end)
end
function main(splash)
use_crawlera(splash)
splash:go(splash.args.url)
splash:wait(30)
return splash:html()
end
end
and this is my start_request
yield SplashRequest(index_url,
self.parse_kawanlama_index,
endpoint='execute',
args={
'lua_source': lua_script,
'wait' : 5,
'html' : 1,
'url': index_url,
'timeout': 10,
'crawlera_user':self.crawlera_apikey
},
# tell Splash to cache the lua script, to avoid sending it for every request
cache_args=['lua_source'],
)
but it doesnt seems to work because response.body that i got in self.parse(response) is contains no html.
Related
I would like to call an external graphql API (without authentication for the moment).
Here is my code :
local open_api = "https://graphqlzero.almansi.me/api"
local payload = '{"query": "query { post(id: 1) { id title body }}"}'
local headers = {
}
local function craftCall()
local response
local data
pcall(function ()
response = HttpService:PostAsync(open_api, payload, Enum.HttpContentType.ApplicationJson, false, headers)
data = HttpService:JSONDecode(response)
end)
if not data then return false end
print(data)
return false
end
if craftCall() then
print("Success")
else
print("Something went wrong")
end
I get always something went wrong. I need some help on what is going wrong... Specially I don't know if am I correctly formatting the Payload.
After your http call, you never return a success result. You've only outlined failure cases :
if not data then return false end
print(data)
return false
So your conditional, if craftCall() then always evaluates to false.
Why not make it return true or data after the print(data)? Then you'll know that it made it to the end of the call successfully.
local function craftCall()
local success, result = pcall(function()
local response = HttpService:PostAsync(open_api, payload, Enum.HttpContentType.ApplicationJson, false, headers)
return HttpService:JSONDecode(response)
end)
if not success then
warn("PostAsync failed with error : ", result)
return false
end
-- return the parsed data
return result
end
I'm trying to make a Minecraft client and I cant figure out how to get a session ID to start the game. I've done some googling and cant find anyway to get it bar from this answer to Launch Minecraft from command line - username and password as prefix that doesn't work.
Minecraft mc = Minecraft.getMinecraft();
mc.getSession().getToken();
You can manually crash your game by holding down F3 + C. In the crash log will be your session id.
I made a little script that returns the session id.
def GetSessionID(Username, Password):
# Url = f'https://login.minecraft.net?user={Username}&password={Password}&version=13'
Url = "https://authserver.mojang.com/authenticate"
# LoginInfo = requests.post(Url)
# LoginInfoList = LoginInfo.split(':')
# SessionID = LoginInfoList[3]
token = str(uuid.uuid4())
requestData = GetAuthenticationBody(Username, Password, token)
response = requests.post(url=Url, json=requestData)
responseData = response.json()
SessionID = responseData['accessToken']
return SessionID
def GetAuthenticationBody(username, password, token):
body = {
"agent": {
"name": "Minecraft",
"version": 1
},
"username": username,
"password": password,
"clientToken": token,
"requestUser": True
}
return body
Is there way to access rest api with pure lua script
GET / POST both way need to access and display response
i already tried
local api = nil
local function iniit()
if api == nil then
-- body
api = require("http://api.com")
.create()
.on_get(function ()
return {name = "Apple",
id = 12345}
end)
end
end
In linux , mac we can easily install luarocks , and then we can install curl package. It's easiest way to unix like os.
-- HTTP Get
local curl = require('curl')
curl.easy{
url = 'api.xyz.net?a=data',
httpheader = {
"X-Test-Header1: Header-Data1",
"X-Test-Header2: Header-Data2",
},
writefunction = io.stderr -- use io.stderr:write()
}
:perform()
:close()
In windows i faced several problems. Cant install luarocks correctly. then luarock install command not work correctl, etc..
In first dwnload lua from official site, and then create structure like (below web site)
http://fuchen.github.io/dev/2013/08/24/install-luarocks-on-windows/
then i download lua luadist
http://luadist.org/
then i got same structure luadist extracted folder and lua folder.
merged luadist folder and lua folder
Finaly we can use http.soket
local http=require("socket.http");
local request_body = [[login=user&password=123]]
local response_body = {}
local res, code, response_headers = http.request{
url = "api.xyz.net?a=data",
method = "GET",
headers =
{
["Content-Type"] = "application/x-www-form-urlencoded";
["Content-Length"] = #request_body;
},
source = ltn12.source.string(request_body),
sink = ltn12.sink.table(response_body),
}
print(res)
print(code)
if type(response_headers) == "table" then
for k, v in pairs(response_headers) do
print(k, v)
end
end
print("Response body:")
if type(response_body) == "table" then
print(table.concat(response_body))
else
print("Not a table:", type(response_body))
end
IF YOU DO THESE STEPS CORRECTLY , THIS WILL BE WORK 1000% SURE
I'm developing a plugin to Kong API Gateway. I created a service pointing it to another service in the local network and basically every request to my service is redirected to the other one, so far so good.
What the plugin has to do is grab the field Authorization Bearer in the header, and pass to the upstream service as part of the URI. E.g.
Request is received on:
localhost/service
In its header, it have a Authorization Bearer that contains a JWT
The plugin has to receive it, take the JWT and parse it to URI to the upstream service:
productionServer/service/9a8udoadzlkndid813gru1gr <-JWT took from header
My attempt till now:
local singletons = require "kong.singletons"
local BasePlugin = require "kong.plugins.base_plugin"
local responses = require "kong.tools.responses"
local constants = require "kong.constants"
local multipart = require "multipart"
local cjson = require "cjson"
local url = require "socket.url"
local access = require "kong.plugins.ctk.access"
local CtkHandler = BasePlugin:extend()
CtkHandler.PRIORITY = 3505
CtkHandler.VERSION = "0.1.0"
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- JUST EXTENDED THE BASE PLUGIN ---")
function CtkHandler:new()
CtkHandler.super.new(self, "ctk")
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- INSTACIATED ITSELF ---")
end
function CtkHandler:access(conf)
CtkHandler.super.access(self)
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- STARTED THE ACCESS PART ---")
do_authentication()
access.execute(conf)
end
file:close()
return CtkHandler
The idea, is that after every request, the access block at the end be executed, then, he will redirect to my access file
local singletons = require "kong.singletons"
local BasePlugin = require "kong.plugins.base_plugin"
local responses = require "kong.tools.responses"
local constants = require "kong.constants"
local multipart = require "multipart"
local cjson = require "cjson"
local url = require "socket.url"
local basic_serializer = require "kong.plugins.log-serializers.basic"
local string_format = string.format
local ngx_set_header = ngx.req.set_header
local get_method = ngx.req.get_method
local req_set_uri_args = ngx.req.set_uri_args
local req_get_uri_args = ngx.req.get_uri_args
local req_set_header = ngx.req.set_header
local req_get_headers = ngx.req.get_headers
local req_clear_header = ngx.req.clear_header
local req_set_method = ngx.req.set_method
local ngx_decode_args = ngx.decode_args
local ngx_re_gmatch = ngx.re.gmatch
local string_format = string.format
local cjson_encode = cjson.encode
local ipairs = ipairs
local request = ngx.request
local function retrieve_token(request, conf)
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- RUNNING RETRIEVE TOKEN ---")
local uri_parameters = request.get_uri_args()
for _, v in ipairs(conf.uri_param_names) do
if uri_parameters[v] then
return uri_parameters[v]
end
end
local ngx_var = ngx.var
for _, v in ipairs(conf.cookie_names) do
local jwt_cookie = ngx_var["cookie_" .. v]
if jwt_cookie and jwt_cookie ~= "" then
return jwt_cookie
end
end
local authorization_header = request.get_headers()["authorization"]
if authorization_header then
local iterator, iter_err = ngx_re_gmatch(authorization_header, "\\s*[Bb]earer\\s+(.+)")
if not iterator then
return nil, iter_err
end
local m, err = iterator()
if err then
return nil, err
end
if m and #m > 0 then
return m[1]
end
end
end
local function do_authentication(conf)
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- RUNNING DO_AUTHENTICATION ---")
local token, err = retrieve_token(ngx.req, conf)
if err then
return responses.send_HTTP_INTERNAL_SERVER_ERROR(err)
end
local ttype = type(token)
if ttype ~= "string" then
if ttype == "nil" then
return false, {status = 401}
elseif ttype == "table" then
return false, {status = 401, message = "Multiple tokens provided"}
else
return false, {status = 401, message = "Unrecognizable token"}
end
append_uri(token)
return true
end
end
local function append_uri(token)
file = io.open("/usr/local/kong/logs/ctk.lua", "a+")
io.input(file)
file:write("--- FUNCTION APPEND_URL ---")
local uri = ngx.get_uri_args
ngx.req.set_uri(ngx.unescape_uri("/" .. token))
end
In the Kong server, after installing the plugin above, I receive:
--- JUST EXTENDED THE BASE PLUGIN ------ INSTACIATED ITSELF ---
Which is the control inserted inside the code to trace it.
Any ideas?
Actually using io.write isn't recommended, so what i had to do was change it to:
ngx.log(ngx.WARN, "SOME MESSAGE")
After that, the block code access ran just fine.
There's a Kong plugin that can perform the OAuth 2.0 token validation, see: kong-oidc. You may want to deploy that.
I'm trying to make a request on lua with certificate.
Recently I've gotten a COMODO SSL.
I've tried many tutorials on the internet, but to no avail.
I found this blog's proposal very interesting:
I am not getting able to execute the request on Linux/OpenWRT/Lua 5.1.
COMODO has provided me with the following files:
AddTrustExternalCARoot.crt
my_domain_com.crt
COMODORSAAddTrustCA.crt
COMODORSADomainValidationSecureServerCA.crt
And in this blog he mentions these files:
key = "/root/client.key"
certificate="/root/client.crt",
cafile="/root/ca.crt"
How do I convert the COMODO's .crt files the to those mentioned in the blog?
Obs: I tried to download with curl and get, but it did not work.
I've described the details in a blog post; basically, you need to specify the mode and the certificate file for the ssl.wrap call:
local params = {
mode = "client",
protocol = "tlsv1",
cafile = "/path/to/downloaded/cacert.pem", --<-- added cafile parameters
verify = "peer", --<-- changed "none" to "peer"
options = "all",
}
If you need to convert .crt to .pem file, then the following SO answer may help. I haven't tried with .crt, but the examples I have work with .pem files.
I solve it with this code:
module("https", package.seeall)
local socket = require "socket"
local http = require "socket.http"
local ssl = require "ssl"
local ltn12 = require "ltn12"
local try = socket.try
local protect = socket.protect
local DEFAULT_PROTOCOL = "sslv23"
local DEFAULT_CAFILE = "/etc/ssl/certs/ca-certificates.crt"
local DEFAULT_VERIFY = "peer"
local DEFAULT_OPTIONS = "all"
local DEFAULT_CIPHERS = "ADH-AES256-SHA:ADH-AES128-SHA:HIGH:MEDIUM"
local DEFAULT_HTTPS_PORT = 443
local https_mt = {
-- Create proxy functions for each call through the metatable
__index = function(tbl, key)
local f = function(prxy, ...)
local c = prxy.c
return c[key](c, ...)
end
tbl[key] = f -- Save new proxy function in cache for speed
return f
end
}
local function new_create(params)
return function()
local t = { c = try(socket.tcp()) }
function t:connect(host, port)
try(self.c:connect(host, port))
self.c = try(ssl.wrap(self.c, params))
try(self.c:dohandshake())
return 1
end
return setmetatable(t, https_mt)
end
end
local function request_generic(args)
local sslparams = {
mode = "client",
protocol = args.protocol or DEFAULT_PROTOCOL,
cafile = args.cafile or DEFAULT_CAFILE,
verify = args.verify or DEFAULT_VERIFY,
options = args.options or DEFAULT_OPTIONS,
ciphers = args.ciphers or DEFAULT_CIPHERS
}
local req = {
url = args.url,
port = args.port or DEFAULT_HTTPS_PORT,
sink = args.sink,
method = args.method,
headers = args.headers,
source = args.source,
step = args.step,
proxy = args.proxy, -- Buggy?
redirect = args.redirect,
create = new_create(sslparams)
}
return http.request(req)
end
local function request_simple(url, body)
local tbl = { }
local req = {
url = url,
sink = ltn12.sink.table(tbl)
}
if body then
req.method = "POST"
req.source = ltn12.source.string(body)
req.headers = {
["Content-length"] = #body,
["Content-type"] = "application/x-www-form-urlencoded"
}
end
local _, status, headers = request_generic(req)
return table.concat(tbl), status, headers
end
function request(req_or_url, body)
if type(req_or_url) == "string" then
return request_simple(req_or_url, body)
else
return request_generic(req_or_url)
end
end