how to generate javascript request to scrapy+webkit? - webkit

Question

I need to scrap a webpage generated by Javascript.The href is like this:
[4]
I wrote a WebkitDownloader in scrapy and it works.
class WebkitDownloader( object ):
def process_request( self, request, spider ):
if spider.name in settings.WEBKIT_DOWNLOADER:
if( type(request) is not FormRequest ):
print '===================Processing non-FormRequest=================='
webview = webkit.WebView()
webview.connect( 'load-finished', lambda v,f: gtk.main_quit() )
webview.load_uri( request.url )
gtk.main()
js = jswebkit.JSContext( webview.get_main_frame().get_global_context() )
renderedBody = str( js.EvaluateScript( 'document.documentElement.innerHTML' ) )
#print renderedBody
return HtmlResponse( request.url, body=renderedBody )
I still don't know how to send a request to WebkitDownloader in my spider parse module.
Can I use something like this: request.append('javascript:__doPostBack('pager','4')') and send the Javascript request to the WebkitDownloader? Or how to build the Javascript request?

Answer 1

you can use this code in your middleware
from scrapy.http import HtmlResponse
import gtk
import webkit
import jswebkit
class WebkitDownloader( object ):
def stop_gtk(self, v, f):
gtk.main_quit()
def _get_webview(self):
webview = webkit.WebView()
props = webview.get_settings()
props.set_property('enable-java-applet', False)
props.set_property('enable-plugins', False)
props.set_property('enable-page-cache', False)
return webview
def process_request( self, request, spider ):
if 'renderjs' in request.meta:
webview = self._get_webview()
webview.connect('load-finished', self.stop_gtk)
webview.load_uri(request.url)
gtk.main()
ctx = jswebkit.JSContext(webview.get_main_frame().get_global_context())
url = ctx.EvaluateScript('window.location.href')
html = ctx.EvaluateScript('document.documentElement.innerHTML')
return HtmlResponse(url, encoding='utf-8', body=html.encode('utf-8'))