Scrapy: How to redownload already cached URL if certain conditions apply - scrapy

Question

I have a large amount of http data stored in my cache backend in scrapy. There are certain pages that contain false data. these urls need to be rescheduled for download on the next run of scrapy.
I came up with the idea to modify the dummy cache policy that comes with scrapy. Unfortunately this doesn't seem to work.
Can anyone see what is wrong in the method is_cached_response_fresh?:
import os
import cPickle as pickle
from time import time
from weakref import WeakKeyDictionary
from email.utils import mktime_tz, parsedate_tz
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.utils.httpobj import urlparse_cached
class DummyPolicy(object):
def __init__(self, settings):
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_http_codes = [int(x) for x in settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')]
def should_cache_request(self, request):
return urlparse_cached(request).scheme not in self.ignore_schemes
def should_cache_response(self, response, request):
return response.status not in self.ignore_http_codes
def is_cached_response_fresh(self, response, request):
if "thisstring" in response.body.lower():
print "got mobile page. redownload"
return False
else:
return True
def is_cached_response_valid(self, cachedresponse, response, request):
return True

Answer 1

I think the answer here is that your content is most likely gzipped or deflated.
Try
from scrapy.utils.gz import gunzip
if "thisstring" in gunzip(response.body).lower()
Cannot say that solution is versatile but most likely it'll work in your case.