Why is scrapy so slow when using proxy rotator? - scrapy

Question

I'm new to scrapy and I'm trying to scrape 300 urls per spider for my project, but for testing purposes, I only scrape 2 urls. When I use proxy rotator it takes 120 secs to 500 secs to finish my script, but when I'm not using proxy it takes 2 to 20 secs to finish my script. Is there any solution that would improve the overall performance of my script? Also, I use proxy rotator from proxyrotator.com
I often get TimeoutError when using proxy rotator. Any advice or answer would be appreciated.
settings.py
#settings.py
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 0.25
RETRY_TIMES = 3
CONNECTION_TIMEOUT = 30
COOKIES_ENABLED = False
DEFAULT_REQUEST_HEADERS = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"DNT": "1",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
}
CONCURRENT_REQUESTS = 100
CONCURRENT_REQUESTS_PER_DOMAIN = 100
AUTOTHROTTLE_ENABLED = False
DOWNLOAD_TIMEOUT = 30
DOWNLOADER_MIDDLEWARES = {
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 400,
}
static_spider.py
class ArticleStaticSpider(scrapy.Spider):
logfile("server.log", maxBytes=1e6, backupCount=3)
name = "article_static"
custom_settings = {
'ITEM_PIPELINES': {'news_extractor.pipelines.StaticExtractorPipeline': 300},
"FEEDS": {"articles.json": {"format": "json"}},
}
def __init__(self, urls=None):
self.urls = urls
def start_requests(self):
for url in self.urls:
proxy = get_proxy() # get request for proxy rotator
ip = proxy['ip']
port = proxy['port']
meta_proxy = f"http://{ip}:{port}"
headers = {
"User-Agent": proxy['randomUserAgent']
}
meta = {
"proxy": meta_proxy }
yield scrapy.Request(url, callback=self.parse,headers=headers, meta=meta, errback=self.errback_httpbin)
def parse(self, response):
src = StaticSource(response.url) # module for cleaning html tags
text_format = src.text
news = News(response.url, text_format) # module for extracting data
data = news.generate_data()
print(data)
scraper.py
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
import time
ulrs = [
"http://www.nytimes.com/2021/02/28/technology/seniors-vaccines-technology.html",
"http://www.nytimes.com/2021/02/28/us/schools-reopening-philadelphia-parents.html"
]
def main():
process = CrawlerProcess(get_project_settings())
process.crawl('article_static', urls=urls)
process.crawl('article_static', urls=urls)
process.start()
if __name__ == "__main__":
t1 = time.perf_counter()
main()
t2 = time.perf_counter()
print(f'Finished in {round(t2-t1, 2)} second(s).....')