How can i run different spiders at the same time that they have different Crawler Runner Settings - scrapy

Question

Default usage is:
import scrapy
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
class MySpider1(scrapy.Spider):
# Your first spider definition
...
class MySpider2(scrapy.Spider):
# Your second spider definition
...
configure_logging()
settings = get_project_settings()
runner = CrawlerRunner(settings)
runner.crawl(MySpider1)
runner.crawl(MySpider2)
d = runner.join()
d.addBoth(lambda _: reactor.stop())
My codes:
import scrapy
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor
runner1 = CrawlerRunner(settings = {
"FEEDS": {
r"file:///C:\\Users\Messi\\1.json": {"format": "json", "overwrite": True}
},
})
runner2 = CrawlerRunner(settings = {
"FEEDS": {
r"file:///C:\\Users\Messi\\2.json": {"format": "json", "overwrite": True}
},
})
runner3 = CrawlerRunner(settings = {
"FEEDS": {
r"file:///C:\\Users\Messi\\3.json": {"format": "json", "overwrite": True}
},
})
h = runner1.crawl(Live1)
h.addBoth(lambda _: reactor.stop())
a = runner2.crawl(Live2)
a.addBoth(lambda _: reactor.stop())
t = runner3.crawl(Live3)
t.addBoth(lambda _: reactor.stop())
reactor.run()
Above code does not work!
How can i run different spiders at the same time that they have different crawler runner settings?
Settings are different so i used different variables for them runner1,runner2, runner3...
What should be the right usage? Could you please help me about this topic.
Thanks very much.

Answer 1

Like I said in the comment, I think that using custom_settings is better.
Anyway, this what worked for me:
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from tempbuffer.spiders.spider1 import ExampleSpider1
from tempbuffer.spiders.spider import ExampleSpider2
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
runner = CrawlerRunner(settings={
"FEEDS": {
r"1.json": {"format": "json", "overwrite": True}
}})
runner.crawl(ExampleSpider1)
runner = CrawlerRunner(settings={
"FEEDS": {
r"2.json": {"format": "json", "overwrite": True}
}})
runner.crawl(ExampleSpider2)
d = runner.join()
d.addBoth(lambda _: reactor.stop())
reactor.run()
Another way:
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from tempbuffer.spiders.spider1 import ExampleSpider1
from tempbuffer.spiders.spider import ExampleSpider2
configure_logging({'LOG_FORMAT': '%(levelname)s: %(message)s'})
runner1 = CrawlerRunner(settings={
"FEEDS": {
r"1.json": {"format": "json", "overwrite": True}
}})
runner2 = CrawlerRunner(settings={
"FEEDS": {
r"2.json": {"format": "json", "overwrite": True}
}})
d = runner1.crawl(ExampleSpider2)
runner2.crawl(ExampleSpider1)
d.addBoth(lambda _: reactor.stop())
reactor.run()
1.json:
[
{"title": "Short Dress", "price": "$24.99"},
{"title": "Patterned Slacks", "price": "$29.99"},
{"title": "Short Chiffon Dress", "price": "$49.99"},
{"title": "Off-the-shoulder Dress", "price": "$59.99"},
{"title": "V-neck Top", "price": "$24.99"},
{"title": "Short Chiffon Dress", "price": "$49.99"},
{"title": "V-neck Top", "price": "$24.99"},
{"title": "V-neck Top", "price": "$24.99"},
{"title": "Short Lace Dress", "price": "$59.99"}
]
2.json:
[
{"title": "Long-sleeved Jersey Top", "price": "$12.99"}
]
I kind of guessed the answer, I'm not sure which of this is better. If someone wants to correct/explain/clarify in the comments I'll be glad.