1、在pipeline里判断爬虫
ITEM_PIPELINES = {
"xxxx.pipelines.MyPipeline": 300,
}
class OneSpider(scrapy.spiders.Spider):
name = "one"
class TwoSpider(scrapy.spiders.Spider):
name = "two"
class MyPipeline(object):
def process_item(self, item, spider):
if spider.name == "one":
print("one")
elif spider.name == "two":
print("two")
return item
2、在爬虫里设置pipeline(scrapy版本必须是1.1以上)
ITEM_PIPELINES = {
"xxxx.pipelines.OneSpiderPipeline": 300,
"xxxx.pipelines.TwoSpiderPipeline": 400,
}
class OneSpider(scrapy.Spider):
name = "one"
custom_settings = {
"ITEM_PIPELINES": {"xxxx.pipelines.OneSpiderPipeline": 300},
}
class TwoSpider(scrapy.Spider):
name = "two"
custom_settings = {
"ITEM_PIPELINES": {"xxxx.pipelines.TwoSpiderPipeline": 400},
}
class OneSpiderPipeline(object):
def process_item(self, item, spider):
print("one")
return item
class TwoSpiderPipeline(object):
def process_item(self, item, spider):
print("two")
return item
文章评论