settings.py

 1 ROBOTSTXT_OBEY = False
 2 
 3 LOG_LEVEL="ERROR"
 4 
 5 USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'
 6 
 7 
 8 ITEM_PIPELINES = {
 9    'testpro.pipelines.TestproPipeline': 300,
10 }