热搜：编程 Python Microsoft 编程语言 C++

python使用scrapy解析js示例

发布时间：2024-04-26 12:11:21 来源：好代码

.

风景，因走过而美丽。命运，因努力而精彩。南国园内看夭红，溪畔临风血艳浓。如果回到年少时光，那间学堂，我愿依靠在你身旁，陪你欣赏古人的诗章，往后的夕阳。


from selenium import selenium
class MySpider(CrawlSpider):
 name = 'cnbeta'
 allowed_domains = ['cnbeta.com']
 start_urls = ['//www.haodaima.com']
 rules = (
 # Extract links matching 'category.php' (but not matching 'subsection.php')
 # and follow links from them (since no callback means follow=True by default).
 Rule(SgmlLinkExtractor(allow=('/articles/.*\.htm', )),
 callback='parse_page', follow=True),
 # Extract links matching 'item.php' and parse them with the spider's method parse_item
 )
 def __init__(self):
 CrawlSpider.__init__(self)
 self.verificationErrors = []
 self.selenium = selenium("localhost", 4444, "*firefox", "//www.haodaima.com")
 self.selenium.start()
 def __del__(self):
 self.selenium.stop()
 print self.verificationErrors
 CrawlSpider.__del__(self)

 def parse_page(self, response):
 self.log('Hi, this is an item page! %s' % response.url)
 sel = Selector(response)
 from webproxy.items import WebproxyItem
 sel = self.selenium
 sel.open(response.url)
 sel.wait_for_page_to_load("30000")
 import time
 time.sleep(2.5)