selenium+PhantomJS爬取豆瓣读书

前端  /  管理员 发布于 7年前   173

本文实例为大家分享了selenium+PhantomJS爬取豆瓣读书的具体代码，供大家参考，具体内容如下

获取关于Python的全部书籍信息；
通过代码测试 request携带‘User-Agent'及 ‘data'数据信息的方式均无法获取到相关信息，获取数据时，部分数据为空，导致获取过程中报错，无法获取全部数据，初步判定豆瓣读书的反爬机制较为严格；通过selenium 模拟浏览器请求的方法测试后发现，可利用 selenium 方法请求获取数据；

#导入需要的模块from selenium import webdriverimport timefrom lxml import etreeimport pymysqlimport re#创建一个函数def my_browers(url, page):  # 获取浏览器对象  browers = webdriver.PhantomJS(executable_path=r'd:\Desktop\pythonjs\phantomjs-2.1.1-windows\bin\phantomjs.exe')    # 用浏览器发起请求  browers.get(url)    #休息两秒，频率低一点，爬的时间久一点，安全就多一点  time.sleep(2)     # 获取页面信息  html = browers.page_source    # 调用页面解析函数  parse_html(html)  # 解析页面信息def parse_html(html):  # 生成一个xpath对象  html = etree.HTML(html)  # 获取所有的书籍信息列表  books = html.xpath('//div[contains(@class,"sc-bZQynM")]')  # 遍历每一本书籍 然后拿到我们想要的数据  for book in books:    # 创建一个存书字典存数据用    book_dict = {}    # 获取封面信息    pic = book.xpath('//img/@src')    if pic:      book_dict['pic'] = pic[0]    else:      book_dict['pic'] = ''    # print(pic)    # 获取书名    book_name = book.xpath('//div[@class="title"]/a/text()')    # print(book_name)    if book_name:      book_name = book_name[0]      # 删除书名中最后出现的引号，      #由于存数据库的时候书名最后面的引号会导致数据库报错，删除可以使代码更健壮      if '"' in book_name:        pattern = re.compile(r'"')        book_name = pattern.sub('', book_name)      if "'" in book_name:        pattern = re.compile(r"'")        book_name = pattern.sub('', book_name)        # 删除书名中最后出现的\，存数据的时候书名最后的\会把sql语句最后的引号转义，        #删除可以使代码更健壮      if '\\' in book_name:        book_name = book_name[:-1]      book_dict['book_name'] = book_name    else:      book_dict['book_name'] = ''    # 获取书籍详情连接    book_url = book.xpath('//div[@class="title"]/a/@href')    if book_url:      book_dict['book_url'] = book_url[0]    else:      book_dict['book_url'] = ''    # 获取评分信息    score_book = book.xpath('//span[@class="rating_nums"]/text()')    if score_book:      book_dict['score_book'] = score_book[0]    else:      book_dict['score_book'] = ''    # 获取出版社信息    book_detail = book.xpath('//div[@class="meta abstract"]/text()')    if book_detail:      # 删除书详情中最后出现的引号；      book_detail = book_detail[0]      if "'" in book_detail:        pattern = re.compile(r"'")        book_detail = pattern.sub('', book_detail)      book_dict['book_detail'] = book_detail    else:      book_dict['book_detail'] = ''    print(book_dict)    # 调用数据库函数    insert_mysql(book_dict)# 插入数据库def insert_mysql(book_dict):  # 连接数据库  conn = pymysql.connect('localhost', 'root', 'root', 'test', charset='utf8')  # 创建操作数据库的对象  cursor = conn.cursor()  pic = book_dict['pic']  book_name = book_dict['book_name']  book_url = book_dict['book_url']  score = book_dict['score_book']  book_detail = book_dict['book_detail']  sql = f"insert into python_book (pic,book_name,book_url,score,book_detail) " \     f"VALUE ('{pic}','{book_name}','{book_url}','{score}','{book_detail}')"  # 执行并提交  cursor.execute(sql)  conn.commit()if __name__ == '__main__':  for i in range(0, 199):    print('=================下载第{}页========================'.format(i + 1))    page = i * 15    base_url = 'https://book.douban.com/subject_search?search_text=python&cat=1001&start={}'.format(page)    my_browers(base_url, page)

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持。