python爬取网页内容转换为PDF文件

Python  /  管理员 发布于 7年前   166

本文实例为大家分享了python爬取网页内容转换为PDF的具体代码，供大家参考，具体内容如下

将廖雪峰的学习教程转换成PDF文件，代码只适合该网站，如果需要其他网站的教程，可靠需要进行稍微的修改。

# coding=utf-8 import os import re import time import pdfkit import requests from bs4 import BeautifulSoup from PyPDF2 import PdfFileMergerimport sysreload(sys)sys.setdefaultencoding('utf8')html_template = """ <!DOCTYPE html> <html lang="en"> <head>   <meta charset="UTF-8"> </head> <body> {content} </body> </html> """ #----------------------------------------------------------------------def parse_url_to_html(url, name):   """   解析URL，返回HTML内容   :param url:解析的url   :param name: 保存的html文件名   :return: html   """   try:     response = requests.get(url)     soup = BeautifulSoup(response.content, 'html.parser')     # 正文     body = soup.find_all(class_="x-wiki-content")[0]     # 标题     title = soup.find('h4').get_text()     # 标题加入到正文的最前面，居中显示     center_tag = soup.new_tag("center")     title_tag = soup.new_tag('h1')     title_tag.string = title     center_tag.insert(1, title_tag)     body.insert(1, center_tag)     html = str(body)     # body中的img标签的src相对路径的改成绝对路径     pattern = "(<img .*?src=\")(.*?)(\")"     def func(m):       if not m.group(3).startswith("http"):         rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)         return rtn       else:         return m.group(1)+m.group(2)+m.group(3)     html = re.compile(pattern).sub(func, html)     html = html_template.format(content=html)     html = html.encode("utf-8")     with open(name, 'wb') as f:       f.write(html)     return name   except Exception as e:    print "解析错误！"#----------------------------------------------------------------------def get_url_list():   """   获取所有URL目录列表   :return:   """   response = requests.get("http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000")   soup = BeautifulSoup(response.content, "html.parser")   menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1]   urls = []   for li in menu_tag.find_all("li"):     url = "http://www.liaoxuefeng.com" + li.a.get('href')     urls.append(url)   return urls #----------------------------------------------------------------------def save_pdf(htmls, file_name):   """   把所有html文件保存到pdf文件   :param htmls: html文件列表   :param file_name: pdf文件名   :return:   """   options = {     'page-size': 'Letter',     'margin-top': '0.75in',     'margin-right': '0.75in',     'margin-bottom': '0.75in',     'margin-left': '0.75in',     'encoding': "UTF-8",     'custom-header': [       ('Accept-Encoding', 'gzip')     ],     'cookie': [       ('cookie-name1', 'cookie-value1'),       ('cookie-name2', 'cookie-value2'),     ],     'outline-depth': 10,   }   pdfkit.from_file(htmls, file_name, options=options) #----------------------------------------------------------------------def main():   start = time.time()   file_name = u"liaoxuefeng_Python3_tutorial"   urls = get_url_list()   for index, url in enumerate(urls):    parse_url_to_html(url, str(index) + ".html")   htmls =[]   pdfs =[]   for i in range(0,124):     htmls.append(str(i)+'.html')     pdfs.append(file_name+str(i)+'.pdf')     save_pdf(str(i)+'.html', file_name+str(i)+'.pdf')     print u"转换完成第"+str(i)+'个html'   merger = PdfFileMerger()   for pdf in pdfs:    merger.append(open(pdf,'rb'))    print u"合并完成第"+str(i)+'个pdf'+pdf   output = open(u"廖雪峰Python_all.pdf", "wb")   merger.write(output)   print u"输出PDF成功！"   for html in htmls:     os.remove(html)     print u"删除临时文件"+html   for pdf in pdfs:     os.remove(pdf)     print u"删除临时文件"+pdf   total_time = time.time() - start   print(u"总共耗时：%f 秒" % total_time)#----------------------------------------------------------------------def changeDir(dir_name):  """  目录切换  """  if not os.path.exists(dir_name):    os.mkdir(dir_name)  os.chdir(dir_name)#----------------------------------------------------------------------if __name__ == '__main__':  #存放文件的路径  dir_name = '/home/Python/Html'    changeDir(dir_name)  main()

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持。


  • 上一条：
    python pandas 对series和dataframe的重置索引reindex方法
    下一条：
    python爬取网页转换为PDF文件

    • 昵称：

    邮箱：

    0条评论 (评论内容有缓存机制,请悉知!)
    Top

    Copyright·© 2019 侯体宗版权所有· 粤ICP备20027696号 PHP交流群

    侯体宗的博客