python实现爬取图书封面

Python  /  管理员 发布于 7年前   181

本文实例为大家分享了python实现爬取图书封面的具体代码，供大家参考，具体内容如下

kongfuzi.py

利用更换代理ip，延迟提交数据，设置请求头破解网站的反爬虫机制

import requestsimport randomimport time  class DownLoad():  def __init__(self):    self.ip_list = ['191.33.179.242:8080', '122.72.108.53:80', '93.190.142.214:80', '189.8.88.125:65301','36.66.55.181:8080', '170.84.102.5:8080', '177.200.72.214:20183', '115.229.115.190:9000']     self.user_agent_list = [      'User-Agent:Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',      'User-Agent:Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',      'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'    ]   def get(self, url, proxy=None, timeout=20, num=5):    print("正在请求%s" % url)    UA = random.choice(self.user_agent_list)    headers = {'User-Agent': UA}     if proxy == None:      try:        return requests.get(url, headers=headers, timeout=timeout)      except:        if num > 0:          time.sleep(10)          return self.get(url, num=num - 1)        else:          time.sleep(10)          IP = ''.join(random.choice(self.ip_list).strip())          proxy = {'http': IP}          return self.get(url, proxy=proxy, timeout=timeout)    else:      try:        IP = ''.join(random.choice(self.ip_list).strip())        proxy = {'http': IP}        return requests.get(url, headers=headers, proxy=proxy, timeout=timeout)      except:        if num > 0:          time.sleep(10)          IP = ''.join(random.choice(self.ip_list).strip())          proxy = {'http': IP}          print("正在更换代理")          print("当前代理%s" % proxy)          return self.get(url, proxy=proxy, num=num - 1)

main.py

将爬取的图片保存到本地，然后展示到界面

import kongfuziimport osimport requestsimport bs4from tkinter import *from PIL import Image, ImageTk  # 下载图片，生成图片地址列表和图书信息列表def download():  baseUrl = "http://search.kongfz.com"  keyword = e1.get()  url = baseUrl + "/product_result/?select=0&key=" + keyword  print("下载链接：" + url)  show(url)  # bs4处理def changesoup(html):  htm = html.content  html_doc = str(htm, 'utf-8')  soup = bs4.BeautifulSoup(html_doc, "html.parser")  return soup  # 图书信息集合def bookinfo(soup):  # 图书价格列表  price = []  soupprice = soup.select(".first-info .f_right .bold")  for i in soupprice:    price.append(i.string)   # 书店名列表  storename = []  soupstorename = soup.select(".text a span")  for each in soupstorename:    if each.string == None:      soupstorename.remove(each)  for i in soupstorename:    storename.append(i.string)   # 商家地区列表  place = []  soupplace = soup.select(".user-place")  for i in soupplace:    place.append(i.string)   # 书名列表  bookname = []  bookname1 = soup.select(    ".search-wrap .search-main .search-main-result .result-content .result-list .item .item-info .title .link")  # print(len(bookname1))  # print(bookname1)  for each in bookname1:    print(each)    # a = bs4.BeautifulSoup(each, "html.parser")    a = each.get_text()    print(a)    # type(a)    # a = bs4.BeautifulSoup(a, "html.parser")    # b = a.get_text()    bookname.append(a)  # print(bookname)  # print(len(bookname))   return bookname, price, place, storename  # 保存图片def imgsave(soup):  dirName = "image"  os.makedirs(dirName, exist_ok=True)  filePathList = []  imgUrl = soup.select(".search-main-result .result-content .result-list .item .item-img .img-box img")   # print(imgUrl)  if not imgUrl:    print("没有找到当前节点下图片")  else:    i = 0    for imageUrls in imgUrl:      # 找到图片地址 获取它      downloadUrl = imageUrls.get('src')      # if downloadUrl == "/searchfront/img/error.jpg":      #   downloadUrl = "http://book.kongfz.com/img/pc/error.jpg"      print("打印要下载的图片地址:", downloadUrl)      #   http://book.kongfz.com/img/pc/error.jpg      # 分割字符      split = downloadUrl.split("/")      # 只保留最后一个元素      fileName = str(i) + "-" + os.path.basename(split[len(split) - 1])      print("文件名：" + fileName)      # 建立一个新路径      filePath = os.path.join(dirName, fileName)      filePathList.append(filePath)      if not os.path.exists(filePath):        imageUrlPath = requests.get(downloadUrl)        # 检查当前网络是否请求成功        imageUrlPath.raise_for_status()        # 'wb'二进制模式打开img适用        imageFile = open(filePath, 'wb')        for image in imageUrlPath.iter_content(10000):          # 把每次遍历的文件图像都存储进文件夹中          imageFile.write(image)        # 关闭文件        imageFile.close()      i = i + 1  return filePathList # 图片展示def show(url):  xz = kongfuzi.DownLoad()  html = xz.get(url)   # 添加代理ip到ip_list  add_ip = e2.get()  xz.ip_list.append(add_ip)   soup = changesoup(html)  bookname, price, place, storename = bookinfo(soup)  # print(bookname)  # print(price)  # print(place)  # print(storename)  filePathList = imgsave(soup)  root1 = Toplevel()  root1.geometry("1720x800")  root1.title("孔网图片爬取")   # 处理图片，转换成可以显示  photo = []  temp = []  for each in filePathList:    temp = Image.open(each)    photo.append(ImageTk.PhotoImage(temp))   canvas = Canvas(root1, width=1700, height=800, scrollregion=(0, 0, 0, 4000)) # 创建canvas  canvas.place(x=10, y=10) # 放置canvas的位置   frame = Frame(canvas) # 把frame放在canvas里  frame.place(width=1680, height=800)   for i in range(50):    # 图片行列    rownum = int(i / 5)    columnnum = i % 5     # photo = ImageTk.PhotoImage(Image.open(filePathList[i]))    imgLabel1 = Label(frame, image=photo[i], width=280, height=280)    imgLabel1.grid(row=rownum * 5, column=columnnum, padx=10, pady=5)     infoLabel1 = Label(frame, text="书名：" + bookname[i], bg="#FFF8DC", justify=LEFT)    infoLabel1.grid(row=rownum * 5 + 1, column=columnnum, padx=45, pady=2, sticky=W)    infoLabel2 = Label(frame, text="价格：" + price[i] + "元", bg="#FFF8DC", justify=LEFT)    infoLabel2.grid(row=rownum * 5 + 2, column=columnnum, padx=45, pady=2, sticky=W)    infoLabel3 = Label(frame, text="发货地区：" + place[i], bg="#FFF8DC", justify=LEFT)    infoLabel3.grid(row=rownum * 5 + 3, column=columnnum, padx=45, pady=2, sticky=W)    infoLabel4 = Label(frame, text="书店：" + storename[i], bg="#FFF8DC", justify=LEFT)    infoLabel4.grid(row=rownum * 5 + 4, column=columnnum, padx=45, pady=2, sticky=W)   vbar = Scrollbar(canvas, orient=VERTICAL) # 竖直滚动条  vbar.place(x=1680, width=20, height=800)  vbar.configure(command=canvas.yview)  canvas.config(yscrollcommand=vbar.set) # 设置  canvas.create_window((800, 2000), window=frame)   mainloop()  if __name__ == '__main__':  # 界面  root = Tk()  root.title("孔网图片爬取")  e1 = Entry(root)  e2 = Entry(root)  e1.grid(row=0, column=0, padx=20, pady=20)  e2.grid(row=0, column=2, padx=20, pady=20)  label1 = Label(root, text="关键字", width=10).grid(row=0, column=1, padx=10, pady=5)  label2 = Label(root, text="添加代理ip", width=10).grid(row=0, column=3, padx=10, pady=5)  btn1 = Button(root, text="搜索", width=10, command=download).grid(row=1, column=1, padx=10, pady=5)  # print(e1.get())  mainloop()

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持。