Python基于分析Ajax请求实现抓取今日头条街拍图集功能示例

Python  /  管理员 发布于 7年前   153

本文实例讲述了Python基于分析Ajax请求实现抓取今日头条街拍图集功能。分享给大家供大家参考，具体如下：

代码:

import osimport reimport jsonimport timefrom hashlib import md5from multiprocessing import Poolimport requestsfrom requests.exceptions import RequestExceptionfrom pymongo import MongoClient# 配置信息OFFSET_START = 0  # 爬去页面的起始下标OFFSET_END = 20  # 爬去页面的结束下标KEYWORD = '街拍'  # 搜索的关键字# mongodb相关配置MONGO_URL = 'localhost'MONGO_DB = 'toutiao'  # 数据库名称MONGO_TABLE = 'jiepai' # 集合名称# 图片保存的文件夹名称IMAGE_PATH = 'images'headers = {  "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}client = MongoClient(host=MONGO_URL)db = client[MONGO_DB]jiepai_table = db[MONGO_TABLE]if not os.path.exists(IMAGE_PATH):  os.mkdir(IMAGE_PATH)def get_html(url, params=None):  try:    response = requests.get(url, params=params, headers=headers)    if response.status_code == 200:      return response.text    return None  except RequestException as e:    print("请求%s失败: " % url, e)    return None# 获取索引页内容def get_index_page(offset, keyword):  basic_url = 'http://www.toutiao.com/search_content/'  params = {    'offset': offset,    'format': 'json',    'keyword': keyword,    'autoload': 'true',    'count': 20,    'cur_tab': 3  }  return get_html(basic_url, params)def parse_index_page(html):  '''  解析索引页内容  返回: 索引页中包含的所有详情页url  '''  if not html:    return  data = json.loads(html)  if 'data' in data:    for item in data['data']:      article_url = item['article_url']      if 'toutiao.com/group' in article_url:        yield article_url# 获取详情页def get_detail_page(url):  return get_html(url)# 解析详情页def parse_detail_page(url, html):  '''    解析详情页    返回对应的标题，url和包含的图片url  '''  title_reg = re.compile('<title>(.*?)</title>')  title = title_reg.search(html).group(1)  gallery_reg = re.compile('var gallery = (.*?);')  gallery = gallery_reg.search(html)  if gallery and 'sub_images' in gallery.group(1):    images = json.loads(gallery.group(1))['sub_images']    image_list = [image['url'] for image in images]    return {      'title': title,      'url': url,      'images': image_list    }  return Nonedef save_to_mongodb(content):  jiepai_table.insert(content)  print("存储到mongdob成功", content)def download_images(image_list):  for image_url in image_list:    try:      response = requests.get(image_url)      if response.status_code == 200:        save_image(response.content)    except RequestException as e:      print("下载图片失败: ", e)def save_image(content):  '''    对图片的二进制内容做hash，构造图片路径,以此保证图片不重复  '''  file_path = '{0}/{1}/{2}.{3}'.format(os.getcwd(),         IMAGE_PATH, md5(content).hexdigest(), 'jpg')  # 去除重复的图片  if not os.path.exists(file_path):    with open(file_path, 'wb') as f:      f.write(content)def jiepai(offset):  html = get_index_page(offset, KEYWORD)  if html is None:    return  page_urls = list(parse_index_page(html))  # print("详情页url列表:" )  # for page_url in page_urls:  #   print(page_url)  for page in page_urls:    print('get detail page:', page)    html = get_detail_page(page)    if html is None:      continue    content = parse_detail_page(page, html)    if content:      save_to_mongodb(content)      download_images(content['images'])      time.sleep(1)  print('-------------------------------------')if __name__ == '__main__':  offset_list = range(OFFSET_START, OFFSET_END)  pool = Pool()  pool.map(jiepai, offset_list)

备注:

其实通过url请求返回的json数据中已经包含了图片列表

import requestsbasic_url = 'http://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=3'url = basic_url.format(0)html = requests.get(url).json()items = html['data']for item in items:  title = item['media_name']  image_list = [image_detail['url'] for image_detail in item['image_detail']]  print(title, image_list)

更多关于Python相关内容感兴趣的读者可查看本站专题：《Python Socket编程技巧总结》、《Python URL操作技巧总结》、《Python数据结构与算法教程》、《Python函数使用技巧总结》、《Python字符串操作技巧汇总》及《Python入门与进阶经典教程》

希望本文所述对大家Python程序设计有所帮助。