php与python实现的线程池多线程爬虫功能示例

Python  /  管理员 发布于 7年前   183

本文实例讲述了php与python实现的线程池多线程爬虫功能。分享给大家供大家参考，具体如下：

多线程爬虫可以用于抓取内容了这个可以提升性能了，这里我们来看php与python 线程池多线程爬虫的例子，代码如下：

php例子

url = $url;}public function run(){$ch = $this->worker->getConnection();curl_setopt($ch, CURLOPT_URL, $this->url);$page = curl_exec($ch);$info = curl_getinfo($ch);$error = curl_error($ch);$this->deal_data($this->url, $page, $info, $error);$this->result = $page;}function deal_data($url, $page, $info, $error){$parts = explode(".", $url);$id = $parts[1];if ($info['http_code'] != 200){$this->show_msg($id, $error);} else{$this->show_msg($id, "OK");}}function show_msg($id, $msg){echo $id."\t$msg\n";}public function getResult(){return $this->result;}protected $url;protected $result;}function check_urls_multi_pthreads(){global $check_urls; //定义抓取的连接$check_urls = array( 'http://xxx.com' => "xx网",);$pool = new Pool(10, "Connect", array()); //建立10个线程池foreach ($check_urls as $url => $name){$pool->submit(new Query($url));}$pool->shutdown();}check_urls_multi_pthreads();python 多线程def handle(sid)://这个方法内执行爬虫数据处理passclass MyThread(Thread):"""docstring for ClassName"""def __init__(self, sid):Thread.__init__(self)self.sid = siddef run():handle(self.sid)threads = []for i in xrange(1,11):t = MyThread(i)threads.append(t)t.start()for t in threads:t.join()

python 线程池爬虫：

from queue import Queuefrom threading import Thread, Lockimport urllib.parseimport socketimport reimport timeseen_urls = set(['/'])lock = Lock()class Fetcher(Thread):  def __init__(self, tasks):    Thread.__init__(self)    self.tasks = tasks    self.daemon = True    self.start()  def run(self):    while True:      url = self.tasks.get()      print(url)      sock = socket.socket()      sock.connect(('localhost', 3000))      get = 'GET {} HTTP/1.0\r\nHost: localhost\r\n\r\n'.format(url)      sock.send(get.encode('ascii'))      response = b''      chunk = sock.recv(4096)      while chunk:        response += chunk        chunk = sock.recv(4096)      links = self.parse_links(url, response)      lock.acquire()      for link in links.difference(seen_urls):        self.tasks.put(link)      seen_urls.update(links)      lock.release()      self.tasks.task_done()  def parse_links(self, fetched_url, response):    if not response:      print('error: {}'.format(fetched_url))      return set()    if not self._is_html(response):      return set()    urls = set(re.findall(r'''(?i)href=["']?([^\s"'<>]+)''',   self.body(response)))    links = set()    for url in urls:      normalized = urllib.parse.urljoin(fetched_url, url)      parts = urllib.parse.urlparse(normalized)      if parts.scheme not in ('', 'http', 'https'):        continue      host, port = urllib.parse.splitport(parts.netloc)      if host and host.lower() not in ('localhost'):        continue      defragmented, frag = urllib.parse.urldefrag(parts.path)      links.add(defragmented)    return links  def body(self, response):    body = response.split(b'\r\n\r\n', 1)[1]    return body.decode('utf-8')  def _is_html(self, response):    head, body = response.split(b'\r\n\r\n', 1)    headers = dict(h.split(': ') for h in head.decode().split('\r\n')[1:])    return headers.get('Content-Type', '').startswith('text/html')class ThreadPool:  def __init__(self, num_threads):    self.tasks = Queue()    for _ in range(num_threads):      Fetcher(self.tasks)  def add_task(self, url):    self.tasks.put(url)  def wait_completion(self):    self.tasks.join()if __name__ == '__main__':  start = time.time()  pool = ThreadPool(4)  pool.add_task("/")  pool.wait_completion()  print('{} URLs fetched in {:.1f} seconds'.format(len(seen_urls),time.time() - start))

更多关于PHP相关内容感兴趣的读者可查看本站专题：《php curl用法总结》、《PHP数组(Array)操作技巧大全》、《php排序算法总结》、《PHP常用遍历算法与技巧总结》、《PHP数据结构与算法教程》、《php程序设计算法总结》、《PHP数学运算技巧总结》、《php正则表达式用法总结》、《PHP运算与运算符用法总结》、《php字符串(string)用法总结》及《php常见数据库操作技巧汇总》

希望本文所述对大家PHP程序设计有所帮助。

您可能感兴趣的文章:

• python爬虫中多线程的使用详解
• Python 微信爬虫完整实例【单线程与多线程】
• python面向对象多线程爬虫爬取搜狐页面的实例代码
• python爬虫爬取快手视频多线程下载功能
• Python之多线程爬虫抓取网页图片的示例代码
• Python3多线程爬虫实例讲解代码
• Python多线程爬虫实战_爬取糗事百科段子的实例
• Python 爬虫多线程详解及实例代码
• python支持多线程的爬虫实例