Python抓取聚划算商品分析页面获取商品信息并以XML格式保存到本地

本文实例为大家分享了Android九宫格图片展示的具体代码，供大家参考，具体内容如下

#!/user/bin/python
# -*- coding: gbk -*-
#Spider.py 

import urllib2
import httplib
import StringIO
import gzip
import re
import chardet
import sys
import os
import datetime
from xml.dom.minidom import Document
from BeautifulSoup import BeautifulSoup 

## 这段代码是用于解决控制台打印汉字报错的问题
reload(sys)
sys.setdefaultencoding("utf8")
##################################################### 

## debug模式开关，开启后可以看到Http请求的头部信息以及debug日志
DEBUG = 1
NO_DEBUG = 0
httplib.HTTPConnection.debuglevel = DEBUG
## 是否显示爬取网页源代码开关
showSrcCode = False
## 压缩方式
ZIP_TYPE = "gzip" 

fileName = "auctions"
location = "d://spiderData/" 

## header
headerConfig = {"User-Agent":"taobao-yanyuan.qzs", "Accept-encoding":ZIP_TYPE}
##################################################### 

#############class SpiderConfig #####################
class SpiderConfig:
  """
    configuration for spider name and url
  """
  def __init__(self, name, url):
    self.name = name
    self.url = url
##################################################### 

##############class SpiderAuctionDomain##############
class SpiderAuctionDomain:
  """
    Store information with auctions spidered by python
  """
  title = ""
  url = ""
  img = ""
  price = "" 

  def __init__(self):
    pass 

##################################################### 

########class SpiderDefaultErrorHandler##############
class SpiderDefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
  def http_error_default(self, req, fp, code, msg, hdrs):
    """
      default error process handler for spider
    """
    result = urllib2.HTTPError(req.get_full_url(), code, msg, hdrs, fp)
    result.status = code
    result.url = req.get_full_url() 

    print "<", result.url, "Exception code :", result.status, ">" 

    return result
##################################################### 

#############class SpiderHandler#####################
class SpiderHandler:
  """
    spider handler
  """ 

  def spider(self, spiderConfig):
    try:
      request = urllib2.Request(spiderConfig.url) 

      ## configure request hreader
      for key,val in headerConfig.items():
        request.add_header(key, val) 

      ## build opener
      opener = urllib2.build_opener(SpiderDefaultErrorHandler()) 

      ## open request
      openRequest = opener.open(request) 

      ## read data
      spiderData = openRequest.read() 

      ## close
      opener.close() 

      if 0 == len(spiderData):
        return 

      if ZIP_TYPE== openRequest.headers.get("Content-Encoding"):
        spiderData = SpiderHandler.gzipData(self, spiderData) 

      if httplib.HTTPConnection.debuglevel == DEBUG and showSrcCode:
        print spiderData 

      # parse html
      SpiderHandler.parse(self, spiderData) 

    except Exception,x:
      print "spider process Exception:", x 

  def parse(self, spiderData):
    """
      parse html content
    """ 

    if httplib.HTTPConnection.debuglevel == DEBUG:
      charsetAnalyze = chardet.detect(spiderData)
      print "analyze spider data encode :",charsetAnalyze["encoding"] 

    print "执行解析", fileName 

    soup = BeautifulSoup(spiderData)
    encode = soup.originalEncoding 

    encoding = lambda x : x.encode(encode) 

    if httplib.HTTPConnection.debuglevel == DEBUG:
      print "识别到编码：", encode
      title = soup.head.title.string
      print encoding(title) 

    spiderContents = soup.findAll(name="div", attrs={"class":"main-box avil"})
    auctions = ["%s" % s for s in spiderContents] 

    if auctions is None:
      return 

    auctionList = [] 

    for auc in auctions:
      auctionDomain = SpiderAuctionDomain()
      # parse auction link
      links = re.search(re.compile(r'<a href=[\"|\']http://ju.taobao.com/tg/life_home.htm\?item_id=([^>]*)[\"|\']', re.IGNORECASE), auc)
      if links is not None :
        auctionDomain.link = encoding("http://ju.taobao.com/tg/life_home.htm?item_id=" + "".join(["%s" % s for s in links.groups() if len(s) > 0])) 

      #parse auction title
      titles = re.search(re.compile(r"([^>]*)</a></h2>", re.IGNORECASE), auc)
      if titles is not None:
        auctionDomain.title = encoding("".join(["%s" % t for t in titles.groups() if len(t) > 0])) 

      #parse auction price
      price = re.search(re.compile(r"<strong class=\"J_juPrices\".*</b>([^<]*)</strong>", re.IGNORECASE), auc)
      if price is not None:
        auctionDomain.price = "".join(["%s" % p for p in price.groups() if len(p) > 0]) 

      #parse image url
      imgs = re.search(re.compile(r"<img src=[\'\"]([^>]*)[\'\"]", re.IGNORECASE), auc)
      if imgs is not None:
        auctionDomain.img = "".join(["%s" % i for i in imgs.groups() if len(i) > 0]) 

      auctionList.append(auctionDomain) 

    print "成功解析商品信息："
    for a in auctionList:
      print "--->",a.title 

    # sort auction list
    auctionList = SpiderHandler.sortAuctionList(self, auctionList) 

    # save in file
    SpiderHandler.save(self, auctionList) 

    print "解析完成" 

    pass 

  def sortAuctionList(self, auctionList):
    """
      冒泡排序，按照价格排序
    """
    length = len(auctionList)
    if length < 2:
      return auctionList
    else:
      for i in range(length-1):
        for j in range(length - i -1):
          if float(auctionList[j].price) > float(auctionList[j+1].price):
            auctionList[j], auctionList[j+1] = auctionList[j+1], auctionList[j]
    return auctionList
    pass 

  def save(self, auctionList):
    if auctionList is not None:
      doc = Document() 

      auctions = doc.createElement("auctions")
      doc.appendChild(auctions) 

      for auc in auctionList:
        auction = doc.createElement("auction")
        auctions.appendChild(auction) 

        SpiderHandler.generateXML(self, doc, auction, "title", auc.title)
        SpiderHandler.generateXML(self, doc, auction, "price", auc.price)
        SpiderHandler.generateXML(self, doc, auction, "img", auc.img)
        SpiderHandler.generateXML(self, doc, auction, "link", auc.link) 

      if False == os.path.exists(location):
        os.mkdir(location) 

      file = open(location+fileName+".xml", 'w')
      file.write(doc.toprettyxml())
      file.close() 

      if httplib.HTTPConnection.debuglevel == DEBUG:
        print doc.toprettyxml() 

  def generateXML(self, doc, f, name, txt):
    c = doc.createElement(name)
    f.appendChild(c)
    c.appendChild(doc.createTextNode(txt)) 

  def gzipData(self, spiderData):
    """
      get data from gzip
    """
    if 0 == len(spiderData):
      return spiderData
    spiderDataStream = StringIO.StringIO(spiderData)
    spiderData = gzip.GzipFile(fileobj=spiderDataStream).read()
    return spiderData
##################################################### 

if __name__ == "__main__":
  nowtime = lambda:datetime.datetime.strftime(datetime.datetime.now(),"%Y年%m月%d日 %H时%m分%S秒") 

  needSpiderUrl = {"suzhou":"http://ju.taobao.com/suzhou",
           "hangzhou":"http://ju.taobao.com/hangzhou",
           "shanghai":"http://ju.taobao.com/shanghai",
           "beijing":"http://ju.taobao.com/beijing",
           "chengdu":"http://ju.taobao.com/chengdu"} 

  configList = []
  for k,v in needSpiderUrl.items():
    spiderConfig = SpiderConfig(k, v)
    configList.append(spiderConfig) 

  spiderHandler = SpiderHandler() 

  print "爬虫执行开始时间：",nowtime()
  for spiderConfig in configList:
    fileName = spiderConfig.name
    spiderHandler.spider(spiderConfig) 

  print "爬虫执行完毕时间：",nowtime() 

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持我们。

您可能感兴趣的文章:

• Python 抓取动态网页内容方案详解
• 零基础写python爬虫之使用urllib2组件抓取网页内容
• 使用Python编写简单网络爬虫抓取视频下载资源
• python采用requests库模拟登录和抓取数据的简单示例
• python抓取网页图片示例(python爬虫)
• Python实现抓取页面上链接的简单爬虫分享
• 通过抓取淘宝评论为例讲解Python爬取ajax动态生成的数据(经典)
• python 自动提交和抓取网页
• Python爬虫抓取手机APP的传输数据
• python制作爬虫并将抓取结果保存到excel中