Scrapy项目实战之爬取某社区用户详情
本文介绍了Scrapy项目实战之爬取某社区用户详情，分享给大家，具有如下：
get_cookies.py
from selenium import webdriver
from pymongo import MongoClient
from scrapy.crawler import overridden_settings
# from segmentfault import settings
import time
import settings
class GetCookies(object):
def __init__(self):
# 初始化组件
# 设定webdriver选项
self.opt = webdriver.ChromeOptions()
# self.opt.add_argument("--headless")
# 初始化用户列表
self.user_list = settings.USER_LIST
# 初始化MongoDB参数
self.client = MongoClient(settings.MONGO_URI)
self.db = self.client[settings.MONGO_DB]
self.collection = self.db["cookies"]
def get_cookies(self,username,password):
"""
:param username:
:param password:
:return: cookies
"""
# 使用webdriver选项创建driver
driver = webdriver.Chrome(executable_path="/Users/Hank/scrapy/segmentfault/segmentfault/chromedriver",options=self.opt)
driver.get("https://segmentfault.com/user/login")
driver.find_element_by_name("username").send_keys(username)
driver.find_element_by_name("password").send_keys(password)
driver.find_element_by_xpath("//button[@type='submit']").click()
time.sleep(2)
driver.get("https://segmentfault.com/u/luwangmeilun/users/following")
# 登陆之后获取页面cookies
cookies = driver.get_cookies()
driver.quit()
return cookies
def format_cookies(self,cookies):
"""
:param cookies:
从driver.get_cookies的形式为：
[{'domain': 'segmentfault.com', 'httpOnly': False, 'name': 'PHPSESSID',
'path': '/', 'secure': False, 'value': 'web2~5grmfa89j12eksub8hja3bvaq4'},
{'domain': '.segmentfault.com', 'expiry': 1581602940, 'httpOnly': False,
'name': 'Hm_lvt_e23800c454aa573c0ccb16b52665ac26', 'path': '/', 'secure': False,
'value': '1550066940'},
{'domain': '.segmentfault.com', 'httpOnly': False,
'name': 'Hm_lpvt_e23800c454aa573c0ccb16b52665ac26',
'path': '/', 'secure': False, 'value': '1550066940'},
{'domain': '.segmentfault.com', 'expiry': 1550067000, 'httpOnly': False,
'name': '_gat', 'path': '/', 'secure': False, 'value': '1'},
{'domain': '.segmentfault.com', 'expiry': 1550153340, 'httpOnly': False,
'name': '_gid', 'path': '/', 'secure': False, 'value': 'GA1.2.783265084.1550066940'},
{'domain': '.segmentfault.com', 'expiry': 1613138940, 'httpOnly': False, 'name': '_ga',
'path': '/', 'secure': False, 'value': 'GA1.2.1119166665.1550066940'}]
只需提取每一项的name与value即可
:return:
"""
c = dict()
for item in cookies:
c[item['name']] = item['value']
return c
def save(self):
print("开始获取Cookies....")
# 从用户列表中获取用户名与密码，分别登陆获取cookies
for username,password in self.user_list:
cookies = self.get_cookies(username,password)
f_cookies = self.format_cookies(cookies)
print("insert cookie:{}".format(f_cookies))
# 将格式整理后的cookies插入MongoDB数据库
self.collection.insert_one(f_cookies)
# s = db[self.collection].find()
# for i in s:
# print(i)
if __name__ == '__main__':
cookies = GetCookies()
for i in range(20):
cookies.save()
item.py
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://doc.scrapy.org/en/latest/topics/items.html import scrapy class SegmentfaultItem(scrapy.Item): # define the fields for your item here like: # 个人属性 # 姓名 name = scrapy.Field() # 声望 rank = scrapy.Field() # 学校 school = scrapy.Field() # 专业 majors = scrapy.Field() # 公司 company = scrapy.Field() # 工作 job = scrapy.Field() # blog blog = scrapy.Field() # 社交活动数据 # 关注人数 following = scrapy.Field() # 粉丝数 fans = scrapy.Field() # 回答数 answers = scrapy.Field() # 提问数 questions = scrapy.Field() # 文章数 articles = scrapy.Field() # 讲座数 lives = scrapy.Field() # 徽章数 badges = scrapy.Field() # 技能属性 # 点赞数 like = scrapy.Field() # 技能 skills = scrapy.Field() # 注册日期 register_date = scrapy.Field() # 问答统计 # 回答最高得票数 answers_top_score = scrapy.Field() # 得票数最高的回答对应的问题的标题 answers_top_title = scrapy.Field() # 得票数最高的回答对应的问题的标签 answers_top_tags = scrapy.Field() # 得票数最高的回答对应的问题的内容 answers_top_question = scrapy.Field() # 得票数最高的回答对应的问题的内容 answers_top_content = scrapy.Field()
pipeline.py
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymongo
class SegmentfaultPipeline(object):
# 设定MongoDB集合名称
collection_name = 'userinfo'
def __init__(self,mongo_uri,mongo_db):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
# 通过crawler获取settings.py中设定的MongoDB连接信息
@classmethod
def from_crawler(cls,crawler):
return cls(
mongo_uri = crawler.settings.get('MONGO_URI'),
mongo_db = crawler.settings.get('MONGO_DB','segmentfault')
)
# 当爬虫启动时连接MongoDB
def open_spider(self,spider):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
# 当爬虫关闭时断开MongoDB连接
def close_spider(self,spider):
self.client.close()
# 将Item插入数据库保存
def process_item(self, item, spider):
self.db[self.collection_name].insert_one(dict(item))
return item
settings.py
# -*- coding: utf-8 -*-
# Scrapy settings for segmentfault project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'segmentfault'
SPIDER_MODULES = ['segmentfault.spiders']
NEWSPIDER_MODULE = 'segmentfault.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
CONCURRENT_REQUESTS = 100
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 2
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 32
# CONCURRENT_REQUESTS_PER_IP = 32
# Disable cookies (enabled by default)
# COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
RETRY_ENABLED = False
REDIRECT_ENABLED = False
DOWNLOAD_TIMEOUT = 5
# HTTPALLOW
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
SPIDER_MIDDLEWARES = {
'segmentfault.middlewares.SegmentfaultSpiderMiddleware': 543,
}
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
# 'segmentfault.middlewares.SegmentfaultHttpProxyMiddleware': 543,
'segmentfault.middlewares.SegmentfaultUserAgentMiddleware':643,
'segmentfault.middlewares.SegmentfaultCookiesMiddleware':743,
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': None,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
# 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':None,
}
# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'segmentfault.pipelines.SegmentfaultPipeline': 300,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# # The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# # The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# # The average number of requests Scrapy should be sending in parallel to
# # each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# # Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# 配置MONGODB
MONGO_URI = 'localhost:27017'
MONGO_DB = 'segmentfault'
# 用户列表
USER_LIST = [
("798549150@qq.com","guoqing1010"),
("learnscrapy@163.com","guoqing1010"),
]
# 配置代理列表
PROXY_LIST = [
'http://115.182.212.169:8080',
'http://121.61.25.149:9999',
'http://180.118.247.189:9000',
'http://115.151.3.12:9999',
'http://183.154.213.160:9000',
'http://113.128.9.106:9999',
'http://124.42.68.152:90',
'http://49.70.48.50:9999',
'http://113.128.11.172:9999',
'http://111.177.177.40:9999',
'http://59.62.83.253:9999',
'http://39.107.84.185:8123',
'http://124.94.195.107:9999',
'http://111.177.160.132:9999',
'http://120.25.203.182:7777'
]
USER_AGENT_LIST = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60',
'Opera/8.0 (Windows NT 5.1; U; en)',
'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.50',
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)',
'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36'
]
userinfo.py
# -*- coding: utf-8 -*-
import scrapy
import time
from scrapy import Request
from pymongo import MongoClient
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider,Rule
from scrapy.http import FormRequest
from segmentfault.items import SegmentfaultItem
class UserinfoSpider(CrawlSpider):
name = 'userinfo'
allowed_domains = ['segmentfault.com']
start_urls = ['https://segmentfault.com/u/mybigbigcat/users/following']
rules = (
# 用户主页地址，跟进并进行解析
Rule(LinkExtractor(allow=r'/u/\w+$'),callback='parse_item',follow=True),
# 用户关注列表，跟进列表页面，抓取用户主页地址进行后续操作
# Rule(LinkExtractor(allow=r'/users/followed$'),follow=True),
# 用户粉丝列表，跟进列表页面，抓取用户主页地址进行后续操作
Rule(LinkExtractor(allow=r'/users/following$'),follow=True),
# 跟进其他页面地址
# Rule(LinkExtractor(allow=r'/users/[followed|following]?page=\d+'),follow=True),
)
def start_requests(self):
# 从MongoDB中获取一条cookie，添加到开始方法
client = MongoClient(self.crawler.settings['MONGO_URI'])
db = client[self.crawler.settings['MONGO_DB']]
cookies_collection = db.cookies
# 获取一条cookie
cookies = cookies_collection.find_one()
# cookie中的'Hm_lpvt_e23800c454aa573c0ccb16b52665ac26'参数是当前时间的10位表示法，因此重新填充
cookies['Hm_lpvt_e23800c454aa573c0ccb16b52665ac26'] = str(int(time.time()))
return [Request("https://segmentfault.com",
cookies=cookies,
meta={'cookiejar':1},
callback=self.after_login)]
# 登录之后从start_url中开始抓取数据
def after_login(self,response):
for url in self.start_urls:
return self.make_requests_from_url(url)
# def after_login(self,response):
# yield Request(self.start_urls[0],
# meta={'cookiejar':response.meta['cookiejar']},
# callback=self.parse_item)
def parse_item(self, response):
"""
:param response:
:return:
"""
item = SegmentfaultItem()
# 个人属性模块
profile_head = response.css('.profile__heading')
# 姓名
item['name'] = profile_head.css('h2[class*=name]::text').re_first(r'\w+')
# 声望
item['rank'] = profile_head.css('.profile__rank-btn > span::text').extract_first()
# 学校专业信息
school_info = profile_head.css('.profile__school::text').extract()
if school_info:
# 学校
item['school'] = school_info[0]
# 专业
item['majors'] = school_info[1].strip()
else:
item['school'] = ''
item['majors'] = ''
# 公司职位信息
company_info = profile_head.css('.profile__company::text').extract()
if company_info:
# 公司
item['company'] = company_info[0]
# 职位
item['job'] = company_info[1].strip()
else:
item['company'] = ''
item['job'] = ''
# 个人博客
item['blog'] = profile_head.css('a[class*=other-item-link]::attr(href)').extract_first()
# 统计面板模块
profile_active = response.xpath("//div[@class='col-md-2']")
# 关注人数
item['following'] = profile_active.css('div[class*=info] a > .h5::text').re(r'\d+')[0]
# 粉丝人数
item['fans'] = profile_active.css('div[class*=info] a > .h5::text').re(r'\d+')[1]
# 回答问题数
item['answers'] = profile_active.css('a[href*=answer] .count::text').re_first(r'\d+')
# 提问数
item['questions'] = profile_active.css('a[href*=questions] .count::text').re_first(r'\d+')
# 文章数
item['articles'] = profile_active.css('a[href*=articles] .count::text').re_first(r'\d+')
# 讲座数
item['lives'] = profile_active.css('a[href*=lives] .count::text').re_first(r'\d+')
# 徽章数
item['badges'] = profile_active.css('a[href*=badges] .count::text').re_first(r'\d+')
# 徽章详细页面地址
badge_url = profile_active.css('a[href*=badges]::attr(href)').extract_first()
# 技能面板模块
profile_skill = response.xpath("//div[@class='col-md-3']")
# 技能标签列表
item['skills'] = profile_skill.css('.tag::text').re(r'\w+')
# 获得的点赞数
item['like'] = profile_skill.css('.authlist').re_first(r'获得 (\d+) 次点赞')
# 注册日期
item['register_date'] = profile_skill.css('.profile__skill--other p::text').extract_first()
# if register_time:
# item['register_date'] = ''.join(re.findall(r'\d+',register_time))
# else:
# item['register_date'] = ''
# 产出数据模块
profile_work = response.xpath("//div[@class='col-md-7']")
# 回答获得的最高分
item['answers_top_score'] = profile_work.css('#navAnswer .label::text').re_first(r'\d+')
# 最高分回答对应的问题的标题
item['answers_top_title'] = profile_work.css('#navAnswer div[class*=title-warp] > a::text').extract_first()
# 最高分回答对应的问题的url
answer_url = profile_work.css('#navAnswer div[class*=title-warp] > a::attr(href)').extract_first()
# 将需要继续跟进抓取数据的url与item作为参数传递给相应方法继续抓取数据
request = scrapy.Request(
# 问题详细页url
url=response.urljoin(answer_url),
meta={
# item需要传递
'item':item,
# 徽章的url
'badge_url':response.urljoin(badge_url)},
# 调用parse_ansser继续处理
callback=self.parse_answer)
yield request
def parse_answer(self,response):
# 取出传递的item
item = response.meta['item']
# 取出传递的徽章详细页url
badge_url = response.meta['badge_url']
# 问题标签列表
item['answers_top_tags'] = response.css('.question__title--tag .tag::text').re(r'\w+')
# 先获取组成问题内容的字符串列表
question_content = response.css('.widget-question__item p').re(r'>(.*?)<')
# 拼接后传入item
item['answers_top_question'] = ''.join(question_content)
# 先获取组成答案的字符串列表
answer_content = response.css('.qa-answer > article .answer').re(r'>(.*?)<')
# 拼接后传入item
item['answers_top_content'] = ''.join(answer_content)
# 问题页面内容抓取后继续抓取徽章页内容，并将更新后的item继续传递
request = scrapy.Request(url=badge_url,
meta={'item':item},
callback=self.parse_badge)
yield request
def parse_badge(self,response):
item = response.meta['item']
badge_name = response.css('span.badge span::text').extract()
badge_count = response.css('span[class*=badges-count]::text').re(r'\d+')
name_count = {}
for i in range(len(badge_count)):
name_count[badge_name[i]] = badge_count[i]
item['badges'] = name_count
yield item
middlewars.py
# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
import random
import re
import datetime
import scrapy
import logging
import time
from scrapy.conf import settings
from pymongo import MongoClient
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
import pymongo
logger = logging.getLogger(__name__)
class SegmentfaultSpiderMiddleware(object):
"""
处理Item中保存的三种类型注册日期数据：
1. 注册于 2015年12月12日
2. 注册于 3 天前
3. 注册于 5 小时前
"""
def process_spider_output(self,response,result,spider):
"""
输出response时调用此方法处理item中register_date
:param response:
:param result: 包含item
:param spider:
:return:处理过注册日期的item
"""
for item in result:
# 判断获取的数据是否是scrapy.item类型
if isinstance(item,scrapy.Item):
# 获取当前时间
now = datetime.datetime.now()
register_date = item['register_date']
logger.info("获取注册日志格式为{}".format(register_date))
# 提取注册日期字符串，如'注册于2015年12月12日' => '20151212'
day = ''.join(re.findall(r'\d+',register_date))
# 如果提取数字字符串长度大于4位，则为'注册于2015年12月12日'形式
if len(day) > 4:
date = day
# 如果‘时'在提取的字符串中，则为'注册于8小时前'形式
elif '时' in register_date:
d = now - datetime.timedelta(hours=int(day))
date = d.strftime("%Y%m%d")
# 最后一种情况就是'注册于3天前'形式
else:
d = now - datetime.timedelta(days=int(day))
date = d.strftime("%Y%m%d")
# 更新register_date值
item['register_date'] = date
yield item
class SegmentfaultHttpProxyMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
def __init__(self):
self.proxy_list = settings['PROXY_LIST']
def process_request(self, request, spider):
proxy = random.choice(self.proxy_list)
logger.info('使用代理:{}'.format(proxy))
request.meta['proxy'] = proxy
class SegmentfaultUserAgentMiddleware(object):
def __init__(self):
self.useragent_list = settings['USER_AGENT_LIST']
def process_request(self,request,spider):
user_agent = random.choice(self.useragent_list)
# logger.info('使用的USE USER-AGENT:{}'.format(user_agent))
request.headers['User-Agent'] = user_agent
class SegmentfaultCookiesMiddleware(object):
client = MongoClient(settings['MONGO_URI'])
db = client[settings['MONGO_DB']]
collection = db['cookies']
def get_cookies(self):
"""
随机获取cookies
:return:
"""
cookies = random.choice([cookie for cookie in self.collection.find()])
# 将不需要的"_id"与"_gat"参数删除
cookies.pop('_id')
cookies.pop('_gat')
# 将"Hm_lpvt_e23800c454aa573c0ccb16b52665ac26"填充当前时间
cookies['Hm_lpvt_e23800c454aa573c0ccb16b52665ac26'] = str(int(time.time()))
return cookies
def remove_cookies(self,cookies):
"""
删除已失效的cookies
:param cookies:
:return:
"""
# 随机获取cookies中的一对键值,返回结果是一个元祖
i = cookies.popitem()
# 删除cookies
try:
logger.info("删除cookies{}".format(cookies))
self.collection.remove({i[0]:i[1]})
except Exception as e:
logger.info("No this cookies:{}".format(cookies))
def process_request(self,request,spider):
"""
为每一个request添加一个cookie
:param request:
:param spider:
:return:
"""
cookies = self.get_cookies()
request.cookies = cookies
def process_response(self,request,response,spider):
"""
对于登录失效的情况，可能会重定向到登录页面，这时添加新的cookies继续，将请求放回调度器
:param request:
:param response:
:param spider:
:return:
"""
if response.status in [301,302]:
logger.info("Redirect response:{}".format(response))
redirect_url = response.headers['location']
if b'/user/login' in redirect_url:
logger.info("Cookies失效")
# 请求失败，重新获取一个cookie，添加到request，并停止后续中间件处理此request，将此request放入调度器
new_cookie = self.get_cookies()
logger.info("获取新cookie:{}".format(new_cookie))
# 删除旧cookies
self.remove_cookies(request.cookies)
request.cookies = new_cookie
return request
#
return response
run.py
from scrapy import cmdline
# from segmentfault.get_cookies import GetCookies
from get_cookies import GetCookies
if __name__ == '__main__':
cookies = GetCookies()
cookies.save()
name = 'userinfo'
""
cmd = 'scrapy crawl {}'.format(name)
cmdline.execute(cmd.split())
到此这篇关于Scrapy项目实战之爬取某社区用户详情的文章就介绍到这了,更多相关Scrapy 爬取某社区用户内容请搜索我们以前的文章或继续浏览下面的相关文章希望大家以后多多支持我们！
赞 (0)
相关推荐
-
Scrapy框架爬取西刺代理网免费高匿代理的实现代码
分析 需求: 爬取西刺代理网免费高匿代理,并保存到MySQL数据库中. 这里只爬取前10页中的数据. 思路: 分析网页结构,确定数据提取规则 创建Scrapy项目 编写item,定义数据字段 编写spider,实现数据抓取 编写Pipeline,保存数据到数据库中 配置settings.py文件 运行爬虫项目 代码实现 items.py import scrapy class XicidailiItem(scrapy.Item): # 国家 country=scrapy.Field() # IP
-
python爬虫框架scrapy实战之爬取京东商城进阶篇
前言 之前的一篇文章已经讲过怎样获取链接,怎样获得参数了,详情请看python爬取京东商城普通篇,本文将详细介绍利用python爬虫框架scrapy如何爬取京东商城,下面话不多说了,来看看详细的介绍吧. 代码详解 1.首先应该构造请求,这里使用scrapy.Request,这个方法默认调用的是start_urls构造请求,如果要改变默认的请求,那么必须重载该方法,这个方法的返回值必须是一个可迭代的对象,一般是用yield返回. 代码如下: def start_requests(self): fo
-
使用Python的Scrapy框架十分钟爬取美女图
简介 scrapy 是一个 python 下面功能丰富.使用快捷方便的爬虫框架.用 scrapy 可以快速的开发一个简单的爬虫,官方给出的一个简单例子足以证明其强大: 快速开发 下面开始10分钟倒计时: 当然开始前,可以先看看之前我们写过的 scrapy 入门文章 <零基础写python爬虫之使用Scrapy框架编写爬虫 1. 初始化项目 scrapy startproject mzt cd mzt scrapy genspider meizitu meizitu.com 2. 添加 spide
-
在scrapy中使用phantomJS实现异步爬取的方法
使用selenium能够非常方便的获取网页的ajax内容,并且能够模拟用户点击和输入文本等诸多操作,这在使用scrapy爬取网页的过程中非常有用. 网上将selenium集成到scrapy的文章很多,但是很少有能够实现异步爬取的,下面这段代码就重写了scrapy的downloader,同时实现了selenium的集成以及异步. 使用时需要PhantomJSDownloadHandler添加到配置文件的DOWNLOADER中. # encoding: utf-8 from __future__ i
-
Scrapy基于selenium结合爬取淘宝的实例讲解
在对于淘宝,京东这类网站爬取数据时,通常直接使用发送请求拿回response数据,在解析获取想要的数据时比较难的,因为数据只有在浏览网页的时候才会动态加载,所以要想爬取淘宝京东上的数据,可以使用selenium来进行模拟操作 对于scrapy框架,下载器来说已经没多大用,因为获取的response源码里面没有想要的数据,因为没有加载出来,所以要在请求发给下载中间件的时候直接使用selenium对请求解析,获得完整response直接返回,不经过下载器下载,上代码 from selenium im
-
Python使用Scrapy爬虫框架全站爬取图片并保存本地的实现代码
大家可以在Github上clone全部源码. Github:https://github.com/williamzxl/Scrapy_CrawlMeiziTu Scrapy官方文档:http://scrapy-chs.readthedocs.io/zh_CN/latest/index.html 基本上按照文档的流程走一遍就基本会用了. Step1: 在开始爬取之前,必须创建一个新的Scrapy项目. 进入打算存储代码的目录中,运行下列命令: scrapy startproject CrawlMe
-
使用Scrapy爬取动态数据
对于动态数据的爬取,可以选择selenium和PhantomJS两种方式,本文选择的是PhantomJS. 网址: https://s.taobao.com/search?q=%E7%AC%94%E8%AE%B0%E6%9C%AC%E7%94%B5%E8%84%91&imgfile=&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=a21bo.2017.201856-taobao-it
-
Python使用Scrapy爬取妹子图
Python Scrapy爬虫,听说妹子图挺火,我整站爬取了,上周一共搞了大概8000多张图片.和大家分享一下. 核心爬虫代码 # -*- coding: utf-8 -*- from scrapy.selector import Selector import scrapy from scrapy.contrib.loader import ItemLoader, Identity from fun.items import MeizituItem class MeizituSpider(sc
-
Scrapy框架爬取Boss直聘网Python职位信息的源码
分析 使用CrawlSpider结合LinkExtractor和Rule爬取网页信息 LinkExtractor用于定义链接提取规则,一般使用allow参数即可 LinkExtractor(allow=(), # 使用正则定义提取规则 deny=(), # 排除规则 allow_domains=(), # 限定域名范围 deny_domains=(), # 排除域名范围 restrict_xpaths=(), # 使用xpath定义提取队则 tags=('a', 'area'), attrs=(
-
Scrapy项目实战之爬取某社区用户详情
本文介绍了Scrapy项目实战之爬取某社区用户详情,分享给大家,具有如下: get_cookies.py from selenium import webdriver from pymongo import MongoClient from scrapy.crawler import overridden_settings # from segmentfault import settings import time import settings class GetCookies(object
-
scrapy与selenium结合爬取数据(爬取动态网站)的示例代码
scrapy框架只能爬取静态网站.如需爬取动态网站,需要结合着selenium进行js的渲染,才能获取到动态加载的数据. 如何通过selenium请求url,而不再通过下载器Downloader去请求这个url? 方法:在request对象通过中间件的时候,在中间件内部开始使用selenium去请求url,并且会得到url对应的源码,然后再将 源 代码通过response对象返回,直接交给process_response()进行处理,再交给引擎.过程中相当于后续中间件的process_req
-
如何在scrapy中集成selenium爬取网页的方法
1.背景 我们在爬取网页时一般会使用到三个爬虫库:requests,scrapy,selenium.requests一般用于小型爬虫,scrapy用于构建大的爬虫项目,而selenium主要用来应付负责的页面(复杂js渲染的页面,请求非常难构造,或者构造方式经常变化). 在我们面对大型爬虫项目时,肯定会优选scrapy框架来开发,但是在解析复杂JS渲染的页面时,又很麻烦. 尽管使用selenium浏览器渲染来抓取这样的页面很方便,这种方式下,我们不需要关心页面后台发生了怎样的请求,也不需要分析整
-
Python多线程爬虫实战_爬取糗事百科段子的实例
多线程爬虫:即程序中的某些程序段并行执行, 合理地设置多线程,可以让爬虫效率更高 糗事百科段子普通爬虫和多线程爬虫 分析该网址链接得出: https://www.qiushibaike.com/8hr/page/页码/ 多线程爬虫也就和JAVA的多线程差不多,直接上代码 ''' #此处代码为普通爬虫 import urllib.request import urllib.error import re headers = ("User-Agent","Mozilla/5.0
-
Python爬虫实战之爬取携程评论
一.分析数据源 这里的数据源是指html网页?还是Aajx异步.对于爬虫初学者来说,可能不知道怎么判断,这里辰哥也手把手过一遍. 提示:以下操作均不需要登录(当然登录也可以) 咱们先在浏览器里面搜索携程,然后在携程里面任意搜索一个景点:长隆野生动物世界,这里就以长隆野生动物世界为例,讲解如何去爬取携程评论数据. 页面下方则是评论数据 从上面两张图可以看出,点击评论下一页,浏览器的链接没有变化,说明数据是Ajax异步请求.因此我们就找到了数据是异步加载过来的,这时候需要去network里面是查
-
Python爬虫实战之爬取京东商品数据并实实现数据可视化
一.开发工具 Python版本:3.6.4 相关模块: DecryptLogin模块: argparse模块: 以及一些python自带的模块. 二.环境搭建 安装Python并添加到环境变量,pip安装需要的相关模块即可. 三.原理简介 原理其实挺简单的,首先,我们利用之前开源的DecryptLogin库来实现一下微博的模拟登录操作: '''模拟登录京东''' @staticmethod def login(): lg = login.Login() infos_return, session
-
Python爬虫实战之爬取某宝男装信息
目录 知识点介绍 实现步骤 1. 分析目标网站 2. 获取单个商品界面 3. 获取多个商品界面 4. 获取商品信息 5. 保存到MySQL数据库 完整代码 知识点介绍 本次爬取用到的知识点有: 1. selenium 2. pymysql 3 pyquery 实现步骤 1. 分析目标网站 1. 打开某宝首页, 输入"男装"后点击"搜索", 则跳转到"男装"的搜索界面. 2. 空白处"右击"再点击"检查"审
-
Python实战实现爬取天气数据并完成可视化分析详解
1.实现需求: 从网上(随便一个网址,我爬的网址会在评论区告诉大家,dddd)获取某一年的历史天气信息,包括每天最高气温.最低气温.天气状况.风向等,完成以下功能: (1)将获取的数据信息存储到csv格式的文件中,文件命名为”城市名称.csv”,其中每行数据格式为“日期,最高温,最低温,天气,风向”: (2)在数据中增加“平均温度”一列,其中:平均温度=(最高温+最低温)/2,在同一张图中绘制两个城市一年平均气温走势折线图: (3)统计两个城市各类天气的天数,并绘制条形图进行对比,假设适合旅游的
-
利用Python 爬取股票实时数据详情
东方财富网地址如下: http://quote.eastmoney.com/center/gridlist.html#hs_a_board 我们通过点击该网站的下一页发现,网页内容有变化,但是网站的 URL 却不变,也就是说这里使用了 Ajax 技术,动态从服务器拉取数据,这种方式的好处是可以在不重新加载整幅网页的情况下更新部分数据,减轻网络负荷,加快页面加载速度. 我们通过 F12 来查看网络请求情况,可以很容易的发现,网页上的数据都是通过如下地址请求的 http://38.push2.eas