V2EX = way to explore V2EX 是一个关于分享和探索的地方
Sign Up Now For Existing Member Sign In
推荐学习书目 Learn Python the Hard Way Python Sites PyPI - Python Package Index http://diveintopython.org/toc/index.html Pocoo 值得关注的项目 PyPy Celery Jinja2 Read the Docs gevent pyenv virtualenv Stackless Python Beautiful Soup 结巴中文分词 Green Unicorn Sentry Shovel Pyflakes pytest Python 编程 pep8 Checker Styles PEP 8 Google Python Style Guide Code Style from The Hitchhiker's Guide
This topic created in 2657 days ago, the information mentioned may be changed or developed.
from scrapy.http.headers import Headers
from Espider.tools.get_cookies import get_cookies
import pymongo,random
from Espider.tools.user_agents import user_agents
from fake_useragent import UserAgent
class zhipincookiemiddleware():
def __init__(self, mongodbHost, mongodbPort, mongodbName):
self.mOngodbHost= mongodbHost
self.mOngodbPort= mongodbPort
self.mOngodbName= mongodbName
@classmethod
def from_crawler(cls, crawler):
return cls(mOngodbHost=crawler.settings.get('MONGODB_HOST'), mOngodbPort=crawler.settings.get('MONGODB_PORT'),
mOngodbName=crawler.settings.get('MONGODB_DBNAME'))
def process_request(self, request, spider):
ua=UserAgent()
self.client = pymongo.MongoClient(self.mongodbHost, self.mongodbPort)
self.mOngodb= self.client[self.mongodbName]
self.collection = self.mongodb[spider.name + '_cookie']
self.cookies_str = self.collection.find_one()['cookie']
self.headers = {
"User-Agent":ua.random,
"cookie": random.choice(self.cookies_str)}
request.headers = Headers(self.headers)
框架里面写了一个 cookiemiddleware
我这里写了一个 random cookie，在每次请求的时候 会重新随机一下 cookie 吗？
from Espider.tools.get_cookies import get_cookies
import pymongo,random
from Espider.tools.user_agents import user_agents
from fake_useragent import UserAgent
class zhipincookiemiddleware():
def __init__(self, mongodbHost, mongodbPort, mongodbName):
self.mOngodbHost= mongodbHost
self.mOngodbPort= mongodbPort
self.mOngodbName= mongodbName
@classmethod
def from_crawler(cls, crawler):
return cls(mOngodbHost=crawler.settings.get('MONGODB_HOST'), mOngodbPort=crawler.settings.get('MONGODB_PORT'),
mOngodbName=crawler.settings.get('MONGODB_DBNAME'))
def process_request(self, request, spider):
ua=UserAgent()
self.client = pymongo.MongoClient(self.mongodbHost, self.mongodbPort)
self.mOngodb= self.client[self.mongodbName]
self.collection = self.mongodb[spider.name + '_cookie']
self.cookies_str = self.collection.find_one()['cookie']
self.headers = {
"User-Agent":ua.random,
"cookie": random.choice(self.cookies_str)}
request.headers = Headers(self.headers)
框架里面写了一个 cookiemiddleware
我这里写了一个 random cookie，在每次请求的时候 会重新随机一下 cookie 吗？
Select Language