几个性能问题:
1. 商品图像信息加入pgsql, 字段类型: model.BinaryField （django 1.6)
2. django1.6提供数据库长连接功能 CONN_MAX_AGE 参数，可加快访问速度
3. gevent.spawn() 创建所有task，将导致进程被直接Kill掉，因为内存疯长，所涉及的资源达到顶端将被OS杀死,包括：内存overload,file handle...
使用 gevent.pool.Pool控制并发数量
4. postgresql.conf 修改max_connections参数到300并发数
5. 导入数据时，可先去除Table的索引，加快插入速度
6 . django关闭DEBUG模式(会导致内存泄露不释放)
#--coding:utf-8--
'''
yixun_crawler - 易迅网站商品信息收集器
author: scott
date:
目前仅考虑单点设备登录
lxml 的xpath还存在部分问题（还是自己没完全领会呢？）
source-code and db-sample:
http://114.215.178.29/static/projects/crawler/
'''import imp
imp.load_source(
'init',
'../init_script.py')
import gevent
import os,os.path,sys,struct,time,traceback,signal,threading,copy,base64,datetime
# from django.db import connection
# from django.db.models import Sum
from django.db
import transaction
import yixun.models as yixun
from bson.objectid
import ObjectId
import lxml.etree as etree
import urllib2,urlparse
dbfile =
'goods.txt'fdbfile = open(dbfile,
'wb')
class ResourceItem:
def __init__(self,text,href=None,tag=None,parent=None):
self.text = text
self.tag=tag
self.href=href
self.children=[]
self.parent = parent
def scrape_page(url,pageIndex,cat1,cat2,cat3):
print 'scrape_page:',url
req = urllib2.urlopen(url)
data = req.read()
# savefile(data)
html = etree.HTML(data.decode(
'utf-8'))
#page size
curPage = 0
r = html.xpath(
'//*[@id="list"]/div[5]/div[2]/span/b/text()')
if not r:
return False
curPage = r[0]
r = html.xpath(
'//*[@id="list"]/div[5]/div[2]/span/text()')
if not r :
return False
pageNum = int(r[0][1:])
print pageNum,curPage
#有一种情况，传入大于总page数量的值，server会返回第一个page
if pageIndex > pageNum:
return False
#检索品牌
goods = html.xpath(u
"//div[@class='mod_goods']")
if not goods:
print 'skipped..' return False
for g
in goods:
for e
in g.getchildren():
if e.get(
'class') ==
'mod_goods_info':
#一下search动作用xpath无法实现，所以只好挨个查找
name =
'' price =None
link =
'' for p
in e.getchildren():
if p.get(
'class')==
'mod_goods_tit':
a= p.getchildren()[0]
name = a.text.encode(
'utf-8')
link = a.get(
'href')
if p.get(
'class')==
'mod_goods_price':
price = p.getchildren()[0].getchildren()[1].text.encode(
'utf-8')
if name
and price
and link:
# print name , price ,link
text =
"%s || %s || %s || %s || %s || %s\n"%(cat1,cat2,cat3,name,price,link.strip())
print text
gitem = yixun.GoodsItem()
gitem.cat1 = cat1
gitem.cat2 = cat2
gitem.cat3 = cat3
gitem.name = name
gitem.cat5 = link
try:
gitem.price = float(price)
except:
pass gitem.save()
# fdbfile.write(text)
# fdbfile.flush()
return True
# ss= p.xpath('..//dd/a')
'''
http://searchex.yixun.com/705740t705741-1-/?YTAG=2.1738456040037
http://searchex.yixun.com/html?path=705740t705741&area=1&sort=0&show=0&page=2&size=40&pf=0&as=0&charset=utf-8&YTAG=2.1738456040037#list
http://searchex.yixun.com/html?path=705740t705741&area=1&sort=0&show=0&page=1&size=40&pf=0&as=0&charset=utf-8&YTAG=2.1738456040037#list
'''def scrape_cat(cat,yPageId,yPageLevel,tag,cat1,cat2,cat3):
try:
print cat.href
#parse url
url = cat.href
fs = urlparse.urlparse(url)
path,qs=fs[2],fs[4]
cat_idx = path[1:].split(
'-')[0]
# tag = qs.split('=')[1]
tag =
"%s.%s%s"%(yPageLevel,yPageId,tag)
#make path url
for page
in range(1,500):
url =
"http://searchex.yixun.com/html?path=%s&area=1&sort=0&show=0&page=%s&size=40&pf=0&as=0&charset=utf-8&YTAG=%s#list"%(cat_idx,page,tag)
if not scrape_page(url,page,cat1,cat2,cat3):
break return except:
traceback.print_exc()
# print 'page is null,skipped..'
def savefile(d,filename=
'sample.html'):
f = open(filename,
'w')
f.write(d)
f.close()
def test():
try:
url =
'http://searchex.yixun.com/705740t705741-1-/?YTAG=2.1738456040037' fs = urlparse.urlparse(url)
path,qs=fs[2],fs[4]
cat_idx = path[1:].split(
'-')[0]
tag = qs.split(
'=')[1]
print cat_idx,tag
return all_url =
'http://searchex.yixun.com/html?YTAG=3.705766287001&path=705882t705893' req = urllib2.urlsplit(all_url)
html = req.read()
savefile(html)
dom = etree.HTML(html.decode(
'utf-8'))
p = dom.xpath(u
"//div[@title='品牌']")[0]
ss= p.xpath(
'..//dd/a')
print ss[0].text.encode(
'utf-8')
except:
traceback.print_exc()
def craw_start():
import re
try:
all_url =
'http://searchex.yixun.com/?YTAG=2.1738456090000' req = urllib2.urlopen(all_url)
html = req.read()
# group = re.search("window\.yPageId ='(.*?)'",html)
yPageId = re.search(
"window\.yPageId\s*=\s*'(\d+?)'",html).group(1)
yPageLevel = re.search(
"window\.yPageLevel\s*=\s*'(\d+?)'",html).group(1)
print yPageId,yPageLevel
dom = etree.HTML(html.decode(
'gb2312'))
all_cats=[]
cat1_list = dom.xpath(
"//div[@class='m_classbox']")
for cat
in cat1_list:
cat1_text = cat.xpath(
'h3/text()')[0]
cat1_e = ResourceItem(cat1_text)
all_cats.append(cat1_e)
print cat1_e.text.encode(
'utf-8')
div = cat.xpath(
"div")[0]
for dl
in div.xpath(
'dl'):
cat2 = dl.xpath(
'dt/a')[0]
cat2_e = ResourceItem(cat2.text,href=cat2.attrib[
'href'],tag=cat2.attrib[
'ytag'],parent=cat1_e)
cat1_e.children.append(cat2_e)
print ' '*4,cat1_e.text.encode(
'utf-8'),cat2_e.href,cat2_e.tag
for cat3
in dl.xpath(
'dd/a'):
cat3_e = ResourceItem(cat3.text,href=cat3.attrib[
'href'],tag=cat3.attrib[
'ytag'],parent=cat2_e)
cat2_e.children.append(cat3_e)
print ' '*8,cat3_e.text.encode(
'utf-8'),cat3_e.href,cat3_e.tag
tasks =[]
for e1
in all_cats:
print '-'*1,e1.text.encode(
'utf-8')
for e2
in e1.children:
print ' '*2 ,e2.text.encode(
'utf-8')
for e3
in e2.children:
print ' '*4,e3.text.encode(
'utf-8')
task = gevent.spawn(scrape_cat,e3,yPageId,yPageLevel,e2.tag,e1.text.encode(
'utf-8'),e2.text.encode(
'utf-8'),e3.text.encode(
'utf-8'))
tasks.append(task)
# scrape_cat(e3,yPageId,yPageLevel,e2.tag,e1.text.encode('utf-8'),e2.text.encode('utf-8'),e3.text.encode('utf-8'))
# return
gevent.joinall(tasks)
except:
traceback.print_exc()
if __name__ ==
'__main__':
craw_start()
# test()
pass