WooYun.org

#! /usr/bin/env python
#coding=utf-8
import sys
import requests
import tablib
from bs4 import BeautifulSoup
infolist =[]
url = "http://m.mfa.go.th/ajax/ajaxOrgMember.php?dID="
content = '';
def initCode():
    reload(sys)
    sys.setdefaultencoding('utf8')
    
def getContent(url):
    try:
        response = requests.get(url,timeout=2)
        response.raise_for_status()
        response.encoding = 'utf-8'
        getPersonInfo(response.text)
        sys.stdout.write('ing...\n')
        sys.stdout.flush()
        
    except Exception,ex:
        print Exception,":",ex
def getPersonInfo(html):
    global infolist
    soup = BeautifulSoup(html,'lxml')
    infobox = soup.findAll("dl",attrs={"class":"info"})
    for i in range(len(infobox)-1):
        infodir = {"name":"","position":"","department":"","tel":"","email":""}
        if infobox[i+1].dd==None:
            infodir['name'] = 'name'
        else:
            infodir['name'] = infobox[i+1].dd.text.encode('utf-8')
            
        if infobox[i+1].find(attrs={"class":"position"})==None:
            infodir['position'] = 'position'
        else:
            infodir['position'] = infobox[i+1].find(attrs={"class":"position"}).text.encode('utf-8')
            
        if infobox[i+1].find(attrs={"class":"department"})==None:
            infodir['department'] = 'department'
        else:
            infodir['department'] = infobox[i+1].find(attrs={"class":"department"}).text.encode('utf-8')
        
        if infobox[i+1].find(attrs={"class":"tel"})==None:
            infodir['tel'] = 'tel'
        else:
            infodir['tel'] = infobox[i+1].find(attrs={"class":"tel"}).text.encode('utf-8')
            
        if infobox[i+1].find(attrs={"class":"email"})==None:
            infodir['email'] = 'email'
        else:
            infodir['email'] = infobox[i+1].find(attrs={"class":"email"}).text.encode('utf-8')
           
        infolist.append(infodir)
        
def saveContent(infolist):
    headers = ('name','position','department','tel','email')
    data=[]
    data = tablib.Dataset(*data, headers=headers)
    for i in range(len(infolist)):
        #print infolist[i]['name']
        data.append((infolist[i]['name'],infolist[i]['position'],infolist[i]['department'],infolist[i]['tel'],infolist[i]['email']))
        
    with open('data.html', 'a') as f:
        f.write(data.html)
    
if __name__ == "__main__":
    initCode()
    for i in range(300):
        getContent(url+str(i))
    saveContent(infolist)