python_web_crawler/user.py at master · Xavier-i/python_web_crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import requests
from bs4 import BeautifulSoup
import re
import js

class Quora_Crawler():

    '''
    basic crawler
    '''
    def __init__(self, url, option="print_data_out"):
        '''
        initialize the crawler
        '''

        self.option = option
        self.url = url
        self.header = {}
        self.header[
            "User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0"
        self.header["Referer"] = "www.quora.com"
        # cookie
        self.cookies = js.result


    def send_request(self):
        '''
        send a request to get HTML source
        '''
        added_followee_url = self.url# + "/following"
        try:
            r = requests.get(added_followee_url, cookies=self.cookies,
                             headers=self.header, verify=False)
        except:
            re_crawl_url(self.url)
            return
        content = r.content
        print(content)
        if r.status_code == 200:
            self.parse_user_profile(content)
            #print(r.text)


    def parse_user_profile(self, html_source):
        '''
        parse the user's profile to mongo
        '''

        # initialize variances

        self.user_name = ''
        self.fuser_gender = ''
        self.user_location = ''
        self.user_followees = ''
        self.user_followers = ''
        self.user_questions=''
        self.user_answers = ''
        self.user_be_viewed = ''
        self.user_education_school = ''
        self.user_education_subject = ''
        self.user_employment = ''
        self.user_employment_extra = ''
        self.user_info = ''
        self.user_intro = ''

        soup = BeautifulSoup(html_source,'html.parser')

        self.user_name = soup.find('a',attrs={'class':'user'}).get_text()

        infoSession=soup.find('div',attrs={'class':'AboutSection'}).find('div',attrs={'class':'contents'})
        listofInfo=infoSession.find_all('span', attrs={'class': 'main_text'})

        for singleInfo in listofInfo:
            text = singleInfo.get_text()
            print(text,singleInfo.parent['class'])
            if text.startswith("Works at "):
                self.user_employment=text.replace("Works at ","",1)

            elif text.startswith("Lives in "):
                self.user_location = text.replace("Lives in ","",1)
            elif text.endswith(" answer views"):
                self.user_be_viewed = text.replace(" answer views","",1)
            elif text.startswith("Studied at "):
                self.user_education_school = text.replace("Studied at ","",1)
            elif singleInfo.parent.name=="div":
                if ('WorkCredentialListItem' in singleInfo.parent['class']):
                    self.user_employment = text
                elif ('SchoolCredentialListItem' in singleInfo.parent['class']):
                    self.user_education_school = text
        profileInfoSession = soup.find('div',attrs={'class':'EditableList NavList ProfileNavList'})
        listofProfileInfo = profileInfoSession.find_all('a')

        for singleProfileInfo in listofProfileInfo:
            text = singleProfileInfo.get_text()
            valueList = re.findall('[\d\,]+', text)
            value = valueList[0] if len(valueList)!=0 else ""
            if text.startswith("Answers"):
                self.user_answers = value
            elif text.startswith("Questions"):
                self.user_quetions = value
            elif text.startswith("Followers"):
                self.user_followers = value

            elif text.startswith("Following"):
                self.user_followees =value
        if self.option == "print_data_out":
            self.print_data_out()
        else:
            self.store_data_to_mongo()


     def print_data_out(self):
        '''
        print out the user data
        '''
        print("*" * 60)
        print('user name:%s\n' % self.user_name)
        print('address:%s\n' % self.user_location)
        print("questions:%s\n" % self.user_quetions)
        print("answers:%s\n" % self.user_answers)
        print("num of followers:%s\n" % self.user_followers)
        print("num of following:%s\n" % self.user_followees)
        print("work:%s\n" % self.user_employment)
        print("education:%s\n" % self.user_education_school)
        print("answer be viewed:%s" % self.user_be_viewed)
        print("*" * 60)

spider = Quora_Crawler(url='xxx')

spider.send_request()