Главная Обзор Помощь
Вход
zhensolid
/
Python-100-Days
1
0
Ответвить 0
Файлы Задачи 0 Запросы на слияние 0 Вики
Просмотр исходного кода

更新了爬虫第1天代码

jackfrued 7 лет назад
Родитель
402e056498
Сommit
452b6f1441
3 измененных файлов с 137 добавлено и 0 удалено
Единый вид Показать статистику Diff
  1. 60 0
      Day66-75/code/example01.py
  2. 50 0
      Day66-75/code/example02.py
  3. 27 0
      Day66-75/code/example03.py

+ 60 - 0
Day66-75/code/example01.py
Просмотреть файл 

@@ -0,0 +1,60 @@
 
+from urllib.error import URLError
 
+from urllib.request import urlopen
 
+
 
+import re
 
+import pymysql
 
+
 
+
 
+def get_page_code(start_url, *, retry_times=3, charsets=('utf-8', )):
 
+    try:
 
+        for charset in charsets:
 
+            try:
 
+                html = urlopen(start_url).read().decode(charset)
 
+                break
 
+            except UnicodeDecodeError:
 
+                html = None
 
+    except URLError as ex:
 
+        print('Error:', ex)
 
+        return get_page_code(start_url, retry_times=retry_times - 1, charsets=charsets) if \
 
+            retry_times > 0 else None
 
+    return html
 
+
 
+
 
+def main():
 
+    url_list = ['http://sports.sohu.com/nba_a.shtml']
 
+    visited_list = set({})
 
+    while len(url_list) > 0:
 
+        current_url = url_list.pop(0)
 
+        visited_list.add(current_url)
 
+        print(current_url)
 
+        html = get_page_code(current_url, charsets=('utf-8', 'gbk', 'gb2312'))
 
+        if html:
 
+            link_regex = re.compile(r'<a[^>]+href=["\'](.*?)["\']', re.IGNORECASE)
 
+            link_list = re.findall(link_regex, html)
 
+            url_list += link_list
 
+            conn = pymysql.connect(host='localhost', port=3306,
 
+                                   db='crawler', user='root',
 
+                                   passwd='123456', charset='utf8')
 
+            try:
 
+                for link in link_list:
 
+                    if link not in visited_list:
 
+                        visited_list.add(link)
 
+                        print(link)
 
+                        html = get_page_code(link, charsets=('utf-8', 'gbk', 'gb2312'))
 
+                        if html:
 
+                            title_regex = re.compile(r'<h1>(.*)<span', re.IGNORECASE)
 
+                            match_list = title_regex.findall(html)
 
+                            if len(match_list) > 0:
 
+                                title = match_list[0]
 
+                                with conn.cursor() as cursor:
 
+                                    cursor.execute('insert into tb_result (rtitle, rurl) values (%s, %s)',
 
+                                               (title, link))
 
+                                conn.commit()
 
+            finally:
 
+                conn.close()
 
+    print('执行完成!')
 
+
 
+
 
+if __name__ == '__main__':
 
+    main()
 
+

+ 50 - 0
Day66-75/code/example02.py
Просмотреть файл 

@@ -0,0 +1,50 @@
 
+from bs4 import BeautifulSoup
 
+
 
+import re
 
+
 
+
 
+def main():
 
+    html = """
 
+    <!DOCTYPE html>
 
+    <html lang="en">
 
+    <head>
 
+        <meta charset="UTF-8">
 
+        <title>首页</title>
 
+    </head>
 
+    <body>
 
+        <h1>Hello, world!</h1>
 
+        <p>Good!!!</p>
 
+        <hr>
 
+        <div>
 
+            <h2>这是一个例子程序</h2>
 
+            <p>静夜思</p>
 
+            <p class="foo">床前明月光</p>
 
+            <p id="bar">疑似地上霜</p>
 
+            <p class="foo">举头望明月</p>
 
+            <div><a href="http://www.baidu.com"><p>低头思故乡</p></a></div>
 
+        </div>
 
+        <a class="foo" href="http://www.qq.com">腾讯网</a>
 
+        <img src="./img/pretty-girl.png" alt="美女">
 
+        <img src="./img/hellokitty.png" alt="凯蒂猫">
 
+        <img src="./static/img/pretty-girl.png" alt="美女">
 
+        <goup>Hello, Goup!</goup>
 
+    </body>
 
+    </html>
 
+    """
 
+    # resp = requests.get('http://sports.sohu.com/nba_a.shtml')
 
+    # html = resp.content.decode('gbk')
 
+    soup = BeautifulSoup(html, 'lxml')
 
+    print(soup.title)
 
+    # JavaScript: document.body.h1
 
+    # JavaScript: document.forms[0]
 
+    print(soup.body.h1)
 
+    print(soup.find_all(re.compile(r'p$')))
 
+    print(soup.find_all('img', {'src': re.compile(r'\./img/\w+.png')}))
 
+    print(soup.find_all(lambda x: len(x.attrs) == 2))
 
+    print(soup.find_all('p', {'class': 'foo'}))
 
+    for elem in soup.select('a[href]'):
 
+        print(elem.attrs['href'])
 
+
 
+
 
+if __name__ == '__main__':
 
+    main()

+ 27 - 0
Day66-75/code/example03.py
Просмотреть файл 

@@ -0,0 +1,27 @@
 
+from bs4 import BeautifulSoup
 
+
 
+import requests
 
+
 
+import re
 
+
 
+
 
+def main():
 
+    # 通过requests第三方库的get方法获取页面
 
+    resp = requests.get('http://sports.sohu.com/nba_a.shtml')
 
+    # 对响应的字节串(bytes)进行解码操作(搜狐的部分页面使用了GBK编码)
 
+    html = resp.content.decode('gbk')
 
+    # 创建BeautifulSoup对象来解析页面(相当于JavaScript的DOM)
 
+    bs = BeautifulSoup(html, 'lxml')
 
+    # 通过CSS选择器语法查找元素并通过循环进行处理
 
+    # for elem in bs.find_all(lambda x: 'test' in x.attrs):
 
+    for elem in bs.select('a[test]'):
 
+        # 通过attrs属性(字典)获取元素的属性值
 
+        link_url = elem.attrs['href']
 
+        resp = requests.get(link_url)
 
+        bs_sub = BeautifulSoup(resp.text, 'lxml')
 
+        # 使用正则表达式对获取的数据做进一步的处理
 
+        print(re.sub(r'[\r\n]', '', bs_sub.find('h1').text))
 
+
 
+
 
+if __name__ == '__main__':
 
+    main()