Beautiful Soup 库

理解

基本元素

from bs4 import BeautifulSoup
soup = BeautifulSoup(demo,'html.parser')
r = requests.get("http://python123.io/ws/demo.html")
r.text
'\r\n\r\n
The demo python introduces several python courses.
\r\n
Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\r\nBasic Python and Advanced Python.
\r\n'
demo = r.text
#bs4的prettify()方法可以使HTML内容更加友好的显示
soup.prettify()  
'\n \n  \n \n \n  
\n   \n    The demo python introduces several python courses.\n   \n  
\n  
\n   Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n   \n    Basic Python\n   \n   and\n   \n    Advanced Python\n   \n   .\n  
\n \n'
print(soup.prettify()) 

 
  
 
 
  
class="title">
   
    The demo python introduces several python courses.
   
  
  
class="course">
   Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:
   class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">
    Basic Python
   
   and
   class="py2" href="http://www.icourse163.org/course/BIT-1001870001" id="link2">
    Advanced Python
   
   .
  
 

>>> soup.title

#.prettify()为HTML文本<>及其内容增加 '\n'
#.prettify()可用于标签，方法 : .prettify()
print(soup.a.prettify())
class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">
 Basic Python

>>> soup.title

# 任何存在于HTML语法中的标签都可用soup.访问获得
# 当文档中存在多个相同对应内容时，soup.返回第一个
>>> tag = soup.a
>>> tag
class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python
>>> tag = soup.p
>>> tag
class="title">The demo python introduces several python courses.
>>> tag = soup.a

>>> tag.attrs
{'href': 'http://www.icourse163.org/course/BIT-268001', 'class': ['py1'], 'id': 'link1'}
#每个都有自己的名字，通过.name获取，得到的名字为字符串类型
>>> soup.a.name
'a'
>>> soup.p.name
'p'
>>> soup.a.parent.parent.name
'body'
#一个可有0个或多个属性，得到的属性表示方式为字典类型
>>> tag.attrs
{'href': 'http://www.icourse163.org/course/BIT-268001', 'class': ['py1'], 'id': 'link1'}
>>> tag.attrs['class']
['py1']
>>> tag.attrs['href']
'http://www.icourse163.org/course/BIT-268001'
>>> type(tag.attrs)
<class 'dict'>
>>> type(tag)
<class 'bs4.element.Tag'>

#Tag的NavigableString

#Tag的Comment

 基于bs4库的HTML内容遍历方法

 >>>标签树的下行遍历    BeautifulSoup类型是标签树的根节点

>>> soup = BeautifulSoup(demo,"html.parser")
>>> soup.head

>>> soup.head.contents
[]
>>> soup.body

class="title">The demo python introduces several python courses.
class="course">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by track
ing the following courses:

class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python and class="py2" href="http://www.icour
se163.org/course/BIT-1001870001" id="link2">Advanced Python.

#######
>>> soup.body.contents
[[0]'\n',[1] 
class="title">The demo python introduces several python courses.
, [2]'\n',[3] 
class="course">Python is a won
derful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:

class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python and class="py2" href="http://www.icour
se163.org/course/BIT-1001870001" id="link2">Advanced Python.
,[4] '\n']
>>> len(soup.body.contents)
5
>>> soup.body.contents[2]
'\n'
###########################################################
for child in soup.body.childern:
    print(child)   #遍历儿子节点
for child in soup.body.descendants:
    print(child)   #遍历子孙节点

>>>标签树的上行遍历

>>> soup = BeautifulSoup(demo,"html.parser")
>>> for parent in soup.a.parents:
    if parent is None:
        print(parent)
    else:
        print(parent.name)

p
body
html
[document]
>>> 

>>>标签树的平行遍历

>>> soup


class="title">The demo python introduces several python courses.
class="course">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:

class="py1" href="http://www.icourse163.org/course/BIT-268001" id="link1">Basic Python and class="py2" href="http://www.icourse163.org/course/BIT-1001870001" id="link2">Advanced Python.

>>> soup.a.next_sibling
' and '
>>> soup.a.next_sibling.next_sibling
class="py2" href="http://www.icourse163.org/course/BIT-1001870001" id="link2">Advanced Python

#标签树的平行遍历
for sibling in soup.a.next_sibling:
    print(sibling)            #遍历后续节点
for sibling in soup.a.previous_sibling:
    print(sibling)           #遍历前续节点