#

和beautifulsoup类似，首先我们需要得到一个文档树

• 把文本转换成一个文档树对象

from lxml import etree
if __name__ == '__main__':
    doc='''
        <div>
            <ul>
                 <li class="item-0"><a href="link1.html">first itema>li>
                 <li class="item-1"><a href="link2.html">second itema>li>
                 <li class="item-inactive"><a href="link3.html">third itema>li>
                 <li class="item-1"><a href="link4.html">fourth itema>li>
                 <li class="item-0"><a href="link5.html">fifth itema> # 注意，此处缺少一个 li> 闭合标签
             ul>
         div>
        '''

    html = etree.HTML(doc)
    result = etree.tostring(html)
    print(str(result,'utf-8'))

• 把文件转换成一个文档树对象

from lxml import etree

# 读取外部文件 index.html
html = etree.parse('./index.html')
result = etree.tostring(html, pretty_print=True)    #pretty_print=True 会格式化输出
print(result)

均会打印出文档内容

节点、元素、属性、内容#

    /   根节点，节点分隔符，
    //  任意位置
    .   当前节点
    ..  父级节点
    @   属性

通配符#

使用中括号来限定元素，称为谓语

    //a[n] n为大于零的整数，代表子元素排在第n个位置的元素
    //a[last()]   last()  代表子元素排在最后个位置的元素
    //a[last()-]  和上面同理，代表倒数第二个
    //a[position()<3] 位置序号小于3，也就是前两个，这里我们可以看出xpath中的序列是从1开始
    //a[@href]    拥有href的元素
    //a[@href='www.baidu.com']    href属性值为'www.baidu.com'的元素
    //book[@price>2]   price值大于2的元素

from lxml import etree

if __name__ == '__main__':
    doc='''
        

            

                 
first item
                 
second item
                 
third item
                 
fourth item
                 
fifth item # 注意，此处缺少一个 
 闭合标签
             
         
        '''

    html = etree.HTML(doc)
    print(html.xpath("//li"))
    print(html.xpath("//p")) 

【结果为】

[at 0x2b41b749848>, at 0x2b41b749808>, at 0x2b41b749908>, at 0x2b41b749948>, at 0x2b41b749988>]
[]  #没找到p元素

html = etree.HTML(doc)
print(etree.tostring(html.xpath("//li[@class='item-inactive']")[0]))
print(html.xpath("//li[@class='item-inactive']")[0].text)
print(html.xpath("//li[@class='item-inactive']/a")[0].text)
print(html.xpath("//li[@class='item-inactive']/a/text()"))
print(html.xpath("//li[@class='item-inactive']/.."))
print(html.xpath("//li[@class='item-inactive']/../li[@class='item-0']"))

【结果为】

b'
class="item-inactive">"link3.html">third item
\n                 '
None    #因为第三个li下面没有直接text，None
third item  # 
['third item']
[]
[, ]

使用函数#

有的时候，class作为选择条件的时候不合适@class='....' 这个是完全匹配，当网页样式发生变化时，class或许会增加或减少像active的class。用contains就能很方便

from lxml import etree
if __name__ == '__main__':
    doc='''
        <div>
            <ul>
                 <p class="item-0 active"><a href="link1.html">first itema>p>
                 <li class="item-1"><a href="link2.html">second itema>li>
                 <li class="item-inactive"><a href="link3.html">third itema>li>
                 <li class="item-1"><a href="link4.html">fourth itema>li>
                 <li class="item-0"><a href="link5.html">fifth itema> # 注意，此处缺少一个 li> 闭合标签
             ul>
         div>
        '''

    html = etree.HTML(doc)
    print(html.xpath("//*[contains(@class,'item')]"))

【结果为】

[at 0x23f4a9d12c8>, at 0x23f4a9d13c8>, at 0x23f4a9d1408>, at 0x23f4a9d1448>, at 0x23f4a9d1488>]

starts-with#

print(html.xpath("//*[ends-with(@class,'ul')]"))

【结果为】

Traceback (most recent call last):
  File "F:/OneDrive/pprojects/shoes-show-spider/test/xp5_test.py", line 18, in 
    print(html.xpath("//*[ends-with(@class,'ul')]"))
  File "src\lxml\etree.pyx", line 1582, in lxml.etree._Element.xpath
  File "src\lxml\xpath.pxi", line 305, in lxml.etree.XPathElementEvaluator.__call__
  File "src\lxml\xpath.pxi", line 225, in lxml.etree._XPathEvaluatorBase._handle_result
lxml.etree.XPathEvalError: Unregistered function

看来python的lxml并不支持有的xpath函数列表

upper-case#

#最后一个li被限定了
print(html.xpath("//li[last()]/a/text()"))

#会得到所有的``元素的内容，因为每个标签都是各自父元素的最后一个元素。
#本来每个li就只有一个子元素，所以都是最后一个
print(html.xpath("//li/a[last()]/text()"))  

print(html.xpath("//li/a[contains(text(),'third')]"))

['fifth item']
['second item', 'third item', 'fourth item', 'fifth item']
[]

print(html.xpath("//ul/li[@class='item-inactive']/node()"))
print(html.xpath("//ul/node()"))

[0x239a0d197c8>]
['\n                 ', 0x239a0d19788>, '\n                 ', 0x239a0d19888>, '\n                 ', 0x239a0d19908>, '\n                 ', 0x239a0d19948>, '\n                 ', 0x239a0d198c8>, ' 闭合标签\n             ']

print(html.xpath("//a/@href"))
print(html.xpath("//li/@class"))

['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
['item-0 active', 'item-1', 'item-inactive', 'item-1', 'item-0']

//*[count(@*)=1]    属性只有一个的元素（因为元素的属性本身就是子节点)
//*[count(*)=1]    任意一种属性如果为1，就满足
//*[count(a)=1] 子元素中有一个``的元素

If a document does not have a DTD, then no element in the document will have a unique ID.