今日目标：代码总结

今日总结：

import requests

import pymysql

from bs4 import BeautifulSoup

db = pymysql.connect('127.0.0.1',

                     port=3306,

                     user='root',

                     password='123',

                     db='mytest',

                     charset='utf8')

cursor = db.cursor()

headers={

        "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"

        }

url="http://openaccess.thecvf.com/CVPR2019.py"

html=requests.get(url)

soup=BeautifulSoup(html.content,'html.parser')

soup.a.contents=='pdf'

pdfs=soup.findAll(name="a",text="pdf")

lis = []

jianjie=""

for i,pdf in enumerate(pdfs):

    pdf_name=pdf["href"].split('/')[-1]

    name=pdf_name.split('.')[0].replace("_CVPR_2019_paper","")

    link="http://openaccess.thecvf.com/content_CVPR_2019/html/"+name+"_CVPR_2019_paper.html"

    url1=link

    html1 = requests.get(url1)

    soup1 = BeautifulSoup(html1.content, 'html.parser')

    weizhi = soup1.find('div', attrs={'id':'abstract'})

    if weizhi:

        jianjie =weizhi.get_text();

    print("这是第"+str(i)+"条数据")

    keyword = str(name).split('_')

    keywords = ''

    for k in range(len(keyword)):

        if (k == 0):

            keywords += keyword[k]

        else:

            keywords += ',' + keyword[k]

    info = {}

    info['title'] = name

    info['link'] =link

    info['abstract']=jianjie

    info['keywords']=keywords

    lis.append(info)

cursor = db.cursor()

for i in range(len(lis)):

    cols = ", ".join('`{}`'.format(k) for k in lis[i].keys())

    print(cols)  # '`name`, `age`'

    val_cols = ', '.join('%({})s'.format(k) for k in lis[i].keys())

    print(val_cols)  # '%(name)s, %(age)s'

    sql = "insert into lunwen(%s) values(%s)"

    res_sql = sql % (cols, val_cols)

    print(res_sql)

    cursor.execute(res_sql, lis[i])  # 将字典a传入

    db.commit()

    num=1

    print(num)

    print("成功")

　二、分析、查找关键词

 借助Map存储关键词， key为关键词，value为出现的次数。遍历到相同的关键词value值+1，然后根据value值排序。

dao层：

package dao;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.LinkedHashMap;

import java.util.Map;

import java.util.stream.Collectors;

import  Bean.copy.*;

import jdbc.Util;

import java.sql.Connection;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import java.sql.Statement;

import java.util.ArrayList;

import java.util.List;

import com.sun.xml.internal.ws.policy.privateutil.PolicyUtils.Collections;

public class Dao {

    public static Map getrc()

    {

        String sql="select * from lunwen";

        Mapmap= new HashMap();

        Mapresults= new LinkedHashMap();

        Connection con=null;

        Statement state=null;

        ResultSet rs=null;

        con=Util.getConn();

        try {

            state=con.createStatement();

            rs=state.executeQuery(sql);

            while(rs.next())

            {

                String keywords=rs.getString("keywords");

                String[] split = keywords.split(",");

                for(int i=0;i
                {
                    if(map.get(split[i])==null)
                    {
                        map.put(split[i],0);
                    }
                    else
                    {
                        map.replace(split[i], map.get(split[i])+1);
                    }
                }
            }
        } catch (SQLException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        Util.close(rs, state, con);
        map.entrySet()                
        .stream()               
        .sorted((p1, p2) -> p2.getValue().compareTo(p1.getValue()))                
        .collect(Collectors.toList())
        .forEach(ele -> results.put(ele.getKey(), ele.getValue()));
 
       
        return results;
    }
    
    
    public List list(String keywords) { // 查询所有信息
 
 
        List list = new ArrayList(); // 创建集合
        Connection conn = Util.getConn();
        String sql = "select * from lunwen where keywords like "+"'%"+keywords+"%'"; // SQL查询语句
 
        try {
 
            PreparedStatement pst = conn.prepareStatement(sql);
 
            ResultSet rs = pst.executeQuery();
            
            Data data = null;
            
            while (rs.next()) {
 
 
                String title = rs.getString("title");
                
                String link = rs.getString("link");
                
                String as= rs.getString("abstract");
                
 
                
 
                    data = new Data(title,link,as,keywords);
                
                list.add(data);
 
            }
 
            rs.close(); // 关闭
 
            pst.close(); // 关闭
 
        } catch (SQLException e1) {
 
            e1.printStackTrace(); // 抛出异常
 
        }
 
        return list; // 返回一个集合
 
    }
 
 
 
 
}
　　servlet层：
package servlet;
 
import java.io.IOException;
import java.util.Map;
 
import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
 
import dao.Dao;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
 
@WebServlet("/RcServlet")
public class RcServlet extends HttpServlet {
    private static final long serialVersionUID = 1L;
       
    /**
     * @see HttpServlet#HttpServlet()
     */
    public RcServlet() {
        super();
        // TODO Auto-generated constructor stub
    }
 
    /**
     * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
 
        this.doPost(request, response);
    }
 
    /**
     * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response)
     */
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        request.setCharacterEncoding("utf-8");
        MapsortMap=Dao.getrc();
        JSONArray json =new JSONArray();
        int k=0;
        for (Map.Entry entry : sortMap.entrySet()) 
        {
            JSONObject ob=new JSONObject();
            ob.put("name", entry.getKey());
            ob.put("value", entry.getValue());
            if(!(entry.getKey().equals("for")||entry.getKey().equals("and")||entry.getKey().equals("With")||entry.getKey().equals("of")||entry.getKey().equals("in")||entry.getKey().equals("From")||entry.getKey().equals("A")||entry.getKey().equals("to")||entry.getKey().equals("a")||entry.getKey().equals("the")||entry.getKey().equals("by")))
            {
                json.add(ob);
                k++;
            }
            if(k==10)
                break;
        }
        System.out.println(json.toString());
        
        response.getWriter().write(json.toString());
    
    }
 
}
　　三、生成热词汇云图
<%@ page language="java" contentType="text/html; charset=UTF-8"
    pageEncoding="UTF-8"%>
<%@taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
 
<%request.setCharacterEncoding("utf-8"); 
response.setCharacterEncoding("utf-8");%>






 

     
<script src="js/jquery-3.4.1.min.js"></script>
<script src="js/echarts.min.js"></script>
<script src="js/echarts-cloud.js"></script>

 

 



热词云






 

  
    
      
    
    
      
        
      
    
  


论文连接



${item.title}