spark word count

1、创建一个maven工程

2、添加maven依赖

    
        org.apache.spark
        spark-core_2.12
        3.0.0
    


    
        
        
            net.alchim31.maven
            scala-maven-plugin
            3.2.2
            
                
                    
                    
                        testCompile
                    
                
            
        
        
            org.apache.maven.plugins
            maven-assembly-plugin
            3.1.0
            
                
                    jar-with-dependencies
                
            
            
                
                    make-assembly
                    package
                    
                        single
                    
                
            
        
    

3、写代码

package wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Spark_wordcount {


  def main(args: Array[String]): Unit = {
    //创建spark上下文配置对象
    val sparkconf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
    //配置spark上下文环境对象
    val sc: SparkContext = new SparkContext(sparkconf)
    //读取数据
    val fileRdd: RDD[String] = sc.textFile("D:\\gmall2021-parent\\spark\\input\\word.txt")
    //将文件中的数据进行分词
    val wordRDD: RDD[String] = fileRdd.flatMap(_.split(" "))
    //转换数据结构 word =>(word,1)
    val word2: RDD[(String, Int)] = wordRDD.map((_,1))
    //转换后的数据按照相同的单词进行聚合
    val wordCount: RDD[(String, Int)] = word2.reduceByKey(_+_)
    //将聚合结构采集到内存中
    val tuples: Array[(String, Int)] = wordCount.collect()
    //打印
    tuples.foreach(println)


  }

}
4、准备数据
word.text

word count
hello word
hell spark

5、运行 结束
结果

(word,2)
(hello,1)
(spark,1)
(hell,1)
(count,1)