博客

假数据：放到G:\person.log

http://bigdata.zpark.cn/laozhang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laozhao
http://bigdata.zpark.cn/laoduan
http://bigdata.zpark.cn/laoduan
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/xiaoxu
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://javaee.zpark.cn/laoyang
http://php.zpark.cn/laoli
http://php.zpark.cn/laoliu
http://php.zpark.cn/laoli
http://php.zpark.cn/laoli

package com.fengrui

import java.net.URL
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object MostJobPerson {
  def main(args: Array[String]): Unit = {
    //初始化
    val conf = new SparkConf()
    conf.setAppName("MostJobPerson")
    conf.setMaster("local")
    val sc = new SparkContext(conf)
    //读文件
    val lines: RDD[String] = sc.textFile("G:\\person.log")
    //map():第一次处理数据，把job和person当成key  1当成value
    val jobPersonAndOne: RDD[((String, String), Int)] = lines.map(line => {
      val i: Int = line.lastIndexOf("/")
      val person: String = line.substring(i + 1)
      val urlString: String = line.substring(0, i)
      val url: URL = new URL(urlString)
      val host: String = url.getHost
      val strings: Array[String] = host.split("\\.")
      val job: String = strings(0)
      ((job, person), 1)
    })
    //聚合，把key（job+person）相同的数据聚合到一起，value值累加了
    val reduced: RDD[((String, String), Int)] = jobPersonAndOne.reduceByKey((x, y) => x+y)
    //分组：_代表（（javaee,laoyang）,9）,_1代表（javaee,laoyang）,_1代表javaee，
    val grouped: RDD[(String, Iterable[((String, String), Int)])] = reduced.groupBy(_._1._1)
    //key不变，对没有value  Iterable[((String,String),Int)]进行扫描
    //调用.toList把Iterable迭代器转成list
    val mapval: RDD[(String, List[((String, String), Int)])] = grouped.mapValues(x => {
      val list: List[((String, String), Int)] = x.toList
      print(list)
      //_代表((javaee,xiaoxu),6) ._2代表6 .reverse：倒序
      val sorted: List[((String, String), Int)] = list.sortBy(_._2).reverse
      //.take(3)作用取出前三条
      val tuples: List[((String, String), Int)] = sorted.take(3)
      //不要遗漏最后的tuples,作用是向下传递
      tuples
    })

    //collect满世界去收集
    val tuples: Array[(String, List[((String, String), Int)])] = mapval.collect()
    print(tuples.toBuffer)
    sc.stop()



  }

}

运行结果：