博客

• 自定义UDF函数

package com.ws.spark
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.{SparkConf, SparkContext}

object MyUDF{
  def main(args: Array[String]): Unit = {
    val sparkConf: SparkConf = new SparkConf().setAppName("RangeTest").setMaster("local")
    val sparkContext: SparkContext = new SparkContext(sparkConf)

    val hiveContext: HiveContext = new HiveContext(sparkContext)

    //注册一个UDF函数
    hiveContext.udf.register("myAdd",(x : Int)=> x * 100)

    val dataFrame: DataFrame = hiveContext.sql("select myAdd(age) from ws.t_hive")
    //显示4个结果
    dataFrame.show(4)

    sparkContext.stop()
  }
}

• 自定义UDAF函数

package com.ws.spark

import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sq