博客

案例：把sparksql的程序提交到spark的单机模式下执行

package demo01

import org.apache.spark.SparkContext

import org.apache.spark.sql.SQLContext

import org.apache.spark.SparkConf

object Demo02 {


def main(args: Array[String]): Unit = {


//创建配置

val conf = new SparkConf()

conf.setAppName(“Demo02”)      //创建应用程序first

conf.setMaster(“local”)

//                conf.set(“spark.shuffle.manager”,”hash”)

//基于配置生成sc

val sc = new SparkContext(conf)

//创建出sqlContext

val sqlContext = new SQLContext(sc)

import sqlContext.implicits._

val rdd = sc.makeRDD(List((1,”zhang”),(2,”li”),(3,”wang”)))

val df = rdd.toDF(“id”,”name”)

//                df.registerTempTable(“tabx”);

df.createTempView(“tabx”)

val df2 = sqlContext.sql(“select * from tabx order by name”)

val rdd2 = df2.toJavaRDD.repartition(1)

rdd2.saveAsTextFile(“file:///home/hadoop/df.txt”);

//                rdd2.saveAsTextFile(“d:/df”)

sc.stop()

}

}

打成jar–> sql.jar

提交集群运行  ./spark-submit –class Demo02 ./sql.jar

java.lang.ClassNotFoundException: Demo02

at java.net.URLClassLoader.findClass(URLClassLoader.java:381)

at java.lang.ClassLoader.loadClass(ClassLoader.java:424)

at java.lang.ClassLoader.loadClass(ClassLoader.java:357)

at java.lang.Class.forName0(Native Method)

at java.lang.Class.forName(Class.java:348)

at org.apache.spark.util.Utils$.classForName(Utils.scala:230)

at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:732)

at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)

at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)

at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)

at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

解决办法：添加类的全路径名称  ./spark-submit –class demo01.Demo02 ./sql.jar