Spark submit failing with Hive - hive

Question

I am trying to get a Spark 1.1.0 program written in Scala to work, but I'm having a hard time with it. I have a Hive query that is very simple:
select json, score from data
When I run the following command from spark-shell everything works (I need the MYSQL_CONN in the driver class path as I'm using Hive with a MySQL metadata store)
bin/spark-shell --master $SPARK_URL --driver-class-path $MYSQL_CONN
import org.apache.spark.sql.hive.HiveContext
val sqlContext = new HiveContext(sc)
sqlContext.sql("select json from data").map(t => t.getString(0)).take(10).foreach(println)
I get ten lines of json just like I want. However, when I run this with spark-submit as follows I get a problem
bin/spark-submit --master $SPARK_URL --class spark.Main --driver-class-path $MYSQL_CONN target/spark-testing-1.0-SNAPSHOT.jar
Here is my whole Spark program
package spark
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.{SparkContext, SparkConf}
object Main {
def main(args: Array[String]) {
val sc = new SparkContext(new SparkConf().setAppName("Gathering Data"))
val sqlContext = new HiveContext(sc)
sqlContext.sql("select json from data").map(t => t.getString(0)).take(10).foreach(println)
}
}
And here is the resultant stack
14/12/01 21:30:04 WARN TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, match1hd17.dc1): java.lang.ClassNotFoundException: spark.Main$$anonfun$main$1
java.net.URLClassLoader$1.run(URLClassLoader.java:200)
java.security.AccessController.doPrivileged(Native Method)
java.net.URLClassLoader.findClass(URLClassLoader.java:188)
java.lang.ClassLoader.loadClass(ClassLoader.java:307)
java.lang.ClassLoader.loadClass(ClassLoader.java:252)
java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320)
java.lang.Class.forName0(Native Method)
java.lang.Class.forName(Class.java:247)
org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:59)
java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575)
java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947)
java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947)
java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:62)
org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:87)
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:57)
org.apache.spark.scheduler.Task.run(Task.scala:54)
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:177)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
14/12/01 21:30:10 ERROR TaskSetManager: Task 0 in stage 0.0 failed 4 times; aborting job
Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 3, match1hd12.dc1m): java.lang.ClassNotFoundException: spark.Main$$anonfun$main$1
java.net.URLClassLoader$1.run(URLClassLoader.java:200)
java.security.AccessController.doPrivileged(Native Method)
java.net.URLClassLoader.findClass(URLClassLoader.java:188)
java.lang.ClassLoader.loadClass(ClassLoader.java:307)
java.lang.ClassLoader.loadClass(ClassLoader.java:252)
java.lang.ClassLoader.loadClassInternal(ClassLoader.java:320)
java.lang.Class.forName0(Native Method)
java.lang.Class.forName(Class.java:247)
org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:59)
java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1575)
java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1496)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1732)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947)
java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1947)
java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1871)
java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1753)
java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1329)
java.io.ObjectInputStream.readObject(ObjectInputStream.java:351)
org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:62)
org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:87)
org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:57)
org.apache.spark.scheduler.Task.run(Task.scala:54)
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:177)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1185)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1174)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1173)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1173)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:688)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:688)
at scala.Option.foreach(Option.scala:236)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:688)
at org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1391)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
at akka.actor.ActorCell.invoke(ActorCell.scala:456)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
at akka.dispatch.Mailbox.run(Mailbox.scala:219)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
I have spent hours on this already, and I have no idea why this only works with spark-shell. I looked at the stderr output on the individual nodes and they have the same cryptic error message. If anyone can shed some light as to why this only works with spark-shell and not spark-submit that would be awesome.
Thanks
UPDATE:
I've been playing around and the following program works fine.
package spark
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.{SparkContext, SparkConf}
object Main {
def main(args: Array[String]) {
val sc = new SparkContext(new SparkConf().setAppName("Gathering Data"))
val sqlContext = new HiveContext(sc)
sqlContext.sql("select json from data").take(10).map(t => t.getString(0)).foreach(println)
}
}
Obviously this won't work for a large amount of data, but it shows that the problem appears to be in the ScehmaRDD.map() function.

Answer 1

It seems there is a problem with the spark context initialization.
Please try the below code:
val sparkConf = new SparkConf().setAppName("Gathering Data");
val sc = new SparkContext(sparkConf);