sql - SPARK Exception thrown in awaitResult -

i running spark locally (i not using mesos), , when running join such d3=join(d1,d2) , d5=(d3, d4) getting the following exception "org.apache.spark.sparkexception: exception thrown in awaitresult”.  googling it, found following 2 related links:

1) https://github.com/apache/spark/commit/947b9020b0d621bc97661a0a056297e6889936d3 2) https://github.com/apache/spark/pull/12433

which both explain why happens nothing solve it. 

a bit more running configuration:

1) using spark-core_2.11, spark-sql_2.11

sparksession spark = sparksession                      .builder()                      .master("local[6]").appname("datasetforcasenew").config("spark.executor.memory", "4g").config("spark.shuffle.blocktransferservice", "nio").getorcreate(); 

 

3) public dataset builddataset(){ ...

// step // join prdds cmpds           dataset<row> prdds_join_cmpds                 = res1                                           .join(res2, (res1.col("prd_asin#100")).equalto(res2.col("cmp_asin")), "inner");                  prdds_join_cmpds.take(1);          // step b // join prdds cmpds dataset<row> prdds_join_cmpds_join                 = prdds_join_cmpds                                           .join(res3, prdds_join_cmpds.col("prd_asin#100").equalto(res3.col("ord_asin")), "inner");         prdds_join_cmpds_join.take(1);         prdds_join_cmpds_join.show();  } 

the exception (see below stack trace) thrown when computation reaches step b, until step fine. 

is there wrong or missing?

thanks in advance. 

best regards, carlo

=== stack trace  tests run: 1, failures: 0, errors: 1, skipped: 0, time elapsed: 422.102 sec <<< failure! testbuilddataset(org.mksmart.amaretto.ml.datasetperhourveronetest)  time elapsed: 421.994 sec  <<< error! org.apache.spark.sparkexception: exception thrown in awaitresult:  @ org.apache.spark.util.threadutils$.awaitresult(threadutils.scala:194) @ org.apache.spark.sql.execution.exchange.broadcastexchangeexec.doexecutebroadcast(broadcastexchangeexec.scala:102) @ org.apache.spark.sql.execution.inputadapter.doexecutebroadcast(wholestagecodegenexec.scala:229) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executebroadcast$1.apply(sparkplan.scala:125) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executebroadcast$1.apply(sparkplan.scala:125) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executequery$1.apply(sparkplan.scala:136) @ org.apache.spark.rdd.rddoperationscope$.withscope(rddoperationscope.scala:151) @ org.apache.spark.sql.execution.sparkplan.executequery(sparkplan.scala:133) @ org.apache.spark.sql.execution.sparkplan.executebroadcast(sparkplan.scala:124) @ org.apache.spark.sql.execution.joins.broadcasthashjoinexec.preparebroadcast(broadcasthashjoinexec.scala:98) @ org.apache.spark.sql.execution.joins.broadcasthashjoinexec.codegeninner(broadcasthashjoinexec.scala:197) @ org.apache.spark.sql.execution.joins.broadcasthashjoinexec.doconsume(broadcasthashjoinexec.scala:82) @ org.apache.spark.sql.execution.codegensupport$class.consume(wholestagecodegenexec.scala:153) @ org.apache.spark.sql.execution.joins.sortmergejoinexec.consume(sortmergejoinexec.scala:35) @ org.apache.spark.sql.execution.joins.sortmergejoinexec.doproduce(sortmergejoinexec.scala:565) @ org.apache.spark.sql.execution.codegensupport$$anonfun$produce$1.apply(wholestagecodegenexec.scala:83) @ org.apache.spark.sql.execution.codegensupport$$anonfun$produce$1.apply(wholestagecodegenexec.scala:78) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executequery$1.apply(sparkplan.scala:136) @ org.apache.spark.rdd.rddoperationscope$.withscope(rddoperationscope.scala:151) @ org.apache.spark.sql.execution.sparkplan.executequery(sparkplan.scala:133) @ org.apache.spark.sql.execution.codegensupport$class.produce(wholestagecodegenexec.scala:78) @ org.apache.spark.sql.execution.joins.sortmergejoinexec.produce(sortmergejoinexec.scala:35) @ org.apache.spark.sql.execution.joins.broadcasthashjoinexec.doproduce(broadcasthashjoinexec.scala:77) @ org.apache.spark.sql.execution.codegensupport$$anonfun$produce$1.apply(wholestagecodegenexec.scala:83) @ org.apache.spark.sql.execution.codegensupport$$anonfun$produce$1.apply(wholestagecodegenexec.scala:78) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executequery$1.apply(sparkplan.scala:136) @ org.apache.spark.rdd.rddoperationscope$.withscope(rddoperationscope.scala:151) @ org.apache.spark.sql.execution.sparkplan.executequery(sparkplan.scala:133) @ org.apache.spark.sql.execution.codegensupport$class.produce(wholestagecodegenexec.scala:78) @ org.apache.spark.sql.execution.joins.broadcasthashjoinexec.produce(broadcasthashjoinexec.scala:38) @ org.apache.spark.sql.execution.wholestagecodegenexec.docodegen(wholestagecodegenexec.scala:304) @ org.apache.spark.sql.execution.wholestagecodegenexec.doexecute(wholestagecodegenexec.scala:343) @ org.apache.spark.sql.execution.sparkplan$$anonfun$execute$1.apply(sparkplan.scala:115) @ org.apache.spark.sql.execution.sparkplan$$anonfun$execute$1.apply(sparkplan.scala:115) @ org.apache.spark.sql.execution.sparkplan$$anonfun$executequery$1.apply(sparkplan.scala:136) @ org.apache.spark.rdd.rddoperationscope$.withscope(rddoperationscope.scala:151) @ org.apache.spark.sql.execution.sparkplan.executequery(sparkplan.scala:133) @ org.apache.spark.sql.execution.sparkplan.execute(sparkplan.scala:114) @ org.apache.spark.sql.execution.sparkplan.getbytearrayrdd(sparkplan.scala:240) @ org.apache.spark.sql.execution.sparkplan.executetake(sparkplan.scala:323) @ org.apache.spark.sql.execution.collectlimitexec.executecollect(limit.scala:38) @ org.apache.spark.sql.dataset$$anonfun$org$apache$spark$sql$dataset$$execute$1$1.apply(dataset.scala:2122) @ org.apache.spark.sql.execution.sqlexecution$.withnewexecutionid(sqlexecution.scala:57) @ org.apache.spark.sql.dataset.withnewexecutionid(dataset.scala:2436) @ org.apache.spark.sql.dataset.org$apache$spark$sql$dataset$$execute$1(dataset.scala:2121) @ org.apache.spark.sql.dataset.org$apache$spark$sql$dataset$$collect(dataset.scala:2128) @ org.apache.spark.sql.dataset$$anonfun$head$1.apply(dataset.scala:1862) @ org.apache.spark.sql.dataset$$anonfun$head$1.apply(dataset.scala:1861) @ org.apache.spark.sql.dataset.withtypedcallback(dataset.scala:2466) @ org.apache.spark.sql.dataset.head(dataset.scala:1861) @ org.apache.spark.sql.dataset.take(dataset.scala:2078) @ org.mksmart.amaretto.ml.datasetperhourverone.builddataset(datasetperhourverone.java:115) @ org.mksmart.amaretto.ml.datasetperhourveronetest.testbuilddataset(datasetperhourveronetest.java:76) @ sun.reflect.nativemethodaccessorimpl.invoke0(native method) @ sun.reflect.nativemethodaccessorimpl.invoke(nativemethodaccessorimpl.java:62) @ sun.reflect.delegatingmethodaccessorimpl.invoke(delegatingmethodaccessorimpl.java:43) @ java.lang.reflect.method.invoke(method.java:497) @ org.junit.runners.model.frameworkmethod$1.runreflectivecall(frameworkmethod.java:50) @ org.junit.internal.runners.model.reflectivecallable.run(reflectivecallable.java:12) @ org.junit.runners.model.frameworkmethod.invokeexplosively(frameworkmethod.java:47) @ org.junit.internal.runners.statements.invokemethod.evaluate(invokemethod.java:17) @ org.junit.internal.runners.statements.runbefores.evaluate(runbefores.java:26) @ org.junit.internal.runners.statements.runafters.evaluate(runafters.java:27) @ org.junit.runners.parentrunner.runleaf(parentrunner.java:325) @ org.junit.runners.blockjunit4classrunner.runchild(blockjunit4classrunner.java:78) @ org.junit.runners.blockjunit4classrunner.runchild(blockjunit4classrunner.java:57) @ org.junit.runners.parentrunner$3.run(parentrunner.java:290) @ org.junit.runners.parentrunner$1.schedule(parentrunner.java:71) @ org.junit.runners.parentrunner.runchildren(parentrunner.java:288) @ org.junit.runners.parentrunner.access$000(parentrunner.java:58) @ org.junit.runners.parentrunner$2.evaluate(parentrunner.java:268) @ org.junit.internal.runners.statements.runbefores.evaluate(runbefores.java:26) @ org.junit.internal.runners.statements.runafters.evaluate(runafters.java:27) @ org.junit.runners.parentrunner.run(parentrunner.java:363) @ org.apache.maven.surefire.junit4.junit4testset.execute(junit4testset.java:53) @ org.apache.maven.surefire.junit4.junit4provider.executetestset(junit4provider.java:123) @ org.apache.maven.surefire.junit4.junit4provider.invoke(junit4provider.java:104) @ sun.reflect.nativemethodaccessorimpl.invoke0(native method) @ sun.reflect.nativemethodaccessorimpl.invoke(nativemethodaccessorimpl.java:62) @ sun.reflect.delegatingmethodaccessorimpl.invoke(delegatingmethodaccessorimpl.java:43) @ java.lang.reflect.method.invoke(method.java:497) @ org.apache.maven.surefire.util.reflectionutils.invokemethodwitharray(reflectionutils.java:164) @ org.apache.maven.surefire.booter.providerfactory$providerproxy.invoke(providerfactory.java:110) @ org.apache.maven.surefire.booter.surefirestarter.invokeprovider(surefirestarter.java:175) @ org.apache.maven.surefire.booter.surefirestarter.runsuitesinprocesswhenforked(surefirestarter.java:107) @ org.apache.maven.surefire.booter.forkedbooter.main(forkedbooter.java:68) caused by: java.util.concurrent.timeoutexception: futures timed out after [300 seconds] @ scala.concurrent.impl.promise$defaultpromise.ready(promise.scala:219) @ scala.concurrent.impl.promise$defaultpromise.result(promise.scala:223) @ scala.concurrent.await$$anonfun$result$1.apply(package.scala:190) @ scala.concurrent.blockcontext$defaultblockcontext$.blockon(blockcontext.scala:53) @ scala.concurrent.await$.result(package.scala:190) @ org.apache.spark.util.threadutils$.awaitresult(threadutils.scala:190) ... 85 more