We are sending a file path as message through azure event hub and when passing received messages to spark.read.json as parameter getting an error message"overloaded method value json with alternatives:"
========================================
import org.apache.spark.sql.types._
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._
import org.apache.spark.eventhubs._
import org.apache.spark.eventhubs.ConnectionStringBuilder
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.spark.sql.DataFrame
val connectionString = ConnectionStringBuilder("Endpoint=sb://tst.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=dfgdfgdfOcXdfgdfghT3dsfgdfgsdRoIM=").setEventHubName("tsthub").build
val customEventhubParameters =EventHubsConf(connectionString).setMaxEventsPerTrigger(5)
val incomingStream = spark.readStream.format("eventhubs").options(customEventhubParameters.toMap).load()
val rawData = incomingStream.selectExpr("cast(body as string) as json")
val filepath = incomingStream.withColumn("Offset", $"offset".cast(LongType))
.withColumn("Time", $"enqueuedTime".cast(TimestampType))
.withColumn("Timestamp", $"enqueuedTime".cast(LongType))
.withColumn("Body", $"body".cast("String"))
.select("Body")
filepath.printSchema
//filepath="\\usimachine\wklyreport.json"
val CHG_DF = spark.read.json(filepath)
CHG_DF.show()
val query = filepath.writeStream.format("console").start()
query.awaitTermination()
Regards
Suresh SM