Filter and Map

Filter data

//create person.txt

Name, Age

Vinayak, 35

Nilesh, 37

Raju, 30

Karthik, 28

Shreshta,1

Siddhish, 2

 

 

val personfile = sc.textFile("C:/Users/Redirection/pateln7/Downloads/SPARK/person.txt")

val first = personfile.first

val person = personfile.filter(x=>x!=first)

val personmap = person.map(x=>{

  val splitted = x.split(",")

  val name = splitted(0).trim

  val age = splitted(1).trim.toInt

  (name,age)

})

personmap.foreach(println)

// ========= Method 1 : Scala

val output = personmap.filter(x=>x._2>30)

output.foreach(println)

// ========= Method 2 : DataFrame

//option 1 :

val persondf=personmap.toDF

persondf.show

val output=persondf.filter($"_2">30)

output.show

//option 2 :

val persondf=personmap.toDF("name","age")

persondf.show

val output=persondf.filter("age>30")

output.show

// ========= Method 3 : Spark sql //(didn’t worked in wondows)

persondf.registerTempTable("person")

val sqlcontext = org.apache.spark.sql.SQLContext

// ========= Method 4 : sqlContext

val persondf=personmap.toDF("name","age")

persondf.registerTempTable("Person")

val sqlcontext = persondf.sqlContext

val output = sqlcontext.sql("select * from person where age>30").show

//OR

val persondf=personmap.toDF

persondf.registerTempTable("Person2")

val sqlcontext = persondf.sqlContext

val output = sqlcontext.sql("select * from person2 where _2>30").show