-
大数据云计算
站-
热门城市 全国站>
-
其他省市
-
-
15692118659
小标
2018-12-05
来源 ：
阅读 958
评论 0
摘要：本文主要向大家介绍了【云计算】Spark实现排序，通过具体的内容向大家展现，希望对大家学习云计算有所帮助。
本文主要向大家介绍了【云计算】Spark实现排序，通过具体的内容向大家展现，希望对大家学习云计算有所帮助。
question：用spark对数据进行排序，首先按照颜值的从高到低进行排序，如果颜值相等，在根据年龄的升序排序
1.User类继承ordered，并且序列化
package cn.edu360.spark.day06
import org.apache.log4j.{Level, Logger}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
/**
* 自定义排序
* Created by zhangjingcun on 2018/9/27 17:13.
*/
object CustomSort1 {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
val conf = new SparkConf().setAppName("IPLocation").setMaster("local[*]")
val sc = new SparkContext(conf)
//用spark对数据进行排序，首先按照颜值的从高到低进行排序，如果颜值相等，在根据年龄的升序排序
val users: Array[String] = Array("1,tom,99,34", "2,marry,96,26", "3,mike,98,29", "4,jim,96,30")
//并行化成RDD
val userLines: RDD[String] = sc.makeRDD(users)
//整理数据
val userRdd: RDD[User1] = userLines.map(line => {
val fileds = line.split(",")
val id = fileds(0).toLong
val name = fileds(1)
val fv = fileds(2).toInt
val age = fileds(3).toInt
new User1(id, name, fv, age)
})
//排序
val sorted: RDD[User1] = userRdd.sortBy(u => u)
//收集数据
val result: Array[User1] = sorted.collect()
println(result.toBuffer)
sc.stop()
}
}
class User1 (val id:Long, val name:String, val fv:Int, val age:Int) extends Ordered[User1] with Serializable {
override def compare(that: User1): Int = {
//颜值相等的时候
if (that.fv == this.fv) {
this.age - that.age
} else {
-(this.fv - that.fv)
}
}
override def toString: String = {
s"User:($id, $name, $fv, $age)"
}
}
2.User继承Sorted没有序列化，不需要new
package cn.edu360.spark.day06
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
/**
* Created by zhangjingcun on 2018/9/27 17:32.
*/
object CustomSort2 {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
val conf = new SparkConf().setAppName("IPLocation").setMaster("local[*]")
val sc = new SparkContext(conf)
//用spark对数据进行排序，首先按照颜值的从高到低进行排序，如果颜值相等，在根据年龄的升序排序
val users: Array[String] = Array("1,tom,99,34", "2,marry,96,26", "3,mike,98,29", "4,jim,96,30")
//并行化成RDD
val userLines: RDD[String] = sc.makeRDD(users)
//整理数据
val userRdd: RDD[User2] = userLines.map(line => {
val fileds = line.split(",")
val id = fileds(0).toLong
val name = fileds(1)
val fv = fileds(2).toInt
val age = fileds(3).toInt
User2(id, name, fv, age)
})
//排序
val sorted: RDD[User2] = userRdd.sortBy(u => u)
//收集数据
val result: Array[User2] = sorted.collect()
println(result.toBuffer)
sc.stop()
}
}
//case 可以不使用new关键字
//不需要实现序列化
case class User2 (val id:Long, val name:String, val fv:Int, val age:Int) extends Ordered[User2] {
override def compare(that: User2): Int = {
//颜值相等的时候
if (that.fv == this.fv) {
this.age - that.age
} else {
-(this.fv - that.fv)
}
}
override def toString: String = {
s"User:($id, $name, $fv, $age)"
}
}
3.
package cn.edu360.spark.day06
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
/**
* Created by zhangjingcun on 2018/9/27 17:37.
*/
object CustomSort3 {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
val conf = new SparkConf().setAppName("IPLocation").setMaster("local[*]")
val sc = new SparkContext(conf)
//用spark对数据进行排序，首先按照颜值的从高到低进行排序，如果颜值相等，在根据年龄的升序排序
val users: Array[String] = Array("1,tom,99,34", "2,marry,96,26", "3,mike,98,29", "4,jim,96,30")
//并行化成RDD
val userLines: RDD[String] = sc.makeRDD(users)
//整理数据
val userRdd: RDD[(Long, String, Int, Int)] = userLines.map(line => {
val fileds = line.split(",")
val id = fileds(0).toLong
val name = fileds(1)
val fv = fileds(2).toInt
val age = fileds(3).toInt
(id, name, fv, age)
})
//排序
val sorted: RDD[(Long, String, Int, Int)] = userRdd.sortBy(tp => User3(tp._1, tp._2, tp._3, tp._4))
//收集数据
val result: Array[(Long, String, Int, Int)] = sorted.collect()
println(result.toBuffer)
sc.stop()
}
}
//case 可以不使用new关键字
//不需要实现序列化
case class User3 (val id:Long, val name:String, val fv:Int, val age:Int) extends Ordered[User3] {
override def compare(that: User3): Int = {
//颜值相等的时候
if (that.fv == this.fv) {
this.age - that.age
} else {
-(this.fv - that.fv)
}
}
override def toString: String = {
s"User:($id, $name, $fv, $age)"
}
}
4.
package cn.edu360.spark.day06
import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
/**
* Created by zhangjingcun on 2018/9/27 17:41.
*/
object CustomSort4 {
def main(args: Array[String]): Unit = {
Logger.getLogger("org.apache.spark").setLevel(Level.OFF)
val conf = new SparkConf().setAppName("IPLocation").setMaster("local[*]")
val sc = new SparkContext(conf)
//用spark对数据进行排序，首先按照颜值的从高到低进行排序，如果颜值相等，在根据年龄的升序排序
val users: Array[String] = Array("1,tom,99,34", "2,marry,96,26", "3,mike,98,29", "4,jim,96,30")
//并行化成RDD
val userLines: RDD[String] = sc.makeRDD(users)
//整理数据
val tpRdd: RDD[(Long, String, Int, Int)] = userLines.map(line => {
val fileds = line.split(",")
val id = fileds(0).toLong
val name = fileds(1)
val fv = fileds(2).toInt
val age = fileds(3).toInt
(id, name, fv, age)
})
//利用元祖的比较特点：先比较第一个，如果不相等，按照第一个属性排序，在比较下个属性
implicit val rules = Ordering[(Int, Int)].on[(Long, String, Int, Int)](t => (-t._3, t._4))
val sorted = tpRdd.sortBy(t => t)
//收集数据
val result: Array[(Long, String, Int, Int)] = sorted.collect()
println(result.toBuffer)
sc.stop()
}
}
本文由职坐标整理并发布，希望对同学们有所帮助。了解更多详情请关注职坐标大数据云计算大数据安全频道！
喜欢 | 0
不喜欢 | 0
您输入的评论内容中包含违禁敏感词
我知道了
请输入正确的手机号码
请输入正确的验证码
您今天的短信下发次数太多了，明天再试试吧！
我们会在第一时间安排职业规划师联系您！
您也可以联系我们的职业规划师咨询：
版权所有 职坐标-一站式AI+学习就业服务平台 沪ICP备13042190号-4
上海海同信息科技有限公司 Copyright ©2015 www.zhizuobiao.com，All Rights Reserved.
沪公网安备 31011502005948号
资料领取
答疑解惑
技术交流
职业测评
面试技巧
高薪秘笈
