输入数据:
aa 78
bb 98
aa 80
cc 98
aa 69
cc 87
bb 97
cc 86
aa 97
bb 78
bb 34
cc 85
bb 92
cc 72
bb 32
bb 23
期望输出:
(aa,78)
(aa,80)
(aa,97)
(bb,92)
(bb,97)
(bb,98)
(cc,86)
(cc,87)
(cc,98)
scala代码:
object GroupSortedTopN {def main(args: Array[String]): Unit = {val k = 3val conf = new SparkConf().setMaster("local").setAppName("GroupSortedTopnz")val sc = new SparkContext(conf)val resultRdd = sc.textFile("groupbykey.txt").map(_.split(" ")).filter(_.length == 2).map(arr => (arr(0), arr(1).toInt)).groupByKey().flatMap {case (item1, iter) => {iter.toList.map(_ * -1).sorted.map(_ * -1).take(k).map((item1, _))
// (item1, topKitem2)}}resultRdd.cache()resultRdd.foreachPartition(_.foreach(println))resultRdd.saveAsTextFile("groupByKeyResult")resultRdd.unpersist()}
}