品牌网站怎么做/中国培训网

项目源码：http://www.haha174.top/article/details/252203
文章地址：https://github.com/haha174/spark.git
world前面已经介绍过了那么来介绍一个稍微那么高级一丢丢的基于排序的worldcount。
其实就是在worldcount 的基础上增加了一个 key-value 反转。和排序
下面给出java 示例

public class WorldCountSoft {public static void main(String[] args){SparkConf cf=new SparkConf().setAppName("WorldCountSoft").setMaster("local");JavaSparkContext sc=new JavaSparkContext(cf);JavaRDD<String> rddList=sc.textFile("C:\\Users\\haha174\\Desktop\\test\\test.txt");JavaRDD<String> words=rddList.flatMap(new FlatMapFunction<String, String>() {@Overridepublic Iterator<String> call(String s) throws Exception {return Arrays.asList(s.split(" ")).iterator();}});JavaPairRDD<String,Integer> pairs=words.mapToPair(new PairFunction<String, String, Integer>() {@Overridepublic Tuple2<String, Integer> call(String s) throws Exception {return new Tuple2<>(s,1);}});JavaPairRDD<String, Integer> wordCounts = pairs.reduceByKey(new Function2<Integer, Integer, Integer>() {private static final long serialVersionUID = 1L;public Integer call(Integer v1, Integer v2) throws Exception {return v1 + v2;}});//按照单词出现次数降序排序。//进行ley -value 的反转映射// 需要将其反转成 （1，hello  格式）JavaPairRDD<Integer,String> resultBefore=wordCounts.mapToPair(new PairFunction<Tuple2<String, Integer>, Integer, String>() {@Overridepublic Tuple2<Integer, String> call(Tuple2<String, Integer> stringIntegerTuple2) throws Exception {return       new Tuple2<Integer, String>(stringIntegerTuple2._2,stringIntegerTuple2._1);}});JavaPairRDD<Integer,String> result=resultBefore.sortByKey(false);result.foreach(new VoidFunction<Tuple2<Integer,String>>() {public void call(Tuple2<Integer,String> stringIntegerTuple2) throws Exception {System.out.println(stringIntegerTuple2._2+"     "+stringIntegerTuple2._1);}});sc.close();}
}

下面给出scala 示例

object WorldCountSoft {def main(args: Array[String]) {val sparkConf = new SparkConf().setAppName("WorldCount").setMaster("local");val sc = new SparkContext(sparkConf);//val textFile = sc.textFile("hdfs://spark1:8020/world-count.txt")val textFile = sc.textFile("C:\\Users\\wchen129\\Desktop\\test\\test.txt");var counts = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)val countWorld= counts.map(word=>(word._2,word._1))val softWorld=  countWorld .sortByKey(false)softWorld.foreach(count=>println(count._1+" appeard "+count._2+" times"))//counts.saveAsTextFile("hdfs://spark1:8020/world-count-result.txt")}
}

欢迎关注，更多福利