[学习笔记]
/*没有下面的话， 会报一个错误，java.lang.illegalargumentexception: system memory 259522560 must be at least 4.718592e8（470m）. please use a larger heap size.这是memory不够，导致无法启动sparkcontext*/
conf.set("spark.testing.memory", "2000000000");
javasparkcontext sc = new javasparkcontext(conf);
/*下面的这种倒入的方法也行*/
// javardd<string> text = sc.textfile("hdfs://localhost:9000/readme.txt");
/*原文件是：o1abc 45
o1abc 77
o1abc o1abc */
javardd<string> text = sc.textfile("e://temp//input//friend.txt");
list<string> strlist = text.collect();
/*输出str:o1abc 45
str:o1abc 77
str:o1abc o1abc*/
for (string str : strlist) {
system.out.println("str:" + str);
}
/*interface flatmapfunction<t,r>, iterable<r> call(t t)(注意之后的版本，返回值有所变化。)*/
javardd<string> words = text.flatmap(new flatmapfunction<string, string>() {
/*list的super interface 是java.lang.iterable*/
public iterable<string> call(string line) throws exception {
system.out.println("flatmap once, line is "+line );
string[] wordsarray=line.split(" ");
list<string> wordslist=arrays.aslist(wordsarray);
return wordslist;
}
});
list<string> wordslist = words.collect();
/*输出
flatmap once, line is o1abc 45
flatmap once, line is o1abc 77
flatmap once, line is o1abc o1abc
word:o1abc
word:45
word:o1abc
word:77
word:o1abc
word:o1abc*/
for (string word : wordslist) {
system.out.println("word:" + word);
}
/* http://spark.apache.org/docs/latest/
interface pairfunction<t,k,v>
a function that returns key-value pairs (tuple2<k, v>), and can be used to construct pairrdds.
scala.tuple2<k,v> call(t t)
*/
/*
flatmap once, line is o1abc 45(这句说明前面语句再次被执行)
in tuple2 word: o1abc
文章转载原文：
