Index: /sample/hadoop-0.16/WordCount2.java =================================================================== --- /sample/hadoop-0.16/WordCount2.java (revision 24) +++ /sample/hadoop-0.16/WordCount2.java (revision 24) @@ -0,0 +1,151 @@ +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.StringTokenizer; + +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileOutputFormat; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MapReduceBase; +import org.apache.hadoop.mapred.Mapper; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reducer; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.util.StringUtils; + +public class WordCount2 extends Configured { + + public static class Map extends MapReduceBase implements + Mapper { + + static enum Counters { + INPUT_WORDS + } + + private final static IntWritable one = new IntWritable(1); + + private Text word = new Text(); + + private boolean caseSensitive = true; + + private Set patternsToSkip = new HashSet(); + + private long numRecords = 0; + + private String inputFile; + + public void configure(JobConf job) { + caseSensitive = job.getBoolean("wordcount.case.sensitive", true); + inputFile = job.get("map.input.file"); + + if (job.getBoolean("wordcount.skip.patterns", false)) { + Path[] patternsFiles = new Path[0]; + try { + patternsFiles = DistributedCache.getLocalCacheFiles(job); + } catch (IOException ioe) { + System.err + .println("Caught exception while getting cached files: " + + StringUtils.stringifyException(ioe)); + } + for (Path patternsFile : patternsFiles) { + parseSkipFile(patternsFile); + } + } + } + + private void parseSkipFile(Path patternsFile) { + try { + BufferedReader fis = new BufferedReader(new FileReader( + patternsFile.toString())); + String pattern = null; + while ((pattern = fis.readLine()) != null) { + patternsToSkip.add(pattern); + } + } catch (IOException ioe) { + System.err + .println("Caught exception while parsing the cached file '" + + patternsFile + + "' : " + + StringUtils.stringifyException(ioe)); + } + } + + public void map(LongWritable key, Text value, + OutputCollector output, Reporter reporter) + throws IOException { + String line = (caseSensitive) ? value.toString() : value.toString() + .toLowerCase(); + + for (String pattern : patternsToSkip) { + line = line.replaceAll(pattern, ""); + } + + StringTokenizer tokenizer = new StringTokenizer(line); + while (tokenizer.hasMoreTokens()) { + word.set(tokenizer.nextToken()); + output.collect(word, one); + reporter.incrCounter(Counters.INPUT_WORDS, 1); + } + + if ((++numRecords % 100) == 0) { + reporter.setStatus("Finished processing " + numRecords + + " records " + "from the input file: " + inputFile); + } + } + } + + public static class Reduce extends MapReduceBase implements + Reducer { + public void reduce(Text key, Iterator values, + OutputCollector output, Reporter reporter) + throws IOException { + int sum = 0; + while (values.hasNext()) { + sum += values.next().get(); + } + output.collect(key, new IntWritable(sum)); + } + } + + public static void main(String[] args) throws IOException { + String filename = "/user/waue/input/"; + String outputPath = "sample-counts"; + int mapTasks = 20; + int reduceTasks = 1; + + JobConf conf = new JobConf(WordCount2.class); + conf.setJobName("wordcount"); + + conf.setNumMapTasks(mapTasks); + conf.setNumReduceTasks(reduceTasks); + +// conf.setInputPath(new Path(filename)); + FileInputFormat.setInputPaths(conf,new Path(filename)); + conf.setOutputKeyClass(Text.class); + conf.setOutputValueClass(IntWritable.class); + +// conf.setOutputPath(new Path(outputPath)); + FileOutputFormat.setOutputPath( conf, new Path(filename)); + + + conf.setMapperClass(Map.class); + conf.setCombinerClass(Reduce.class); + conf.setReducerClass(Reduce.class); + + // Delete the output directory if it exists already + Path outputDir = new Path(outputPath); + FileSystem.get(conf).delete(outputDir,true); + JobClient.runJob(conf); + } +} Index: /sample/hadoop-0.16/test.java =================================================================== --- /sample/hadoop-0.16/test.java (revision 24) +++ /sample/hadoop-0.16/test.java (revision 24) @@ -0,0 +1,182 @@ +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.StringTokenizer; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseAdmin; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.JobConf; + + +// this example is testing for static valiable value +// it will show the value of static valiable is related to Class, not object + +class testb { + // object variable + public void read() { + System.out.println(test.sna); + } + +} + +public class test { + int na; + + static int sna; + + static String str; + + public test() { + + } + + public test(String st) { + str = st; + } + + public test(int a, int b, String st) { + na = a; + sna = b; + str = st; + + } + + public void print(String str) { + System.out.println(str); + } + + @SuppressWarnings( { "static-access" }) + public void strToken() throws IOException { + test a = new test(1, 2, "a"); + a.print("a.na = " + a.na); + a.print("test.sna = " + test.sna); + + a.print("a.sna = " + a.sna); // warning about it's not a regular + // accessing method + + test b = new test(3, 4, "a"); + b.print("b.na = " + b.na); + b.print("test.sna = " + test.sna); + b.print("b.sna = " + b.sna); // warning about it's not a regular + // accessing method + + a.print("a.sna = " + a.sna); // a.sna = test.sna -> b.sna + String tmp[] = test.str.split(":"); + String Column_Family = tmp[0] + ":"; + a.print("column family = " + Column_Family); + a.print("ori str = " + test.str); + // test fileoutputstream + FileOutputStream out = new FileOutputStream(new File( + "/home/waue/mr-result.txt")); + out.write("hello".getBytes()); + out.close(); + // test randomaccessfile + RandomAccessFile raf = new RandomAccessFile("/home/waue/mr-result.txt", + "rw"); + raf.seek(raf.length()); // move pointer to end + raf.write("\n go \t ahead".getBytes()); + raf.close(); + // test StringTokenizer + StringTokenizer st = new StringTokenizer("this is a test"); + while (st.hasMoreTokens()) { + System.out.println(st.nextToken()); + } + } + + String parseFirstLine(String in, String ou) throws IOException { + BufferedReader fi = new BufferedReader(new FileReader(new File(in))); + BufferedWriter fw = new BufferedWriter(new FileWriter(new File(ou))); + String first_line, data; + first_line = fi.readLine(); + do { + data = fi.readLine(); + if (data == null) { + break; + } else { + fw.write(data + "\n"); + } + } while (true); + fw.close(); + + return first_line; + } + + boolean deleteFile(String str) throws IOException { + File ttt = new File(str); + + if (ttt.exists()) { + if (!ttt.delete()) { + System.err.print("delete file error !"); + } + } else { + System.out.println("file not exit!"); + } + + return true; + } + + void ttt() throws IOException { + Path pi; + JobConf conf = new JobConf(test.class); + FileStatus[] fi = FileSystem.get(conf).listStatus( + new Path("/user/waue/input/")); + for (int i = 0; i < fi.length; i++) { + pi = fi[i].getPath(); + System.out.println(pi.getName()); + } + } + + public static void main(String[] args) throws IOException { + // test a = new test(); + // a.strToken(); + // System.out.println(a.parseFirstLine("/home/waue/test.txt", + // "/home/waue/out.tmp.txt")); + // ttt(); + + + // setup Table name + String table_name = "test_create_table2"; + Text text_table_name = new Text(table_name); + // setup Column Family + String[] Column_Family = { "http:", "url:", "referrer:" }; + HBaseConfiguration conf = new HBaseConfiguration(); + + HBaseAdmin admin = new HBaseAdmin(conf); + boolean b = admin.tableExists(text_table_name); + System.out.println(b); + /* + if (!admin.tableExists(text_table_name)) { + + System.out.println("HTable : " + table_name + + " creating ... please wait"); + HTableDescriptor tableDesc = new HTableDescriptor(table_name); + for (int i = 0; i < Column_Family.length; i++) { + String st = Column_Family[i]; + // check name format "string:" + if (!st.endsWith(":")) { + Column_Family[i] = st + ":"; + System.out.println("normize :" + st + "->" + + Column_Family[i]); + } + // add column family + tableDesc.addFamily(new HColumnDescriptor(Column_Family[i])); + } + admin.createTable(tableDesc); + } else { + System.out.println("table exist!"); + } + */ + + } +}